##// END OF EJS Templates
schain
RSS
jroIO_base: File size is verified
Miguel Valdez -
r780:52e19e09d5b8
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1698 +1,1704
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import time, datetime
12 import time, datetime
13 #import h5py
13 #import h5py
14 import traceback
14 import traceback
15
15
16 try:
16 try:
17 from gevent import sleep
17 from gevent import sleep
18 except:
18 except:
19 from time import sleep
19 from time import sleep
20
20
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23
23
24 LOCALTIME = True
24 LOCALTIME = True
25
25
26 def isNumber(cad):
26 def isNumber(cad):
27 """
27 """
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29
29
30 Excepciones:
30 Excepciones:
31 Si un determinado string no puede ser convertido a numero
31 Si un determinado string no puede ser convertido a numero
32 Input:
32 Input:
33 str, string al cual se le analiza para determinar si convertible a un numero o no
33 str, string al cual se le analiza para determinar si convertible a un numero o no
34
34
35 Return:
35 Return:
36 True : si el string es uno numerico
36 True : si el string es uno numerico
37 False : no es un string numerico
37 False : no es un string numerico
38 """
38 """
39 try:
39 try:
40 float( cad )
40 float( cad )
41 return True
41 return True
42 except:
42 except:
43 return False
43 return False
44
44
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 """
46 """
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48
48
49 Inputs:
49 Inputs:
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51
51
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 segundos contados desde 01/01/1970.
53 segundos contados desde 01/01/1970.
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 segundos contados desde 01/01/1970.
55 segundos contados desde 01/01/1970.
56
56
57 Return:
57 Return:
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 fecha especificado, de lo contrario retorna False.
59 fecha especificado, de lo contrario retorna False.
60
60
61 Excepciones:
61 Excepciones:
62 Si el archivo no existe o no puede ser abierto
62 Si el archivo no existe o no puede ser abierto
63 Si la cabecera no puede ser leida.
63 Si la cabecera no puede ser leida.
64
64
65 """
65 """
66 basicHeaderObj = BasicHeader(LOCALTIME)
66 basicHeaderObj = BasicHeader(LOCALTIME)
67
67
68 try:
68 try:
69 fp = open(filename,'rb')
69 fp = open(filename,'rb')
70 except IOError:
70 except IOError:
71 print "The file %s can't be opened" %(filename)
71 print "The file %s can't be opened" %(filename)
72 return 0
72 return 0
73
73
74 sts = basicHeaderObj.read(fp)
74 sts = basicHeaderObj.read(fp)
75 fp.close()
75 fp.close()
76
76
77 if not(sts):
77 if not(sts):
78 print "Skipping the file %s because it has not a valid header" %(filename)
78 print "Skipping the file %s because it has not a valid header" %(filename)
79 return 0
79 return 0
80
80
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 return 0
82 return 0
83
83
84 return 1
84 return 1
85
85
86 def isTimeInRange(thisTime, startTime, endTime):
86 def isTimeInRange(thisTime, startTime, endTime):
87
87
88 if endTime >= startTime:
88 if endTime >= startTime:
89 if (thisTime < startTime) or (thisTime > endTime):
89 if (thisTime < startTime) or (thisTime > endTime):
90 return 0
90 return 0
91
91
92 return 1
92 return 1
93 else:
93 else:
94 if (thisTime < startTime) and (thisTime > endTime):
94 if (thisTime < startTime) and (thisTime > endTime):
95 return 0
95 return 0
96
96
97 return 1
97 return 1
98
98
99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 """
100 """
101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102
102
103 Inputs:
103 Inputs:
104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105
105
106 startDate : fecha inicial del rango seleccionado en formato datetime.date
106 startDate : fecha inicial del rango seleccionado en formato datetime.date
107
107
108 endDate : fecha final del rango seleccionado en formato datetime.date
108 endDate : fecha final del rango seleccionado en formato datetime.date
109
109
110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111
111
112 endTime : tiempo final del rango seleccionado en formato datetime.time
112 endTime : tiempo final del rango seleccionado en formato datetime.time
113
113
114 Return:
114 Return:
115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 fecha especificado, de lo contrario retorna False.
116 fecha especificado, de lo contrario retorna False.
117
117
118 Excepciones:
118 Excepciones:
119 Si el archivo no existe o no puede ser abierto
119 Si el archivo no existe o no puede ser abierto
120 Si la cabecera no puede ser leida.
120 Si la cabecera no puede ser leida.
121
121
122 """
122 """
123
123
124
124
125 try:
125 try:
126 fp = open(filename,'rb')
126 fp = open(filename,'rb')
127 except IOError:
127 except IOError:
128 print "The file %s can't be opened" %(filename)
128 print "The file %s can't be opened" %(filename)
129 return None
129 return None
130
130
131 firstBasicHeaderObj = BasicHeader(LOCALTIME)
131 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 systemHeaderObj = SystemHeader()
132 systemHeaderObj = SystemHeader()
133 radarControllerHeaderObj = RadarControllerHeader()
133 radarControllerHeaderObj = RadarControllerHeader()
134 processingHeaderObj = ProcessingHeader()
134 processingHeaderObj = ProcessingHeader()
135
135
136 lastBasicHeaderObj = BasicHeader(LOCALTIME)
136 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137
137
138 sts = firstBasicHeaderObj.read(fp)
138 sts = firstBasicHeaderObj.read(fp)
139
140 if not(sts):
141 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 return None
143
139 sts = systemHeaderObj.read(fp)
144 sts = systemHeaderObj.read(fp)
140 sts = radarControllerHeaderObj.read(fp)
145 sts = radarControllerHeaderObj.read(fp)
141 sts = processingHeaderObj.read(fp)
146 sts = processingHeaderObj.read(fp)
142
147
148 filesize = os.path.getsize(filename)
149
143 offset = processingHeaderObj.blockSize + 24 #header size
150 offset = processingHeaderObj.blockSize + 24 #header size
144
151
152 if filesize <= offset:
153 print "[Reading] %s: This file has not enough data" %filename
154 return None
155
145 fp.seek(-offset, 2)
156 fp.seek(-offset, 2)
146
157
147 sts = lastBasicHeaderObj.read(fp)
158 sts = lastBasicHeaderObj.read(fp)
148
159
149 fp.close()
160 fp.close()
150
161
151 if not(sts):
162 thisDatetime = lastBasicHeaderObj.datatime
152 print "Skipping the file %s because it has not a valid header" %(filename)
163 thisTime_last_block = thisDatetime.time()
153 return None
154
164
155 thisDatetime = firstBasicHeaderObj.datatime
165 thisDatetime = firstBasicHeaderObj.datatime
156 thisDate = thisDatetime.date()
166 thisDate = thisDatetime.date()
157 thisTime_first_block = thisDatetime.time()
167 thisTime_first_block = thisDatetime.time()
158
168
159 thisDatetime = lastBasicHeaderObj.datatime
160 thisTime_last_block = thisDatetime.time()
161
162
163 #General case
169 #General case
164 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
170 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
165 #-----------o----------------------------o-----------
171 #-----------o----------------------------o-----------
166 # startTime endTime
172 # startTime endTime
167
173
168 if endTime >= startTime:
174 if endTime >= startTime:
169 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
175 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
170 return None
176 return None
171
177
172 return thisDatetime
178 return thisDatetime
173
179
174 #If endTime < startTime then endTime belongs to the next day
180 #If endTime < startTime then endTime belongs to the next day
175
181
176
182
177 #<<<<<<<<<<<o o>>>>>>>>>>>
183 #<<<<<<<<<<<o o>>>>>>>>>>>
178 #-----------o----------------------------o-----------
184 #-----------o----------------------------o-----------
179 # endTime startTime
185 # endTime startTime
180
186
181 if (thisDate == startDate) and (thisTime_last_block < startTime):
187 if (thisDate == startDate) and (thisTime_last_block < startTime):
182 return None
188 return None
183
189
184 if (thisDate == endDate) and (thisTime_first_block > endTime):
190 if (thisDate == endDate) and (thisTime_first_block > endTime):
185 return None
191 return None
186
192
187 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
193 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
188 return None
194 return None
189
195
190 return thisDatetime
196 return thisDatetime
191
197
192 def isFolderInDateRange(folder, startDate=None, endDate=None):
198 def isFolderInDateRange(folder, startDate=None, endDate=None):
193 """
199 """
194 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
200 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
195
201
196 Inputs:
202 Inputs:
197 folder : nombre completo del directorio.
203 folder : nombre completo del directorio.
198 Su formato deberia ser "/path_root/?YYYYDDD"
204 Su formato deberia ser "/path_root/?YYYYDDD"
199
205
200 siendo:
206 siendo:
201 YYYY : Anio (ejemplo 2015)
207 YYYY : Anio (ejemplo 2015)
202 DDD : Dia del anio (ejemplo 305)
208 DDD : Dia del anio (ejemplo 305)
203
209
204 startDate : fecha inicial del rango seleccionado en formato datetime.date
210 startDate : fecha inicial del rango seleccionado en formato datetime.date
205
211
206 endDate : fecha final del rango seleccionado en formato datetime.date
212 endDate : fecha final del rango seleccionado en formato datetime.date
207
213
208 Return:
214 Return:
209 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
215 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
210 fecha especificado, de lo contrario retorna False.
216 fecha especificado, de lo contrario retorna False.
211 Excepciones:
217 Excepciones:
212 Si el directorio no tiene el formato adecuado
218 Si el directorio no tiene el formato adecuado
213 """
219 """
214
220
215 basename = os.path.basename(folder)
221 basename = os.path.basename(folder)
216
222
217 if not isRadarFolder(basename):
223 if not isRadarFolder(basename):
218 print "The folder %s has not the rigth format" %folder
224 print "The folder %s has not the rigth format" %folder
219 return 0
225 return 0
220
226
221 if startDate and endDate:
227 if startDate and endDate:
222 thisDate = getDateFromRadarFolder(basename)
228 thisDate = getDateFromRadarFolder(basename)
223
229
224 if thisDate < startDate:
230 if thisDate < startDate:
225 return 0
231 return 0
226
232
227 if thisDate > endDate:
233 if thisDate > endDate:
228 return 0
234 return 0
229
235
230 return 1
236 return 1
231
237
232 def isFileInDateRange(filename, startDate=None, endDate=None):
238 def isFileInDateRange(filename, startDate=None, endDate=None):
233 """
239 """
234 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
240 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
235
241
236 Inputs:
242 Inputs:
237 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
243 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
238
244
239 Su formato deberia ser "?YYYYDDDsss"
245 Su formato deberia ser "?YYYYDDDsss"
240
246
241 siendo:
247 siendo:
242 YYYY : Anio (ejemplo 2015)
248 YYYY : Anio (ejemplo 2015)
243 DDD : Dia del anio (ejemplo 305)
249 DDD : Dia del anio (ejemplo 305)
244 sss : set
250 sss : set
245
251
246 startDate : fecha inicial del rango seleccionado en formato datetime.date
252 startDate : fecha inicial del rango seleccionado en formato datetime.date
247
253
248 endDate : fecha final del rango seleccionado en formato datetime.date
254 endDate : fecha final del rango seleccionado en formato datetime.date
249
255
250 Return:
256 Return:
251 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
257 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
252 fecha especificado, de lo contrario retorna False.
258 fecha especificado, de lo contrario retorna False.
253 Excepciones:
259 Excepciones:
254 Si el archivo no tiene el formato adecuado
260 Si el archivo no tiene el formato adecuado
255 """
261 """
256
262
257 basename = os.path.basename(filename)
263 basename = os.path.basename(filename)
258
264
259 if not isRadarFile(basename):
265 if not isRadarFile(basename):
260 print "The filename %s has not the rigth format" %filename
266 print "The filename %s has not the rigth format" %filename
261 return 0
267 return 0
262
268
263 if startDate and endDate:
269 if startDate and endDate:
264 thisDate = getDateFromRadarFile(basename)
270 thisDate = getDateFromRadarFile(basename)
265
271
266 if thisDate < startDate:
272 if thisDate < startDate:
267 return 0
273 return 0
268
274
269 if thisDate > endDate:
275 if thisDate > endDate:
270 return 0
276 return 0
271
277
272 return 1
278 return 1
273
279
274 def getFileFromSet(path, ext, set):
280 def getFileFromSet(path, ext, set):
275 validFilelist = []
281 validFilelist = []
276 fileList = os.listdir(path)
282 fileList = os.listdir(path)
277
283
278 # 0 1234 567 89A BCDE
284 # 0 1234 567 89A BCDE
279 # H YYYY DDD SSS .ext
285 # H YYYY DDD SSS .ext
280
286
281 for thisFile in fileList:
287 for thisFile in fileList:
282 try:
288 try:
283 year = int(thisFile[1:5])
289 year = int(thisFile[1:5])
284 doy = int(thisFile[5:8])
290 doy = int(thisFile[5:8])
285 except:
291 except:
286 continue
292 continue
287
293
288 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
294 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
289 continue
295 continue
290
296
291 validFilelist.append(thisFile)
297 validFilelist.append(thisFile)
292
298
293 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
299 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
294
300
295 if len(myfile)!= 0:
301 if len(myfile)!= 0:
296 return myfile[0]
302 return myfile[0]
297 else:
303 else:
298 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
304 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
299 print 'the filename %s does not exist'%filename
305 print 'the filename %s does not exist'%filename
300 print '...going to the last file: '
306 print '...going to the last file: '
301
307
302 if validFilelist:
308 if validFilelist:
303 validFilelist = sorted( validFilelist, key=str.lower )
309 validFilelist = sorted( validFilelist, key=str.lower )
304 return validFilelist[-1]
310 return validFilelist[-1]
305
311
306 return None
312 return None
307
313
308 def getlastFileFromPath(path, ext):
314 def getlastFileFromPath(path, ext):
309 """
315 """
310 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
316 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
311 al final de la depuracion devuelve el ultimo file de la lista que quedo.
317 al final de la depuracion devuelve el ultimo file de la lista que quedo.
312
318
313 Input:
319 Input:
314 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
320 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
315 ext : extension de los files contenidos en una carpeta
321 ext : extension de los files contenidos en una carpeta
316
322
317 Return:
323 Return:
318 El ultimo file de una determinada carpeta, no se considera el path.
324 El ultimo file de una determinada carpeta, no se considera el path.
319 """
325 """
320 validFilelist = []
326 validFilelist = []
321 fileList = os.listdir(path)
327 fileList = os.listdir(path)
322
328
323 # 0 1234 567 89A BCDE
329 # 0 1234 567 89A BCDE
324 # H YYYY DDD SSS .ext
330 # H YYYY DDD SSS .ext
325
331
326 for thisFile in fileList:
332 for thisFile in fileList:
327
333
328 year = thisFile[1:5]
334 year = thisFile[1:5]
329 if not isNumber(year):
335 if not isNumber(year):
330 continue
336 continue
331
337
332 doy = thisFile[5:8]
338 doy = thisFile[5:8]
333 if not isNumber(doy):
339 if not isNumber(doy):
334 continue
340 continue
335
341
336 year = int(year)
342 year = int(year)
337 doy = int(doy)
343 doy = int(doy)
338
344
339 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
345 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
340 continue
346 continue
341
347
342 validFilelist.append(thisFile)
348 validFilelist.append(thisFile)
343
349
344 if validFilelist:
350 if validFilelist:
345 validFilelist = sorted( validFilelist, key=str.lower )
351 validFilelist = sorted( validFilelist, key=str.lower )
346 return validFilelist[-1]
352 return validFilelist[-1]
347
353
348 return None
354 return None
349
355
350 def checkForRealPath(path, foldercounter, year, doy, set, ext):
356 def checkForRealPath(path, foldercounter, year, doy, set, ext):
351 """
357 """
352 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
358 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
353 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
359 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
354 el path exacto de un determinado file.
360 el path exacto de un determinado file.
355
361
356 Example :
362 Example :
357 nombre correcto del file es .../.../D2009307/P2009307367.ext
363 nombre correcto del file es .../.../D2009307/P2009307367.ext
358
364
359 Entonces la funcion prueba con las siguientes combinaciones
365 Entonces la funcion prueba con las siguientes combinaciones
360 .../.../y2009307367.ext
366 .../.../y2009307367.ext
361 .../.../Y2009307367.ext
367 .../.../Y2009307367.ext
362 .../.../x2009307/y2009307367.ext
368 .../.../x2009307/y2009307367.ext
363 .../.../x2009307/Y2009307367.ext
369 .../.../x2009307/Y2009307367.ext
364 .../.../X2009307/y2009307367.ext
370 .../.../X2009307/y2009307367.ext
365 .../.../X2009307/Y2009307367.ext
371 .../.../X2009307/Y2009307367.ext
366 siendo para este caso, la ultima combinacion de letras, identica al file buscado
372 siendo para este caso, la ultima combinacion de letras, identica al file buscado
367
373
368 Return:
374 Return:
369 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
375 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
370 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
376 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
371 para el filename
377 para el filename
372 """
378 """
373 fullfilename = None
379 fullfilename = None
374 find_flag = False
380 find_flag = False
375 filename = None
381 filename = None
376
382
377 prefixDirList = [None,'d','D']
383 prefixDirList = [None,'d','D']
378 if ext.lower() == ".r": #voltage
384 if ext.lower() == ".r": #voltage
379 prefixFileList = ['d','D']
385 prefixFileList = ['d','D']
380 elif ext.lower() == ".pdata": #spectra
386 elif ext.lower() == ".pdata": #spectra
381 prefixFileList = ['p','P']
387 prefixFileList = ['p','P']
382 else:
388 else:
383 return None, filename
389 return None, filename
384
390
385 #barrido por las combinaciones posibles
391 #barrido por las combinaciones posibles
386 for prefixDir in prefixDirList:
392 for prefixDir in prefixDirList:
387 thispath = path
393 thispath = path
388 if prefixDir != None:
394 if prefixDir != None:
389 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
395 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
390 if foldercounter == 0:
396 if foldercounter == 0:
391 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
397 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
392 else:
398 else:
393 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
399 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
394 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
400 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
395 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
401 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
396 fullfilename = os.path.join( thispath, filename ) #formo el path completo
402 fullfilename = os.path.join( thispath, filename ) #formo el path completo
397
403
398 if os.path.exists( fullfilename ): #verifico que exista
404 if os.path.exists( fullfilename ): #verifico que exista
399 find_flag = True
405 find_flag = True
400 break
406 break
401 if find_flag:
407 if find_flag:
402 break
408 break
403
409
404 if not(find_flag):
410 if not(find_flag):
405 return None, filename
411 return None, filename
406
412
407 return fullfilename, filename
413 return fullfilename, filename
408
414
409 def isRadarFolder(folder):
415 def isRadarFolder(folder):
410 try:
416 try:
411 year = int(folder[1:5])
417 year = int(folder[1:5])
412 doy = int(folder[5:8])
418 doy = int(folder[5:8])
413 except:
419 except:
414 return 0
420 return 0
415
421
416 return 1
422 return 1
417
423
418 def isRadarFile(file):
424 def isRadarFile(file):
419 try:
425 try:
420 year = int(file[1:5])
426 year = int(file[1:5])
421 doy = int(file[5:8])
427 doy = int(file[5:8])
422 set = int(file[8:11])
428 set = int(file[8:11])
423 except:
429 except:
424 return 0
430 return 0
425
431
426 return 1
432 return 1
427
433
428 def getDateFromRadarFile(file):
434 def getDateFromRadarFile(file):
429 try:
435 try:
430 year = int(file[1:5])
436 year = int(file[1:5])
431 doy = int(file[5:8])
437 doy = int(file[5:8])
432 set = int(file[8:11])
438 set = int(file[8:11])
433 except:
439 except:
434 return None
440 return None
435
441
436 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
442 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
437 return thisDate
443 return thisDate
438
444
439 def getDateFromRadarFolder(folder):
445 def getDateFromRadarFolder(folder):
440 try:
446 try:
441 year = int(folder[1:5])
447 year = int(folder[1:5])
442 doy = int(folder[5:8])
448 doy = int(folder[5:8])
443 except:
449 except:
444 return None
450 return None
445
451
446 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
452 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
447 return thisDate
453 return thisDate
448
454
449 class JRODataIO:
455 class JRODataIO:
450
456
451 c = 3E8
457 c = 3E8
452
458
453 isConfig = False
459 isConfig = False
454
460
455 basicHeaderObj = None
461 basicHeaderObj = None
456
462
457 systemHeaderObj = None
463 systemHeaderObj = None
458
464
459 radarControllerHeaderObj = None
465 radarControllerHeaderObj = None
460
466
461 processingHeaderObj = None
467 processingHeaderObj = None
462
468
463 dtype = None
469 dtype = None
464
470
465 pathList = []
471 pathList = []
466
472
467 filenameList = []
473 filenameList = []
468
474
469 filename = None
475 filename = None
470
476
471 ext = None
477 ext = None
472
478
473 flagIsNewFile = 1
479 flagIsNewFile = 1
474
480
475 flagDiscontinuousBlock = 0
481 flagDiscontinuousBlock = 0
476
482
477 flagIsNewBlock = 0
483 flagIsNewBlock = 0
478
484
479 fp = None
485 fp = None
480
486
481 firstHeaderSize = 0
487 firstHeaderSize = 0
482
488
483 basicHeaderSize = 24
489 basicHeaderSize = 24
484
490
485 versionFile = 1103
491 versionFile = 1103
486
492
487 fileSize = None
493 fileSize = None
488
494
489 # ippSeconds = None
495 # ippSeconds = None
490
496
491 fileSizeByHeader = None
497 fileSizeByHeader = None
492
498
493 fileIndex = None
499 fileIndex = None
494
500
495 profileIndex = None
501 profileIndex = None
496
502
497 blockIndex = None
503 blockIndex = None
498
504
499 nTotalBlocks = None
505 nTotalBlocks = None
500
506
501 maxTimeStep = 30
507 maxTimeStep = 30
502
508
503 lastUTTime = None
509 lastUTTime = None
504
510
505 datablock = None
511 datablock = None
506
512
507 dataOut = None
513 dataOut = None
508
514
509 blocksize = None
515 blocksize = None
510
516
511 getByBlock = False
517 getByBlock = False
512
518
513 def __init__(self):
519 def __init__(self):
514
520
515 raise NotImplementedError
521 raise NotImplementedError
516
522
517 def run(self):
523 def run(self):
518
524
519 raise NotImplementedError
525 raise NotImplementedError
520
526
521 def getDtypeWidth(self):
527 def getDtypeWidth(self):
522
528
523 dtype_index = get_dtype_index(self.dtype)
529 dtype_index = get_dtype_index(self.dtype)
524 dtype_width = get_dtype_width(dtype_index)
530 dtype_width = get_dtype_width(dtype_index)
525
531
526 return dtype_width
532 return dtype_width
527
533
528 class JRODataReader(JRODataIO):
534 class JRODataReader(JRODataIO):
529
535
530
536
531 online = 0
537 online = 0
532
538
533 realtime = 0
539 realtime = 0
534
540
535 nReadBlocks = 0
541 nReadBlocks = 0
536
542
537 delay = 10 #number of seconds waiting a new file
543 delay = 10 #number of seconds waiting a new file
538
544
539 nTries = 3 #quantity tries
545 nTries = 3 #quantity tries
540
546
541 nFiles = 3 #number of files for searching
547 nFiles = 3 #number of files for searching
542
548
543 path = None
549 path = None
544
550
545 foldercounter = 0
551 foldercounter = 0
546
552
547 flagNoMoreFiles = 0
553 flagNoMoreFiles = 0
548
554
549 datetimeList = []
555 datetimeList = []
550
556
551 __isFirstTimeOnline = 1
557 __isFirstTimeOnline = 1
552
558
553 __printInfo = True
559 __printInfo = True
554
560
555 profileIndex = None
561 profileIndex = None
556
562
557 nTxs = 1
563 nTxs = 1
558
564
559 txIndex = None
565 txIndex = None
560
566
561 def __init__(self):
567 def __init__(self):
562
568
563 """
569 """
564 This class is used to find data files
570 This class is used to find data files
565
571
566 Example:
572 Example:
567 reader = JRODataReader()
573 reader = JRODataReader()
568 fileList = reader.findDataFiles()
574 fileList = reader.findDataFiles()
569
575
570 """
576 """
571 pass
577 pass
572
578
573
579
574 def createObjByDefault(self):
580 def createObjByDefault(self):
575 """
581 """
576
582
577 """
583 """
578 raise NotImplementedError
584 raise NotImplementedError
579
585
580 def getBlockDimension(self):
586 def getBlockDimension(self):
581
587
582 raise NotImplementedError
588 raise NotImplementedError
583
589
584 def __searchFilesOffLine(self,
590 def __searchFilesOffLine(self,
585 path,
591 path,
586 startDate=None,
592 startDate=None,
587 endDate=None,
593 endDate=None,
588 startTime=datetime.time(0,0,0),
594 startTime=datetime.time(0,0,0),
589 endTime=datetime.time(23,59,59),
595 endTime=datetime.time(23,59,59),
590 set=None,
596 set=None,
591 expLabel='',
597 expLabel='',
592 ext='.r',
598 ext='.r',
593 walk=True):
599 walk=True):
594
600
595 self.filenameList = []
601 self.filenameList = []
596 self.datetimeList = []
602 self.datetimeList = []
597
603
598 pathList = []
604 pathList = []
599
605
600 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
606 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
601
607
602 if dateList == []:
608 if dateList == []:
603 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
609 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
604 return None, None
610 return None, None
605
611
606 if len(dateList) > 1:
612 if len(dateList) > 1:
607 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
613 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
608 else:
614 else:
609 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
615 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
610
616
611 filenameList = []
617 filenameList = []
612 datetimeList = []
618 datetimeList = []
613
619
614 for thisPath in pathList:
620 for thisPath in pathList:
615 # thisPath = pathList[pathDict[file]]
621 # thisPath = pathList[pathDict[file]]
616
622
617 fileList = glob.glob1(thisPath, "*%s" %ext)
623 fileList = glob.glob1(thisPath, "*%s" %ext)
618 fileList.sort()
624 fileList.sort()
619
625
620 for file in fileList:
626 for file in fileList:
621
627
622 filename = os.path.join(thisPath,file)
628 filename = os.path.join(thisPath,file)
623
629
624 if not isFileInDateRange(filename, startDate, endDate):
630 if not isFileInDateRange(filename, startDate, endDate):
625 continue
631 continue
626
632
627 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
633 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
628
634
629 if not(thisDatetime):
635 if not(thisDatetime):
630 continue
636 continue
631
637
632 filenameList.append(filename)
638 filenameList.append(filename)
633 datetimeList.append(thisDatetime)
639 datetimeList.append(thisDatetime)
634
640
635 if not(filenameList):
641 if not(filenameList):
636 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
642 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
637 return None, None
643 return None, None
638
644
639 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
645 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
640 print
646 print
641
647
642 for i in range(len(filenameList)):
648 for i in range(len(filenameList)):
643 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
649 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
644
650
645 self.filenameList = filenameList
651 self.filenameList = filenameList
646 self.datetimeList = datetimeList
652 self.datetimeList = datetimeList
647
653
648 return pathList, filenameList
654 return pathList, filenameList
649
655
650 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
656 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
651
657
652 """
658 """
653 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
659 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
654 devuelve el archivo encontrado ademas de otros datos.
660 devuelve el archivo encontrado ademas de otros datos.
655
661
656 Input:
662 Input:
657 path : carpeta donde estan contenidos los files que contiene data
663 path : carpeta donde estan contenidos los files que contiene data
658
664
659 expLabel : Nombre del subexperimento (subfolder)
665 expLabel : Nombre del subexperimento (subfolder)
660
666
661 ext : extension de los files
667 ext : extension de los files
662
668
663 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
669 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
664
670
665 Return:
671 Return:
666 directory : eL directorio donde esta el file encontrado
672 directory : eL directorio donde esta el file encontrado
667 filename : el ultimo file de una determinada carpeta
673 filename : el ultimo file de una determinada carpeta
668 year : el anho
674 year : el anho
669 doy : el numero de dia del anho
675 doy : el numero de dia del anho
670 set : el set del archivo
676 set : el set del archivo
671
677
672
678
673 """
679 """
674 if not os.path.isdir(path):
680 if not os.path.isdir(path):
675 return None, None, None, None, None, None
681 return None, None, None, None, None, None
676
682
677 dirList = []
683 dirList = []
678
684
679 if not walk:
685 if not walk:
680 fullpath = path
686 fullpath = path
681 foldercounter = 0
687 foldercounter = 0
682 else:
688 else:
683 #Filtra solo los directorios
689 #Filtra solo los directorios
684 for thisPath in os.listdir(path):
690 for thisPath in os.listdir(path):
685 if not os.path.isdir(os.path.join(path,thisPath)):
691 if not os.path.isdir(os.path.join(path,thisPath)):
686 continue
692 continue
687 if not isRadarFolder(thisPath):
693 if not isRadarFolder(thisPath):
688 continue
694 continue
689
695
690 dirList.append(thisPath)
696 dirList.append(thisPath)
691
697
692 if not(dirList):
698 if not(dirList):
693 return None, None, None, None, None, None
699 return None, None, None, None, None, None
694
700
695 dirList = sorted( dirList, key=str.lower )
701 dirList = sorted( dirList, key=str.lower )
696
702
697 doypath = dirList[-1]
703 doypath = dirList[-1]
698 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
704 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
699 fullpath = os.path.join(path, doypath, expLabel)
705 fullpath = os.path.join(path, doypath, expLabel)
700
706
701
707
702 print "[Reading] %s folder was found: " %(fullpath )
708 print "[Reading] %s folder was found: " %(fullpath )
703
709
704 if set == None:
710 if set == None:
705 filename = getlastFileFromPath(fullpath, ext)
711 filename = getlastFileFromPath(fullpath, ext)
706 else:
712 else:
707 filename = getFileFromSet(fullpath, ext, set)
713 filename = getFileFromSet(fullpath, ext, set)
708
714
709 if not(filename):
715 if not(filename):
710 return None, None, None, None, None, None
716 return None, None, None, None, None, None
711
717
712 print "[Reading] %s file was found" %(filename)
718 print "[Reading] %s file was found" %(filename)
713
719
714 if not(self.__verifyFile(os.path.join(fullpath, filename))):
720 if not(self.__verifyFile(os.path.join(fullpath, filename))):
715 return None, None, None, None, None, None
721 return None, None, None, None, None, None
716
722
717 year = int( filename[1:5] )
723 year = int( filename[1:5] )
718 doy = int( filename[5:8] )
724 doy = int( filename[5:8] )
719 set = int( filename[8:11] )
725 set = int( filename[8:11] )
720
726
721 return fullpath, foldercounter, filename, year, doy, set
727 return fullpath, foldercounter, filename, year, doy, set
722
728
723 def __setNextFileOffline(self):
729 def __setNextFileOffline(self):
724
730
725 idFile = self.fileIndex
731 idFile = self.fileIndex
726
732
727 while (True):
733 while (True):
728 idFile += 1
734 idFile += 1
729 if not(idFile < len(self.filenameList)):
735 if not(idFile < len(self.filenameList)):
730 self.flagNoMoreFiles = 1
736 self.flagNoMoreFiles = 1
731 # print "[Reading] No more Files"
737 # print "[Reading] No more Files"
732 return 0
738 return 0
733
739
734 filename = self.filenameList[idFile]
740 filename = self.filenameList[idFile]
735
741
736 if not(self.__verifyFile(filename)):
742 if not(self.__verifyFile(filename)):
737 continue
743 continue
738
744
739 fileSize = os.path.getsize(filename)
745 fileSize = os.path.getsize(filename)
740 fp = open(filename,'rb')
746 fp = open(filename,'rb')
741 break
747 break
742
748
743 self.flagIsNewFile = 1
749 self.flagIsNewFile = 1
744 self.fileIndex = idFile
750 self.fileIndex = idFile
745 self.filename = filename
751 self.filename = filename
746 self.fileSize = fileSize
752 self.fileSize = fileSize
747 self.fp = fp
753 self.fp = fp
748
754
749 # print "[Reading] Setting the file: %s"%self.filename
755 # print "[Reading] Setting the file: %s"%self.filename
750
756
751 return 1
757 return 1
752
758
753 def __setNextFileOnline(self):
759 def __setNextFileOnline(self):
754 """
760 """
755 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
761 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
756 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
762 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
757 siguientes.
763 siguientes.
758
764
759 Affected:
765 Affected:
760 self.flagIsNewFile
766 self.flagIsNewFile
761 self.filename
767 self.filename
762 self.fileSize
768 self.fileSize
763 self.fp
769 self.fp
764 self.set
770 self.set
765 self.flagNoMoreFiles
771 self.flagNoMoreFiles
766
772
767 Return:
773 Return:
768 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
774 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
769 1 : si el file fue abierto con exito y esta listo a ser leido
775 1 : si el file fue abierto con exito y esta listo a ser leido
770
776
771 Excepciones:
777 Excepciones:
772 Si un determinado file no puede ser abierto
778 Si un determinado file no puede ser abierto
773 """
779 """
774 nFiles = 0
780 nFiles = 0
775 fileOk_flag = False
781 fileOk_flag = False
776 firstTime_flag = True
782 firstTime_flag = True
777
783
778 self.set += 1
784 self.set += 1
779
785
780 if self.set > 999:
786 if self.set > 999:
781 self.set = 0
787 self.set = 0
782 self.foldercounter += 1
788 self.foldercounter += 1
783
789
784 #busca el 1er file disponible
790 #busca el 1er file disponible
785 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
791 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
786 if fullfilename:
792 if fullfilename:
787 if self.__verifyFile(fullfilename, False):
793 if self.__verifyFile(fullfilename, False):
788 fileOk_flag = True
794 fileOk_flag = True
789
795
790 #si no encuentra un file entonces espera y vuelve a buscar
796 #si no encuentra un file entonces espera y vuelve a buscar
791 if not(fileOk_flag):
797 if not(fileOk_flag):
792 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
798 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
793
799
794 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
800 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
795 tries = self.nTries
801 tries = self.nTries
796 else:
802 else:
797 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
803 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
798
804
799 for nTries in range( tries ):
805 for nTries in range( tries ):
800 if firstTime_flag:
806 if firstTime_flag:
801 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
807 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
802 sleep( self.delay )
808 sleep( self.delay )
803 else:
809 else:
804 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
810 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
805
811
806 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
812 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
807 if fullfilename:
813 if fullfilename:
808 if self.__verifyFile(fullfilename):
814 if self.__verifyFile(fullfilename):
809 fileOk_flag = True
815 fileOk_flag = True
810 break
816 break
811
817
812 if fileOk_flag:
818 if fileOk_flag:
813 break
819 break
814
820
815 firstTime_flag = False
821 firstTime_flag = False
816
822
817 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
823 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
818 self.set += 1
824 self.set += 1
819
825
820 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
826 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
821 self.set = 0
827 self.set = 0
822 self.doy += 1
828 self.doy += 1
823 self.foldercounter = 0
829 self.foldercounter = 0
824
830
825 if fileOk_flag:
831 if fileOk_flag:
826 self.fileSize = os.path.getsize( fullfilename )
832 self.fileSize = os.path.getsize( fullfilename )
827 self.filename = fullfilename
833 self.filename = fullfilename
828 self.flagIsNewFile = 1
834 self.flagIsNewFile = 1
829 if self.fp != None: self.fp.close()
835 if self.fp != None: self.fp.close()
830 self.fp = open(fullfilename, 'rb')
836 self.fp = open(fullfilename, 'rb')
831 self.flagNoMoreFiles = 0
837 self.flagNoMoreFiles = 0
832 # print '[Reading] Setting the file: %s' % fullfilename
838 # print '[Reading] Setting the file: %s' % fullfilename
833 else:
839 else:
834 self.fileSize = 0
840 self.fileSize = 0
835 self.filename = None
841 self.filename = None
836 self.flagIsNewFile = 0
842 self.flagIsNewFile = 0
837 self.fp = None
843 self.fp = None
838 self.flagNoMoreFiles = 1
844 self.flagNoMoreFiles = 1
839 # print '[Reading] No more files to read'
845 # print '[Reading] No more files to read'
840
846
841 return fileOk_flag
847 return fileOk_flag
842
848
843 def setNextFile(self):
849 def setNextFile(self):
844 if self.fp != None:
850 if self.fp != None:
845 self.fp.close()
851 self.fp.close()
846
852
847 if self.online:
853 if self.online:
848 newFile = self.__setNextFileOnline()
854 newFile = self.__setNextFileOnline()
849 else:
855 else:
850 newFile = self.__setNextFileOffline()
856 newFile = self.__setNextFileOffline()
851
857
852 if not(newFile):
858 if not(newFile):
853 print '[Reading] No more files to read'
859 print '[Reading] No more files to read'
854 return 0
860 return 0
855
861
856 print '[Reading] Setting the file: %s' % self.filename
862 print '[Reading] Setting the file: %s' % self.filename
857
863
858 self.__readFirstHeader()
864 self.__readFirstHeader()
859 self.nReadBlocks = 0
865 self.nReadBlocks = 0
860 return 1
866 return 1
861
867
862 def __waitNewBlock(self):
868 def __waitNewBlock(self):
863 """
869 """
864 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
870 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
865
871
866 Si el modo de lectura es OffLine siempre retorn 0
872 Si el modo de lectura es OffLine siempre retorn 0
867 """
873 """
868 if not self.online:
874 if not self.online:
869 return 0
875 return 0
870
876
871 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
877 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
872 return 0
878 return 0
873
879
874 currentPointer = self.fp.tell()
880 currentPointer = self.fp.tell()
875
881
876 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
882 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
877
883
878 for nTries in range( self.nTries ):
884 for nTries in range( self.nTries ):
879
885
880 self.fp.close()
886 self.fp.close()
881 self.fp = open( self.filename, 'rb' )
887 self.fp = open( self.filename, 'rb' )
882 self.fp.seek( currentPointer )
888 self.fp.seek( currentPointer )
883
889
884 self.fileSize = os.path.getsize( self.filename )
890 self.fileSize = os.path.getsize( self.filename )
885 currentSize = self.fileSize - currentPointer
891 currentSize = self.fileSize - currentPointer
886
892
887 if ( currentSize >= neededSize ):
893 if ( currentSize >= neededSize ):
888 self.basicHeaderObj.read(self.fp)
894 self.basicHeaderObj.read(self.fp)
889 return 1
895 return 1
890
896
891 if self.fileSize == self.fileSizeByHeader:
897 if self.fileSize == self.fileSizeByHeader:
892 # self.flagEoF = True
898 # self.flagEoF = True
893 return 0
899 return 0
894
900
895 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
901 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
896 sleep( self.delay )
902 sleep( self.delay )
897
903
898
904
899 return 0
905 return 0
900
906
901 def waitDataBlock(self,pointer_location):
907 def waitDataBlock(self,pointer_location):
902
908
903 currentPointer = pointer_location
909 currentPointer = pointer_location
904
910
905 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
911 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
906
912
907 for nTries in range( self.nTries ):
913 for nTries in range( self.nTries ):
908 self.fp.close()
914 self.fp.close()
909 self.fp = open( self.filename, 'rb' )
915 self.fp = open( self.filename, 'rb' )
910 self.fp.seek( currentPointer )
916 self.fp.seek( currentPointer )
911
917
912 self.fileSize = os.path.getsize( self.filename )
918 self.fileSize = os.path.getsize( self.filename )
913 currentSize = self.fileSize - currentPointer
919 currentSize = self.fileSize - currentPointer
914
920
915 if ( currentSize >= neededSize ):
921 if ( currentSize >= neededSize ):
916 return 1
922 return 1
917
923
918 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
924 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
919 sleep( self.delay )
925 sleep( self.delay )
920
926
921 return 0
927 return 0
922
928
923 def __jumpToLastBlock(self):
929 def __jumpToLastBlock(self):
924
930
925 if not(self.__isFirstTimeOnline):
931 if not(self.__isFirstTimeOnline):
926 return
932 return
927
933
928 csize = self.fileSize - self.fp.tell()
934 csize = self.fileSize - self.fp.tell()
929 blocksize = self.processingHeaderObj.blockSize
935 blocksize = self.processingHeaderObj.blockSize
930
936
931 #salta el primer bloque de datos
937 #salta el primer bloque de datos
932 if csize > self.processingHeaderObj.blockSize:
938 if csize > self.processingHeaderObj.blockSize:
933 self.fp.seek(self.fp.tell() + blocksize)
939 self.fp.seek(self.fp.tell() + blocksize)
934 else:
940 else:
935 return
941 return
936
942
937 csize = self.fileSize - self.fp.tell()
943 csize = self.fileSize - self.fp.tell()
938 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
944 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
939 while True:
945 while True:
940
946
941 if self.fp.tell()<self.fileSize:
947 if self.fp.tell()<self.fileSize:
942 self.fp.seek(self.fp.tell() + neededsize)
948 self.fp.seek(self.fp.tell() + neededsize)
943 else:
949 else:
944 self.fp.seek(self.fp.tell() - neededsize)
950 self.fp.seek(self.fp.tell() - neededsize)
945 break
951 break
946
952
947 # csize = self.fileSize - self.fp.tell()
953 # csize = self.fileSize - self.fp.tell()
948 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
954 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
949 # factor = int(csize/neededsize)
955 # factor = int(csize/neededsize)
950 # if factor > 0:
956 # if factor > 0:
951 # self.fp.seek(self.fp.tell() + factor*neededsize)
957 # self.fp.seek(self.fp.tell() + factor*neededsize)
952
958
953 self.flagIsNewFile = 0
959 self.flagIsNewFile = 0
954 self.__isFirstTimeOnline = 0
960 self.__isFirstTimeOnline = 0
955
961
956 def __setNewBlock(self):
962 def __setNewBlock(self):
957
963
958 if self.fp == None:
964 if self.fp == None:
959 return 0
965 return 0
960
966
961 # if self.online:
967 # if self.online:
962 # self.__jumpToLastBlock()
968 # self.__jumpToLastBlock()
963
969
964 if self.flagIsNewFile:
970 if self.flagIsNewFile:
965 self.lastUTTime = self.basicHeaderObj.utc
971 self.lastUTTime = self.basicHeaderObj.utc
966 return 1
972 return 1
967
973
968 if self.realtime:
974 if self.realtime:
969 self.flagDiscontinuousBlock = 1
975 self.flagDiscontinuousBlock = 1
970 if not(self.setNextFile()):
976 if not(self.setNextFile()):
971 return 0
977 return 0
972 else:
978 else:
973 return 1
979 return 1
974
980
975 currentSize = self.fileSize - self.fp.tell()
981 currentSize = self.fileSize - self.fp.tell()
976 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
982 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
977
983
978 if (currentSize >= neededSize):
984 if (currentSize >= neededSize):
979 self.basicHeaderObj.read(self.fp)
985 self.basicHeaderObj.read(self.fp)
980 self.lastUTTime = self.basicHeaderObj.utc
986 self.lastUTTime = self.basicHeaderObj.utc
981 return 1
987 return 1
982
988
983 if self.__waitNewBlock():
989 if self.__waitNewBlock():
984 self.lastUTTime = self.basicHeaderObj.utc
990 self.lastUTTime = self.basicHeaderObj.utc
985 return 1
991 return 1
986
992
987 if not(self.setNextFile()):
993 if not(self.setNextFile()):
988 return 0
994 return 0
989
995
990 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
996 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
991 self.lastUTTime = self.basicHeaderObj.utc
997 self.lastUTTime = self.basicHeaderObj.utc
992
998
993 self.flagDiscontinuousBlock = 0
999 self.flagDiscontinuousBlock = 0
994
1000
995 if deltaTime > self.maxTimeStep:
1001 if deltaTime > self.maxTimeStep:
996 self.flagDiscontinuousBlock = 1
1002 self.flagDiscontinuousBlock = 1
997
1003
998 return 1
1004 return 1
999
1005
1000 def readNextBlock(self):
1006 def readNextBlock(self):
1001
1007
1002 #Skip block out of startTime and endTime
1008 #Skip block out of startTime and endTime
1003 while True:
1009 while True:
1004 if not(self.__setNewBlock()):
1010 if not(self.__setNewBlock()):
1005 return 0
1011 return 0
1006
1012
1007 if not(self.readBlock()):
1013 if not(self.readBlock()):
1008 return 0
1014 return 0
1009
1015
1010 self.getBasicHeader()
1016 self.getBasicHeader()
1011
1017
1012 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1018 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1013
1019
1014 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1020 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1015 self.processingHeaderObj.dataBlocksPerFile,
1021 self.processingHeaderObj.dataBlocksPerFile,
1016 self.dataOut.datatime.ctime())
1022 self.dataOut.datatime.ctime())
1017 continue
1023 continue
1018
1024
1019 break
1025 break
1020
1026
1021 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1027 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1022 self.processingHeaderObj.dataBlocksPerFile,
1028 self.processingHeaderObj.dataBlocksPerFile,
1023 self.dataOut.datatime.ctime())
1029 self.dataOut.datatime.ctime())
1024 return 1
1030 return 1
1025
1031
1026 def __readFirstHeader(self):
1032 def __readFirstHeader(self):
1027
1033
1028 self.basicHeaderObj.read(self.fp)
1034 self.basicHeaderObj.read(self.fp)
1029 self.systemHeaderObj.read(self.fp)
1035 self.systemHeaderObj.read(self.fp)
1030 self.radarControllerHeaderObj.read(self.fp)
1036 self.radarControllerHeaderObj.read(self.fp)
1031 self.processingHeaderObj.read(self.fp)
1037 self.processingHeaderObj.read(self.fp)
1032
1038
1033 self.firstHeaderSize = self.basicHeaderObj.size
1039 self.firstHeaderSize = self.basicHeaderObj.size
1034
1040
1035 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1041 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1036 if datatype == 0:
1042 if datatype == 0:
1037 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1043 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1038 elif datatype == 1:
1044 elif datatype == 1:
1039 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1045 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1040 elif datatype == 2:
1046 elif datatype == 2:
1041 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1047 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1042 elif datatype == 3:
1048 elif datatype == 3:
1043 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1049 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1044 elif datatype == 4:
1050 elif datatype == 4:
1045 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1051 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1046 elif datatype == 5:
1052 elif datatype == 5:
1047 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1053 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1048 else:
1054 else:
1049 raise ValueError, 'Data type was not defined'
1055 raise ValueError, 'Data type was not defined'
1050
1056
1051 self.dtype = datatype_str
1057 self.dtype = datatype_str
1052 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1058 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1053 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1059 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1054 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1060 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1055 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1061 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1056 self.getBlockDimension()
1062 self.getBlockDimension()
1057
1063
1058 def __verifyFile(self, filename, msgFlag=True):
1064 def __verifyFile(self, filename, msgFlag=True):
1059
1065
1060 msg = None
1066 msg = None
1061
1067
1062 try:
1068 try:
1063 fp = open(filename, 'rb')
1069 fp = open(filename, 'rb')
1064 except IOError:
1070 except IOError:
1065
1071
1066 if msgFlag:
1072 if msgFlag:
1067 print "[Reading] File %s can't be opened" % (filename)
1073 print "[Reading] File %s can't be opened" % (filename)
1068
1074
1069 return False
1075 return False
1070
1076
1071 currentPosition = fp.tell()
1077 currentPosition = fp.tell()
1072 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1078 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1073
1079
1074 if neededSize == 0:
1080 if neededSize == 0:
1075 basicHeaderObj = BasicHeader(LOCALTIME)
1081 basicHeaderObj = BasicHeader(LOCALTIME)
1076 systemHeaderObj = SystemHeader()
1082 systemHeaderObj = SystemHeader()
1077 radarControllerHeaderObj = RadarControllerHeader()
1083 radarControllerHeaderObj = RadarControllerHeader()
1078 processingHeaderObj = ProcessingHeader()
1084 processingHeaderObj = ProcessingHeader()
1079
1085
1080 if not( basicHeaderObj.read(fp) ):
1086 if not( basicHeaderObj.read(fp) ):
1081 fp.close()
1087 fp.close()
1082 return False
1088 return False
1083
1089
1084 if not( systemHeaderObj.read(fp) ):
1090 if not( systemHeaderObj.read(fp) ):
1085 fp.close()
1091 fp.close()
1086 return False
1092 return False
1087
1093
1088 if not( radarControllerHeaderObj.read(fp) ):
1094 if not( radarControllerHeaderObj.read(fp) ):
1089 fp.close()
1095 fp.close()
1090 return False
1096 return False
1091
1097
1092 if not( processingHeaderObj.read(fp) ):
1098 if not( processingHeaderObj.read(fp) ):
1093 fp.close()
1099 fp.close()
1094 return False
1100 return False
1095
1101
1096 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1102 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1097 else:
1103 else:
1098 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1104 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1099
1105
1100 fp.close()
1106 fp.close()
1101
1107
1102 fileSize = os.path.getsize(filename)
1108 fileSize = os.path.getsize(filename)
1103 currentSize = fileSize - currentPosition
1109 currentSize = fileSize - currentPosition
1104
1110
1105 if currentSize < neededSize:
1111 if currentSize < neededSize:
1106 if msgFlag and (msg != None):
1112 if msgFlag and (msg != None):
1107 print msg
1113 print msg
1108 return False
1114 return False
1109
1115
1110 return True
1116 return True
1111
1117
1112 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1118 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1113
1119
1114 path_empty = True
1120 path_empty = True
1115
1121
1116 dateList = []
1122 dateList = []
1117 pathList = []
1123 pathList = []
1118
1124
1119 multi_path = path.split(',')
1125 multi_path = path.split(',')
1120
1126
1121 if not walk:
1127 if not walk:
1122
1128
1123 for single_path in multi_path:
1129 for single_path in multi_path:
1124
1130
1125 if not os.path.isdir(single_path):
1131 if not os.path.isdir(single_path):
1126 continue
1132 continue
1127
1133
1128 fileList = glob.glob1(single_path, "*"+ext)
1134 fileList = glob.glob1(single_path, "*"+ext)
1129
1135
1130 if not fileList:
1136 if not fileList:
1131 continue
1137 continue
1132
1138
1133 path_empty = False
1139 path_empty = False
1134
1140
1135 fileList.sort()
1141 fileList.sort()
1136
1142
1137 for thisFile in fileList:
1143 for thisFile in fileList:
1138
1144
1139 if not os.path.isfile(os.path.join(single_path, thisFile)):
1145 if not os.path.isfile(os.path.join(single_path, thisFile)):
1140 continue
1146 continue
1141
1147
1142 if not isRadarFile(thisFile):
1148 if not isRadarFile(thisFile):
1143 continue
1149 continue
1144
1150
1145 if not isFileInDateRange(thisFile, startDate, endDate):
1151 if not isFileInDateRange(thisFile, startDate, endDate):
1146 continue
1152 continue
1147
1153
1148 thisDate = getDateFromRadarFile(thisFile)
1154 thisDate = getDateFromRadarFile(thisFile)
1149
1155
1150 if thisDate in dateList:
1156 if thisDate in dateList:
1151 continue
1157 continue
1152
1158
1153 dateList.append(thisDate)
1159 dateList.append(thisDate)
1154 pathList.append(single_path)
1160 pathList.append(single_path)
1155
1161
1156 else:
1162 else:
1157 for single_path in multi_path:
1163 for single_path in multi_path:
1158
1164
1159 if not os.path.isdir(single_path):
1165 if not os.path.isdir(single_path):
1160 continue
1166 continue
1161
1167
1162 dirList = []
1168 dirList = []
1163
1169
1164 for thisPath in os.listdir(single_path):
1170 for thisPath in os.listdir(single_path):
1165
1171
1166 if not os.path.isdir(os.path.join(single_path,thisPath)):
1172 if not os.path.isdir(os.path.join(single_path,thisPath)):
1167 continue
1173 continue
1168
1174
1169 if not isRadarFolder(thisPath):
1175 if not isRadarFolder(thisPath):
1170 continue
1176 continue
1171
1177
1172 if not isFolderInDateRange(thisPath, startDate, endDate):
1178 if not isFolderInDateRange(thisPath, startDate, endDate):
1173 continue
1179 continue
1174
1180
1175 dirList.append(thisPath)
1181 dirList.append(thisPath)
1176
1182
1177 if not dirList:
1183 if not dirList:
1178 continue
1184 continue
1179
1185
1180 dirList.sort()
1186 dirList.sort()
1181
1187
1182 for thisDir in dirList:
1188 for thisDir in dirList:
1183
1189
1184 datapath = os.path.join(single_path, thisDir, expLabel)
1190 datapath = os.path.join(single_path, thisDir, expLabel)
1185 fileList = glob.glob1(datapath, "*"+ext)
1191 fileList = glob.glob1(datapath, "*"+ext)
1186
1192
1187 if not fileList:
1193 if not fileList:
1188 continue
1194 continue
1189
1195
1190 path_empty = False
1196 path_empty = False
1191
1197
1192 thisDate = getDateFromRadarFolder(thisDir)
1198 thisDate = getDateFromRadarFolder(thisDir)
1193
1199
1194 pathList.append(datapath)
1200 pathList.append(datapath)
1195 dateList.append(thisDate)
1201 dateList.append(thisDate)
1196
1202
1197 dateList.sort()
1203 dateList.sort()
1198
1204
1199 if walk:
1205 if walk:
1200 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1206 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1201 else:
1207 else:
1202 pattern_path = multi_path[0]
1208 pattern_path = multi_path[0]
1203
1209
1204 if path_empty:
1210 if path_empty:
1205 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1211 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1206 else:
1212 else:
1207 if not dateList:
1213 if not dateList:
1208 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1214 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1209
1215
1210 if include_path:
1216 if include_path:
1211 return dateList, pathList
1217 return dateList, pathList
1212
1218
1213 return dateList
1219 return dateList
1214
1220
1215 def setup(self,
1221 def setup(self,
1216 path=None,
1222 path=None,
1217 startDate=None,
1223 startDate=None,
1218 endDate=None,
1224 endDate=None,
1219 startTime=datetime.time(0,0,0),
1225 startTime=datetime.time(0,0,0),
1220 endTime=datetime.time(23,59,59),
1226 endTime=datetime.time(23,59,59),
1221 set=None,
1227 set=None,
1222 expLabel = "",
1228 expLabel = "",
1223 ext = None,
1229 ext = None,
1224 online = False,
1230 online = False,
1225 delay = 60,
1231 delay = 60,
1226 walk = True,
1232 walk = True,
1227 getblock = False,
1233 getblock = False,
1228 nTxs = 1,
1234 nTxs = 1,
1229 realtime=False):
1235 realtime=False):
1230
1236
1231 if path == None:
1237 if path == None:
1232 raise ValueError, "[Reading] The path is not valid"
1238 raise ValueError, "[Reading] The path is not valid"
1233
1239
1234 if ext == None:
1240 if ext == None:
1235 ext = self.ext
1241 ext = self.ext
1236
1242
1237 if online:
1243 if online:
1238 print "[Reading] Searching files in online mode..."
1244 print "[Reading] Searching files in online mode..."
1239
1245
1240 for nTries in range( self.nTries ):
1246 for nTries in range( self.nTries ):
1241 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1247 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1242
1248
1243 if fullpath:
1249 if fullpath:
1244 break
1250 break
1245
1251
1246 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1252 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1247 sleep( self.delay )
1253 sleep( self.delay )
1248
1254
1249 if not(fullpath):
1255 if not(fullpath):
1250 print "[Reading] There 'isn't any valid file in %s" % path
1256 print "[Reading] There 'isn't any valid file in %s" % path
1251 return
1257 return
1252
1258
1253 self.year = year
1259 self.year = year
1254 self.doy = doy
1260 self.doy = doy
1255 self.set = set - 1
1261 self.set = set - 1
1256 self.path = path
1262 self.path = path
1257 self.foldercounter = foldercounter
1263 self.foldercounter = foldercounter
1258 last_set = None
1264 last_set = None
1259
1265
1260 else:
1266 else:
1261 print "[Reading] Searching files in offline mode ..."
1267 print "[Reading] Searching files in offline mode ..."
1262 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1268 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1263 startTime=startTime, endTime=endTime,
1269 startTime=startTime, endTime=endTime,
1264 set=set, expLabel=expLabel, ext=ext,
1270 set=set, expLabel=expLabel, ext=ext,
1265 walk=walk)
1271 walk=walk)
1266
1272
1267 if not(pathList):
1273 if not(pathList):
1268 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1274 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1269 # datetime.datetime.combine(startDate,startTime).ctime(),
1275 # datetime.datetime.combine(startDate,startTime).ctime(),
1270 # datetime.datetime.combine(endDate,endTime).ctime())
1276 # datetime.datetime.combine(endDate,endTime).ctime())
1271
1277
1272 # sys.exit(-1)
1278 # sys.exit(-1)
1273
1279
1274 self.fileIndex = -1
1280 self.fileIndex = -1
1275 self.pathList = []
1281 self.pathList = []
1276 self.filenameList = []
1282 self.filenameList = []
1277 return
1283 return
1278
1284
1279 self.fileIndex = -1
1285 self.fileIndex = -1
1280 self.pathList = pathList
1286 self.pathList = pathList
1281 self.filenameList = filenameList
1287 self.filenameList = filenameList
1282 file_name = os.path.basename(filenameList[-1])
1288 file_name = os.path.basename(filenameList[-1])
1283 basename, ext = os.path.splitext(file_name)
1289 basename, ext = os.path.splitext(file_name)
1284 last_set = int(basename[-3:])
1290 last_set = int(basename[-3:])
1285
1291
1286 self.online = online
1292 self.online = online
1287 self.realtime = realtime
1293 self.realtime = realtime
1288 self.delay = delay
1294 self.delay = delay
1289 ext = ext.lower()
1295 ext = ext.lower()
1290 self.ext = ext
1296 self.ext = ext
1291 self.getByBlock = getblock
1297 self.getByBlock = getblock
1292 self.nTxs = nTxs
1298 self.nTxs = nTxs
1293 self.startTime = startTime
1299 self.startTime = startTime
1294 self.endTime = endTime
1300 self.endTime = endTime
1295
1301
1296 if not(self.setNextFile()):
1302 if not(self.setNextFile()):
1297 if (startDate!=None) and (endDate!=None):
1303 if (startDate!=None) and (endDate!=None):
1298 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1304 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1299 elif startDate != None:
1305 elif startDate != None:
1300 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1306 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1301 else:
1307 else:
1302 print "[Reading] No files"
1308 print "[Reading] No files"
1303
1309
1304 self.fileIndex = -1
1310 self.fileIndex = -1
1305 self.pathList = []
1311 self.pathList = []
1306 self.filenameList = []
1312 self.filenameList = []
1307 return
1313 return
1308
1314
1309 # self.getBasicHeader()
1315 # self.getBasicHeader()
1310
1316
1311 if last_set != None:
1317 if last_set != None:
1312 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1318 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1313 return
1319 return
1314
1320
1315 def getBasicHeader(self):
1321 def getBasicHeader(self):
1316
1322
1317 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1323 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1318
1324
1319 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1325 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1320
1326
1321 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1327 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1322
1328
1323 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1329 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1324
1330
1325 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1331 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1326
1332
1327 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1333 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1328
1334
1329 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1335 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1330
1336
1331 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1337 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1332
1338
1333
1339
1334 def getFirstHeader(self):
1340 def getFirstHeader(self):
1335
1341
1336 raise NotImplementedError
1342 raise NotImplementedError
1337
1343
1338 def getData(self):
1344 def getData(self):
1339
1345
1340 raise NotImplementedError
1346 raise NotImplementedError
1341
1347
1342 def hasNotDataInBuffer(self):
1348 def hasNotDataInBuffer(self):
1343
1349
1344 raise NotImplementedError
1350 raise NotImplementedError
1345
1351
1346 def readBlock(self):
1352 def readBlock(self):
1347
1353
1348 raise NotImplementedError
1354 raise NotImplementedError
1349
1355
1350 def isEndProcess(self):
1356 def isEndProcess(self):
1351
1357
1352 return self.flagNoMoreFiles
1358 return self.flagNoMoreFiles
1353
1359
1354 def printReadBlocks(self):
1360 def printReadBlocks(self):
1355
1361
1356 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1362 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1357
1363
1358 def printTotalBlocks(self):
1364 def printTotalBlocks(self):
1359
1365
1360 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1366 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1361
1367
1362 def printNumberOfBlock(self):
1368 def printNumberOfBlock(self):
1363
1369
1364 if self.flagIsNewBlock:
1370 if self.flagIsNewBlock:
1365 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1371 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1366 self.processingHeaderObj.dataBlocksPerFile,
1372 self.processingHeaderObj.dataBlocksPerFile,
1367 self.dataOut.datatime.ctime())
1373 self.dataOut.datatime.ctime())
1368
1374
1369 def printInfo(self):
1375 def printInfo(self):
1370
1376
1371 if self.__printInfo == False:
1377 if self.__printInfo == False:
1372 return
1378 return
1373
1379
1374 self.basicHeaderObj.printInfo()
1380 self.basicHeaderObj.printInfo()
1375 self.systemHeaderObj.printInfo()
1381 self.systemHeaderObj.printInfo()
1376 self.radarControllerHeaderObj.printInfo()
1382 self.radarControllerHeaderObj.printInfo()
1377 self.processingHeaderObj.printInfo()
1383 self.processingHeaderObj.printInfo()
1378
1384
1379 self.__printInfo = False
1385 self.__printInfo = False
1380
1386
1381
1387
1382 def run(self, **kwargs):
1388 def run(self, **kwargs):
1383
1389
1384 if not(self.isConfig):
1390 if not(self.isConfig):
1385
1391
1386 # self.dataOut = dataOut
1392 # self.dataOut = dataOut
1387 self.setup(**kwargs)
1393 self.setup(**kwargs)
1388 self.isConfig = True
1394 self.isConfig = True
1389
1395
1390 self.getData()
1396 self.getData()
1391
1397
1392 class JRODataWriter(JRODataIO):
1398 class JRODataWriter(JRODataIO):
1393
1399
1394 """
1400 """
1395 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1401 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1396 de los datos siempre se realiza por bloques.
1402 de los datos siempre se realiza por bloques.
1397 """
1403 """
1398
1404
1399 blockIndex = 0
1405 blockIndex = 0
1400
1406
1401 path = None
1407 path = None
1402
1408
1403 setFile = None
1409 setFile = None
1404
1410
1405 profilesPerBlock = None
1411 profilesPerBlock = None
1406
1412
1407 blocksPerFile = None
1413 blocksPerFile = None
1408
1414
1409 nWriteBlocks = 0
1415 nWriteBlocks = 0
1410
1416
1411 fileDate = None
1417 fileDate = None
1412
1418
1413 def __init__(self, dataOut=None):
1419 def __init__(self, dataOut=None):
1414 raise NotImplementedError
1420 raise NotImplementedError
1415
1421
1416
1422
1417 def hasAllDataInBuffer(self):
1423 def hasAllDataInBuffer(self):
1418 raise NotImplementedError
1424 raise NotImplementedError
1419
1425
1420
1426
1421 def setBlockDimension(self):
1427 def setBlockDimension(self):
1422 raise NotImplementedError
1428 raise NotImplementedError
1423
1429
1424
1430
1425 def writeBlock(self):
1431 def writeBlock(self):
1426 raise NotImplementedError
1432 raise NotImplementedError
1427
1433
1428
1434
1429 def putData(self):
1435 def putData(self):
1430 raise NotImplementedError
1436 raise NotImplementedError
1431
1437
1432
1438
1433 def getProcessFlags(self):
1439 def getProcessFlags(self):
1434
1440
1435 processFlags = 0
1441 processFlags = 0
1436
1442
1437 dtype_index = get_dtype_index(self.dtype)
1443 dtype_index = get_dtype_index(self.dtype)
1438 procflag_dtype = get_procflag_dtype(dtype_index)
1444 procflag_dtype = get_procflag_dtype(dtype_index)
1439
1445
1440 processFlags += procflag_dtype
1446 processFlags += procflag_dtype
1441
1447
1442 if self.dataOut.flagDecodeData:
1448 if self.dataOut.flagDecodeData:
1443 processFlags += PROCFLAG.DECODE_DATA
1449 processFlags += PROCFLAG.DECODE_DATA
1444
1450
1445 if self.dataOut.flagDeflipData:
1451 if self.dataOut.flagDeflipData:
1446 processFlags += PROCFLAG.DEFLIP_DATA
1452 processFlags += PROCFLAG.DEFLIP_DATA
1447
1453
1448 if self.dataOut.code is not None:
1454 if self.dataOut.code is not None:
1449 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1455 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1450
1456
1451 if self.dataOut.nCohInt > 1:
1457 if self.dataOut.nCohInt > 1:
1452 processFlags += PROCFLAG.COHERENT_INTEGRATION
1458 processFlags += PROCFLAG.COHERENT_INTEGRATION
1453
1459
1454 if self.dataOut.type == "Spectra":
1460 if self.dataOut.type == "Spectra":
1455 if self.dataOut.nIncohInt > 1:
1461 if self.dataOut.nIncohInt > 1:
1456 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1462 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1457
1463
1458 if self.dataOut.data_dc is not None:
1464 if self.dataOut.data_dc is not None:
1459 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1465 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1460
1466
1461 if self.dataOut.flagShiftFFT:
1467 if self.dataOut.flagShiftFFT:
1462 processFlags += PROCFLAG.SHIFT_FFT_DATA
1468 processFlags += PROCFLAG.SHIFT_FFT_DATA
1463
1469
1464 return processFlags
1470 return processFlags
1465
1471
1466 def setBasicHeader(self):
1472 def setBasicHeader(self):
1467
1473
1468 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1474 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1469 self.basicHeaderObj.version = self.versionFile
1475 self.basicHeaderObj.version = self.versionFile
1470 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1476 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1471
1477
1472 utc = numpy.floor(self.dataOut.utctime)
1478 utc = numpy.floor(self.dataOut.utctime)
1473 milisecond = (self.dataOut.utctime - utc)* 1000.0
1479 milisecond = (self.dataOut.utctime - utc)* 1000.0
1474
1480
1475 self.basicHeaderObj.utc = utc
1481 self.basicHeaderObj.utc = utc
1476 self.basicHeaderObj.miliSecond = milisecond
1482 self.basicHeaderObj.miliSecond = milisecond
1477 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1483 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1478 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1484 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1479 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1485 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1480
1486
1481 def setFirstHeader(self):
1487 def setFirstHeader(self):
1482 """
1488 """
1483 Obtiene una copia del First Header
1489 Obtiene una copia del First Header
1484
1490
1485 Affected:
1491 Affected:
1486
1492
1487 self.basicHeaderObj
1493 self.basicHeaderObj
1488 self.systemHeaderObj
1494 self.systemHeaderObj
1489 self.radarControllerHeaderObj
1495 self.radarControllerHeaderObj
1490 self.processingHeaderObj self.
1496 self.processingHeaderObj self.
1491
1497
1492 Return:
1498 Return:
1493 None
1499 None
1494 """
1500 """
1495
1501
1496 raise NotImplementedError
1502 raise NotImplementedError
1497
1503
1498 def __writeFirstHeader(self):
1504 def __writeFirstHeader(self):
1499 """
1505 """
1500 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1506 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1501
1507
1502 Affected:
1508 Affected:
1503 __dataType
1509 __dataType
1504
1510
1505 Return:
1511 Return:
1506 None
1512 None
1507 """
1513 """
1508
1514
1509 # CALCULAR PARAMETROS
1515 # CALCULAR PARAMETROS
1510
1516
1511 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1517 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1512 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1518 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1513
1519
1514 self.basicHeaderObj.write(self.fp)
1520 self.basicHeaderObj.write(self.fp)
1515 self.systemHeaderObj.write(self.fp)
1521 self.systemHeaderObj.write(self.fp)
1516 self.radarControllerHeaderObj.write(self.fp)
1522 self.radarControllerHeaderObj.write(self.fp)
1517 self.processingHeaderObj.write(self.fp)
1523 self.processingHeaderObj.write(self.fp)
1518
1524
1519 def __setNewBlock(self):
1525 def __setNewBlock(self):
1520 """
1526 """
1521 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1527 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1522
1528
1523 Return:
1529 Return:
1524 0 : si no pudo escribir nada
1530 0 : si no pudo escribir nada
1525 1 : Si escribio el Basic el First Header
1531 1 : Si escribio el Basic el First Header
1526 """
1532 """
1527 if self.fp == None:
1533 if self.fp == None:
1528 self.setNextFile()
1534 self.setNextFile()
1529
1535
1530 if self.flagIsNewFile:
1536 if self.flagIsNewFile:
1531 return 1
1537 return 1
1532
1538
1533 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1539 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1534 self.basicHeaderObj.write(self.fp)
1540 self.basicHeaderObj.write(self.fp)
1535 return 1
1541 return 1
1536
1542
1537 if not( self.setNextFile() ):
1543 if not( self.setNextFile() ):
1538 return 0
1544 return 0
1539
1545
1540 return 1
1546 return 1
1541
1547
1542
1548
1543 def writeNextBlock(self):
1549 def writeNextBlock(self):
1544 """
1550 """
1545 Selecciona el bloque siguiente de datos y los escribe en un file
1551 Selecciona el bloque siguiente de datos y los escribe en un file
1546
1552
1547 Return:
1553 Return:
1548 0 : Si no hizo pudo escribir el bloque de datos
1554 0 : Si no hizo pudo escribir el bloque de datos
1549 1 : Si no pudo escribir el bloque de datos
1555 1 : Si no pudo escribir el bloque de datos
1550 """
1556 """
1551 if not( self.__setNewBlock() ):
1557 if not( self.__setNewBlock() ):
1552 return 0
1558 return 0
1553
1559
1554 self.writeBlock()
1560 self.writeBlock()
1555
1561
1556 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1562 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1557 self.processingHeaderObj.dataBlocksPerFile)
1563 self.processingHeaderObj.dataBlocksPerFile)
1558
1564
1559 return 1
1565 return 1
1560
1566
1561 def setNextFile(self):
1567 def setNextFile(self):
1562 """
1568 """
1563 Determina el siguiente file que sera escrito
1569 Determina el siguiente file que sera escrito
1564
1570
1565 Affected:
1571 Affected:
1566 self.filename
1572 self.filename
1567 self.subfolder
1573 self.subfolder
1568 self.fp
1574 self.fp
1569 self.setFile
1575 self.setFile
1570 self.flagIsNewFile
1576 self.flagIsNewFile
1571
1577
1572 Return:
1578 Return:
1573 0 : Si el archivo no puede ser escrito
1579 0 : Si el archivo no puede ser escrito
1574 1 : Si el archivo esta listo para ser escrito
1580 1 : Si el archivo esta listo para ser escrito
1575 """
1581 """
1576 ext = self.ext
1582 ext = self.ext
1577 path = self.path
1583 path = self.path
1578
1584
1579 if self.fp != None:
1585 if self.fp != None:
1580 self.fp.close()
1586 self.fp.close()
1581
1587
1582 timeTuple = time.localtime( self.dataOut.utctime)
1588 timeTuple = time.localtime( self.dataOut.utctime)
1583 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1589 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1584
1590
1585 fullpath = os.path.join( path, subfolder )
1591 fullpath = os.path.join( path, subfolder )
1586 setFile = self.setFile
1592 setFile = self.setFile
1587
1593
1588 if not( os.path.exists(fullpath) ):
1594 if not( os.path.exists(fullpath) ):
1589 os.mkdir(fullpath)
1595 os.mkdir(fullpath)
1590 setFile = -1 #inicializo mi contador de seteo
1596 setFile = -1 #inicializo mi contador de seteo
1591 else:
1597 else:
1592 filesList = os.listdir( fullpath )
1598 filesList = os.listdir( fullpath )
1593 if len( filesList ) > 0:
1599 if len( filesList ) > 0:
1594 filesList = sorted( filesList, key=str.lower )
1600 filesList = sorted( filesList, key=str.lower )
1595 filen = filesList[-1]
1601 filen = filesList[-1]
1596 # el filename debera tener el siguiente formato
1602 # el filename debera tener el siguiente formato
1597 # 0 1234 567 89A BCDE (hex)
1603 # 0 1234 567 89A BCDE (hex)
1598 # x YYYY DDD SSS .ext
1604 # x YYYY DDD SSS .ext
1599 if isNumber( filen[8:11] ):
1605 if isNumber( filen[8:11] ):
1600 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1606 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1601 else:
1607 else:
1602 setFile = -1
1608 setFile = -1
1603 else:
1609 else:
1604 setFile = -1 #inicializo mi contador de seteo
1610 setFile = -1 #inicializo mi contador de seteo
1605
1611
1606 setFile += 1
1612 setFile += 1
1607
1613
1608 #If this is a new day it resets some values
1614 #If this is a new day it resets some values
1609 if self.dataOut.datatime.date() > self.fileDate:
1615 if self.dataOut.datatime.date() > self.fileDate:
1610 setFile = 0
1616 setFile = 0
1611 self.nTotalBlocks = 0
1617 self.nTotalBlocks = 0
1612
1618
1613 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1619 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1614
1620
1615 filename = os.path.join( path, subfolder, filen )
1621 filename = os.path.join( path, subfolder, filen )
1616
1622
1617 fp = open( filename,'wb' )
1623 fp = open( filename,'wb' )
1618
1624
1619 self.blockIndex = 0
1625 self.blockIndex = 0
1620
1626
1621 #guardando atributos
1627 #guardando atributos
1622 self.filename = filename
1628 self.filename = filename
1623 self.subfolder = subfolder
1629 self.subfolder = subfolder
1624 self.fp = fp
1630 self.fp = fp
1625 self.setFile = setFile
1631 self.setFile = setFile
1626 self.flagIsNewFile = 1
1632 self.flagIsNewFile = 1
1627 self.fileDate = self.dataOut.datatime.date()
1633 self.fileDate = self.dataOut.datatime.date()
1628
1634
1629 self.setFirstHeader()
1635 self.setFirstHeader()
1630
1636
1631 print '[Writing] Opening file: %s'%self.filename
1637 print '[Writing] Opening file: %s'%self.filename
1632
1638
1633 self.__writeFirstHeader()
1639 self.__writeFirstHeader()
1634
1640
1635 return 1
1641 return 1
1636
1642
1637 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1643 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1638 """
1644 """
1639 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1645 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1640
1646
1641 Inputs:
1647 Inputs:
1642 path : directory where data will be saved
1648 path : directory where data will be saved
1643 profilesPerBlock : number of profiles per block
1649 profilesPerBlock : number of profiles per block
1644 set : initial file set
1650 set : initial file set
1645 datatype : An integer number that defines data type:
1651 datatype : An integer number that defines data type:
1646 0 : int8 (1 byte)
1652 0 : int8 (1 byte)
1647 1 : int16 (2 bytes)
1653 1 : int16 (2 bytes)
1648 2 : int32 (4 bytes)
1654 2 : int32 (4 bytes)
1649 3 : int64 (8 bytes)
1655 3 : int64 (8 bytes)
1650 4 : float32 (4 bytes)
1656 4 : float32 (4 bytes)
1651 5 : double64 (8 bytes)
1657 5 : double64 (8 bytes)
1652
1658
1653 Return:
1659 Return:
1654 0 : Si no realizo un buen seteo
1660 0 : Si no realizo un buen seteo
1655 1 : Si realizo un buen seteo
1661 1 : Si realizo un buen seteo
1656 """
1662 """
1657
1663
1658 if ext == None:
1664 if ext == None:
1659 ext = self.ext
1665 ext = self.ext
1660
1666
1661 self.ext = ext.lower()
1667 self.ext = ext.lower()
1662
1668
1663 self.path = path
1669 self.path = path
1664
1670
1665 if set is None:
1671 if set is None:
1666 self.setFile = -1
1672 self.setFile = -1
1667 else:
1673 else:
1668 self.setFile = set - 1
1674 self.setFile = set - 1
1669
1675
1670 self.blocksPerFile = blocksPerFile
1676 self.blocksPerFile = blocksPerFile
1671
1677
1672 self.profilesPerBlock = profilesPerBlock
1678 self.profilesPerBlock = profilesPerBlock
1673
1679
1674 self.dataOut = dataOut
1680 self.dataOut = dataOut
1675 self.fileDate = self.dataOut.datatime.date()
1681 self.fileDate = self.dataOut.datatime.date()
1676 #By default
1682 #By default
1677 self.dtype = self.dataOut.dtype
1683 self.dtype = self.dataOut.dtype
1678
1684
1679 if datatype is not None:
1685 if datatype is not None:
1680 self.dtype = get_numpy_dtype(datatype)
1686 self.dtype = get_numpy_dtype(datatype)
1681
1687
1682 if not(self.setNextFile()):
1688 if not(self.setNextFile()):
1683 print "[Writing] There isn't a next file"
1689 print "[Writing] There isn't a next file"
1684 return 0
1690 return 0
1685
1691
1686 self.setBlockDimension()
1692 self.setBlockDimension()
1687
1693
1688 return 1
1694 return 1
1689
1695
1690 def run(self, dataOut, **kwargs):
1696 def run(self, dataOut, **kwargs):
1691
1697
1692 if not(self.isConfig):
1698 if not(self.isConfig):
1693
1699
1694 self.setup(dataOut, **kwargs)
1700 self.setup(dataOut, **kwargs)
1695 self.isConfig = True
1701 self.isConfig = True
1696
1702
1697 self.putData()
1703 self.putData()
1698
1704
General Comments 0
You need to be logged in to leave comments. Login now