##// END OF EJS Templates
schain
RSS
Bug fixed in jroIO_base: date and time filters were not working when walk=0 and there were many days inside one directory.
Miguel Valdez -
r640:4d492e909122
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1550 +1,1629
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import time, datetime
12 import time, datetime
13 #import h5py
13 #import h5py
14 import traceback
14 import traceback
15
15
16 try:
16 try:
17 from gevent import sleep
17 from gevent import sleep
18 except:
18 except:
19 from time import sleep
19 from time import sleep
20
20
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23
23
24 LOCALTIME = True
24 LOCALTIME = True
25
25
26 def isNumber(cad):
26 def isNumber(cad):
27 """
27 """
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29
29
30 Excepciones:
30 Excepciones:
31 Si un determinado string no puede ser convertido a numero
31 Si un determinado string no puede ser convertido a numero
32 Input:
32 Input:
33 str, string al cual se le analiza para determinar si convertible a un numero o no
33 str, string al cual se le analiza para determinar si convertible a un numero o no
34
34
35 Return:
35 Return:
36 True : si el string es uno numerico
36 True : si el string es uno numerico
37 False : no es un string numerico
37 False : no es un string numerico
38 """
38 """
39 try:
39 try:
40 float( cad )
40 float( cad )
41 return True
41 return True
42 except:
42 except:
43 return False
43 return False
44
44
45 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 """
46 """
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48
48
49 Inputs:
49 Inputs:
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51
51
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 segundos contados desde 01/01/1970.
53 segundos contados desde 01/01/1970.
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 segundos contados desde 01/01/1970.
55 segundos contados desde 01/01/1970.
56
56
57 Return:
57 Return:
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 fecha especificado, de lo contrario retorna False.
59 fecha especificado, de lo contrario retorna False.
60
60
61 Excepciones:
61 Excepciones:
62 Si el archivo no existe o no puede ser abierto
62 Si el archivo no existe o no puede ser abierto
63 Si la cabecera no puede ser leida.
63 Si la cabecera no puede ser leida.
64
64
65 """
65 """
66 basicHeaderObj = BasicHeader(LOCALTIME)
66 basicHeaderObj = BasicHeader(LOCALTIME)
67
67
68 try:
68 try:
69 fp = open(filename,'rb')
69 fp = open(filename,'rb')
70 except IOError:
70 except IOError:
71 traceback.print_exc()
71 traceback.print_exc()
72 raise IOError, "The file %s can't be opened" %(filename)
72 raise IOError, "The file %s can't be opened" %(filename)
73
73
74 sts = basicHeaderObj.read(fp)
74 sts = basicHeaderObj.read(fp)
75 fp.close()
75 fp.close()
76
76
77 if not(sts):
77 if not(sts):
78 print "Skipping the file %s because it has not a valid header" %(filename)
78 print "Skipping the file %s because it has not a valid header" %(filename)
79 return 0
79 return 0
80
80
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 return 0
82 return 0
83
83
84 return 1
84 return 1
85
85
86 def isFileinThisTime(filename, startTime, endTime):
86 def isFileInTimeRange(filename, startTime, endTime):
87 """
87 """
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89
89
90 Inputs:
90 Inputs:
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92
92
93 startTime : tiempo inicial del rango seleccionado en formato datetime.time
93 startTime : tiempo inicial del rango seleccionado en formato datetime.time
94
94
95 endTime : tiempo final del rango seleccionado en formato datetime.time
95 endTime : tiempo final del rango seleccionado en formato datetime.time
96
96
97 Return:
97 Return:
98 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
98 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
99 fecha especificado, de lo contrario retorna False.
99 fecha especificado, de lo contrario retorna False.
100
100
101 Excepciones:
101 Excepciones:
102 Si el archivo no existe o no puede ser abierto
102 Si el archivo no existe o no puede ser abierto
103 Si la cabecera no puede ser leida.
103 Si la cabecera no puede ser leida.
104
104
105 """
105 """
106
106
107
107
108 try:
108 try:
109 fp = open(filename,'rb')
109 fp = open(filename,'rb')
110 except IOError:
110 except IOError:
111 traceback.print_exc()
111 traceback.print_exc()
112 raise IOError, "The file %s can't be opened" %(filename)
112 raise IOError, "The file %s can't be opened" %(filename)
113
113
114 basicHeaderObj = BasicHeader(LOCALTIME)
114 basicHeaderObj = BasicHeader(LOCALTIME)
115 sts = basicHeaderObj.read(fp)
115 sts = basicHeaderObj.read(fp)
116 fp.close()
116 fp.close()
117
117
118 thisDatetime = basicHeaderObj.datatime
118 thisDatetime = basicHeaderObj.datatime
119 thisTime = thisDatetime.time()
119 thisTime = thisDatetime.time()
120
120
121 if not(sts):
121 if not(sts):
122 print "Skipping the file %s because it has not a valid header" %(filename)
122 print "Skipping the file %s because it has not a valid header" %(filename)
123 return None
123 return None
124
124
125 if not ((startTime <= thisTime) and (endTime > thisTime)):
125 if not ((startTime <= thisTime) and (endTime > thisTime)):
126 return None
126 return None
127
127
128 return thisDatetime
128 return thisDatetime
129
129
130 def isFolderInDateRange(folder, startDate=None, endDate=None):
131 """
132 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
133
134 Inputs:
135 folder : nombre completo del directorio.
136 Su formato deberia ser "/path_root/?YYYYDDD"
137
138 siendo:
139 YYYY : Anio (ejemplo 2015)
140 DDD : Dia del anio (ejemplo 305)
141
142 startDate : fecha inicial del rango seleccionado en formato datetime.date
143
144 endDate : fecha final del rango seleccionado en formato datetime.date
145
146 Return:
147 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
148 fecha especificado, de lo contrario retorna False.
149 Excepciones:
150 Si el directorio no tiene el formato adecuado
151 """
152
153 basename = os.path.basename(folder)
154
155 if not isRadarFolder(basename):
156 raise IOError, "The folder %s has not the rigth format" %folder
157
158 if startDate and endDate:
159 thisDate = getDateFromRadarFolder(basename)
160
161 if thisDate < startDate:
162 return 0
163
164 if thisDate > endDate:
165 return 0
166
167 return 1
168
169 def isFileInDateRange(filename, startDate=None, endDate=None):
170 """
171 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
172
173 Inputs:
174 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
175
176 Su formato deberia ser "?YYYYDDDsss"
177
178 siendo:
179 YYYY : Anio (ejemplo 2015)
180 DDD : Dia del anio (ejemplo 305)
181 sss : set
182
183 startDate : fecha inicial del rango seleccionado en formato datetime.date
184
185 endDate : fecha final del rango seleccionado en formato datetime.date
186
187 Return:
188 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
189 fecha especificado, de lo contrario retorna False.
190 Excepciones:
191 Si el archivo no tiene el formato adecuado
192 """
193
194 basename = os.path.basename(filename)
195
196 if not isRadarFile(basename):
197 raise IOError, "The filename %s has not the rigth format" %filename
198
199 if startDate and endDate:
200 thisDate = getDateFromRadarFile(basename)
201
202 if thisDate < startDate:
203 return 0
204
205 if thisDate > endDate:
206 return 0
207
208 return 1
209
130 def getFileFromSet(path, ext, set):
210 def getFileFromSet(path, ext, set):
131 validFilelist = []
211 validFilelist = []
132 fileList = os.listdir(path)
212 fileList = os.listdir(path)
133
213
134 # 0 1234 567 89A BCDE
214 # 0 1234 567 89A BCDE
135 # H YYYY DDD SSS .ext
215 # H YYYY DDD SSS .ext
136
216
137 for thisFile in fileList:
217 for thisFile in fileList:
138 try:
218 try:
139 year = int(thisFile[1:5])
219 year = int(thisFile[1:5])
140 doy = int(thisFile[5:8])
220 doy = int(thisFile[5:8])
141 except:
221 except:
142 continue
222 continue
143
223
144 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
224 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
145 continue
225 continue
146
226
147 validFilelist.append(thisFile)
227 validFilelist.append(thisFile)
148
228
149 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
229 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
150
230
151 if len(myfile)!= 0:
231 if len(myfile)!= 0:
152 return myfile[0]
232 return myfile[0]
153 else:
233 else:
154 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
234 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
155 print 'the filename %s does not exist'%filename
235 print 'the filename %s does not exist'%filename
156 print '...going to the last file: '
236 print '...going to the last file: '
157
237
158 if validFilelist:
238 if validFilelist:
159 validFilelist = sorted( validFilelist, key=str.lower )
239 validFilelist = sorted( validFilelist, key=str.lower )
160 return validFilelist[-1]
240 return validFilelist[-1]
161
241
162 return None
242 return None
163
243
164 def getlastFileFromPath(path, ext):
244 def getlastFileFromPath(path, ext):
165 """
245 """
166 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
246 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
167 al final de la depuracion devuelve el ultimo file de la lista que quedo.
247 al final de la depuracion devuelve el ultimo file de la lista que quedo.
168
248
169 Input:
249 Input:
170 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
250 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
171 ext : extension de los files contenidos en una carpeta
251 ext : extension de los files contenidos en una carpeta
172
252
173 Return:
253 Return:
174 El ultimo file de una determinada carpeta, no se considera el path.
254 El ultimo file de una determinada carpeta, no se considera el path.
175 """
255 """
176 validFilelist = []
256 validFilelist = []
177 fileList = os.listdir(path)
257 fileList = os.listdir(path)
178
258
179 # 0 1234 567 89A BCDE
259 # 0 1234 567 89A BCDE
180 # H YYYY DDD SSS .ext
260 # H YYYY DDD SSS .ext
181
261
182 for thisFile in fileList:
262 for thisFile in fileList:
183
263
184 year = thisFile[1:5]
264 year = thisFile[1:5]
185 if not isNumber(year):
265 if not isNumber(year):
186 continue
266 continue
187
267
188 doy = thisFile[5:8]
268 doy = thisFile[5:8]
189 if not isNumber(doy):
269 if not isNumber(doy):
190 continue
270 continue
191
271
192 year = int(year)
272 year = int(year)
193 doy = int(doy)
273 doy = int(doy)
194
274
195 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
275 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
196 continue
276 continue
197
277
198 validFilelist.append(thisFile)
278 validFilelist.append(thisFile)
199
279
200 if validFilelist:
280 if validFilelist:
201 validFilelist = sorted( validFilelist, key=str.lower )
281 validFilelist = sorted( validFilelist, key=str.lower )
202 return validFilelist[-1]
282 return validFilelist[-1]
203
283
204 return None
284 return None
205
285
206 def checkForRealPath(path, foldercounter, year, doy, set, ext):
286 def checkForRealPath(path, foldercounter, year, doy, set, ext):
207 """
287 """
208 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
288 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
209 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
289 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
210 el path exacto de un determinado file.
290 el path exacto de un determinado file.
211
291
212 Example :
292 Example :
213 nombre correcto del file es .../.../D2009307/P2009307367.ext
293 nombre correcto del file es .../.../D2009307/P2009307367.ext
214
294
215 Entonces la funcion prueba con las siguientes combinaciones
295 Entonces la funcion prueba con las siguientes combinaciones
216 .../.../y2009307367.ext
296 .../.../y2009307367.ext
217 .../.../Y2009307367.ext
297 .../.../Y2009307367.ext
218 .../.../x2009307/y2009307367.ext
298 .../.../x2009307/y2009307367.ext
219 .../.../x2009307/Y2009307367.ext
299 .../.../x2009307/Y2009307367.ext
220 .../.../X2009307/y2009307367.ext
300 .../.../X2009307/y2009307367.ext
221 .../.../X2009307/Y2009307367.ext
301 .../.../X2009307/Y2009307367.ext
222 siendo para este caso, la ultima combinacion de letras, identica al file buscado
302 siendo para este caso, la ultima combinacion de letras, identica al file buscado
223
303
224 Return:
304 Return:
225 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
305 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
226 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
306 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
227 para el filename
307 para el filename
228 """
308 """
229 fullfilename = None
309 fullfilename = None
230 find_flag = False
310 find_flag = False
231 filename = None
311 filename = None
232
312
233 prefixDirList = [None,'d','D']
313 prefixDirList = [None,'d','D']
234 if ext.lower() == ".r": #voltage
314 if ext.lower() == ".r": #voltage
235 prefixFileList = ['d','D']
315 prefixFileList = ['d','D']
236 elif ext.lower() == ".pdata": #spectra
316 elif ext.lower() == ".pdata": #spectra
237 prefixFileList = ['p','P']
317 prefixFileList = ['p','P']
238 else:
318 else:
239 return None, filename
319 return None, filename
240
320
241 #barrido por las combinaciones posibles
321 #barrido por las combinaciones posibles
242 for prefixDir in prefixDirList:
322 for prefixDir in prefixDirList:
243 thispath = path
323 thispath = path
244 if prefixDir != None:
324 if prefixDir != None:
245 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
325 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
246 if foldercounter == 0:
326 if foldercounter == 0:
247 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
327 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
248 else:
328 else:
249 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
329 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
250 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
330 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
251 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
331 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
252 fullfilename = os.path.join( thispath, filename ) #formo el path completo
332 fullfilename = os.path.join( thispath, filename ) #formo el path completo
253
333
254 if os.path.exists( fullfilename ): #verifico que exista
334 if os.path.exists( fullfilename ): #verifico que exista
255 find_flag = True
335 find_flag = True
256 break
336 break
257 if find_flag:
337 if find_flag:
258 break
338 break
259
339
260 if not(find_flag):
340 if not(find_flag):
261 return None, filename
341 return None, filename
262
342
263 return fullfilename, filename
343 return fullfilename, filename
264
344
265 def isRadarFolder(folder):
345 def isRadarFolder(folder):
266 try:
346 try:
267 year = int(folder[1:5])
347 year = int(folder[1:5])
268 doy = int(folder[5:8])
348 doy = int(folder[5:8])
269 except:
349 except:
270 return 0
350 return 0
271
351
272 return 1
352 return 1
273
353
274 def isRadarFile(file):
354 def isRadarFile(file):
275 try:
355 try:
276 year = int(file[1:5])
356 year = int(file[1:5])
277 doy = int(file[5:8])
357 doy = int(file[5:8])
278 set = int(file[8:11])
358 set = int(file[8:11])
279 except:
359 except:
280 return 0
360 return 0
281
361
282 return 1
362 return 1
283
363
284 def getDateFromRadarFile(file):
364 def getDateFromRadarFile(file):
285 try:
365 try:
286 year = int(file[1:5])
366 year = int(file[1:5])
287 doy = int(file[5:8])
367 doy = int(file[5:8])
288 set = int(file[8:11])
368 set = int(file[8:11])
289 except:
369 except:
290 return None
370 return None
291
371
292 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
372 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
293 return thisDate
373 return thisDate
294
374
375 def getDateFromRadarFolder(folder):
376 try:
377 year = int(folder[1:5])
378 doy = int(folder[5:8])
379 except:
380 return None
381
382 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
383 return thisDate
384
295 class JRODataIO:
385 class JRODataIO:
296
386
297 c = 3E8
387 c = 3E8
298
388
299 isConfig = False
389 isConfig = False
300
390
301 basicHeaderObj = None
391 basicHeaderObj = None
302
392
303 systemHeaderObj = None
393 systemHeaderObj = None
304
394
305 radarControllerHeaderObj = None
395 radarControllerHeaderObj = None
306
396
307 processingHeaderObj = None
397 processingHeaderObj = None
308
398
309 online = 0
399 online = 0
310
400
311 dtype = None
401 dtype = None
312
402
313 pathList = []
403 pathList = []
314
404
315 filenameList = []
405 filenameList = []
316
406
317 filename = None
407 filename = None
318
408
319 ext = None
409 ext = None
320
410
321 flagIsNewFile = 1
411 flagIsNewFile = 1
322
412
323 flagDiscontinuousBlock = 0
413 flagDiscontinuousBlock = 0
324
414
325 flagIsNewBlock = 0
415 flagIsNewBlock = 0
326
416
327 fp = None
417 fp = None
328
418
329 firstHeaderSize = 0
419 firstHeaderSize = 0
330
420
331 basicHeaderSize = 24
421 basicHeaderSize = 24
332
422
333 versionFile = 1103
423 versionFile = 1103
334
424
335 fileSize = None
425 fileSize = None
336
426
337 # ippSeconds = None
427 # ippSeconds = None
338
428
339 fileSizeByHeader = None
429 fileSizeByHeader = None
340
430
341 fileIndex = None
431 fileIndex = None
342
432
343 profileIndex = None
433 profileIndex = None
344
434
345 blockIndex = None
435 blockIndex = None
346
436
347 nTotalBlocks = None
437 nTotalBlocks = None
348
438
349 maxTimeStep = 30
439 maxTimeStep = 30
350
440
351 lastUTTime = None
441 lastUTTime = None
352
442
353 datablock = None
443 datablock = None
354
444
355 dataOut = None
445 dataOut = None
356
446
357 blocksize = None
447 blocksize = None
358
448
359 getByBlock = False
449 getByBlock = False
360
450
361 def __init__(self):
451 def __init__(self):
362
452
363 raise ValueError, "Not implemented"
453 raise ValueError, "Not implemented"
364
454
365 def run(self):
455 def run(self):
366
456
367 raise ValueError, "Not implemented"
457 raise ValueError, "Not implemented"
368
458
369 def getDtypeWidth(self):
459 def getDtypeWidth(self):
370
460
371 dtype_index = get_dtype_index(self.dtype)
461 dtype_index = get_dtype_index(self.dtype)
372 dtype_width = get_dtype_width(dtype_index)
462 dtype_width = get_dtype_width(dtype_index)
373
463
374 return dtype_width
464 return dtype_width
375
465
376 class JRODataReader(JRODataIO):
466 class JRODataReader(JRODataIO):
377
467
378 nReadBlocks = 0
468 nReadBlocks = 0
379
469
380 delay = 10 #number of seconds waiting a new file
470 delay = 10 #number of seconds waiting a new file
381
471
382 nTries = 3 #quantity tries
472 nTries = 3 #quantity tries
383
473
384 nFiles = 3 #number of files for searching
474 nFiles = 3 #number of files for searching
385
475
386 path = None
476 path = None
387
477
388 foldercounter = 0
478 foldercounter = 0
389
479
390 flagNoMoreFiles = 0
480 flagNoMoreFiles = 0
391
481
392 datetimeList = []
482 datetimeList = []
393
483
394 __isFirstTimeOnline = 1
484 __isFirstTimeOnline = 1
395
485
396 __printInfo = True
486 __printInfo = True
397
487
398 profileIndex = None
488 profileIndex = None
399
489
400 nTxs = 1
490 nTxs = 1
401
491
402 txIndex = None
492 txIndex = None
403
493
404 def __init__(self):
494 def __init__(self):
405
495
406 """
496 """
407
497
408 """
498 """
409
499
410 # raise NotImplementedError, "This method has not been implemented"
500 # raise NotImplementedError, "This method has not been implemented"
411
501
412
502
413 def createObjByDefault(self):
503 def createObjByDefault(self):
414 """
504 """
415
505
416 """
506 """
417 raise NotImplementedError, "This method has not been implemented"
507 raise NotImplementedError, "This method has not been implemented"
418
508
419 def getBlockDimension(self):
509 def getBlockDimension(self):
420
510
421 raise NotImplementedError, "No implemented"
511 raise NotImplementedError, "No implemented"
422
512
423 def __searchFilesOffLine(self,
513 def __searchFilesOffLine(self,
424 path,
514 path,
425 startDate=None,
515 startDate=None,
426 endDate=None,
516 endDate=None,
427 startTime=datetime.time(0,0,0),
517 startTime=datetime.time(0,0,0),
428 endTime=datetime.time(23,59,59),
518 endTime=datetime.time(23,59,59),
429 set=None,
519 set=None,
430 expLabel='',
520 expLabel='',
431 ext='.r',
521 ext='.r',
432 walk=True):
522 walk=True):
433
523
434 self.filenameList = []
524 self.filenameList = []
435 self.datetimeList = []
525 self.datetimeList = []
436
526
437 pathList = []
527 pathList = []
438
528
439 if not walk:
529 # if not walk:
440 #pathList.append(path)
530 # #pathList.append(path)
441 multi_path = path.split(',')
531 # multi_path = path.split(',')
442 for single_path in multi_path:
532 # for single_path in multi_path:
443
533 #
444 if not os.path.isdir(single_path):
534 # if not os.path.isdir(single_path):
445 continue
535 # continue
446
536 #
447 pathList.append(single_path)
537 # pathList.append(single_path)
448
538 #
449 else:
539 # else:
450 #dirList = []
540 # #dirList = []
451 multi_path = path.split(',')
541 # multi_path = path.split(',')
452 for single_path in multi_path:
542 # for single_path in multi_path:
453
543 #
454 if not os.path.isdir(single_path):
544 # if not os.path.isdir(single_path):
455 continue
545 # continue
456
546 #
457 dirList = []
547 # dirList = []
458 for thisPath in os.listdir(single_path):
548 # for thisPath in os.listdir(single_path):
459 if not os.path.isdir(os.path.join(single_path,thisPath)):
549 # if not os.path.isdir(os.path.join(single_path,thisPath)):
460 continue
550 # continue
461 if not isRadarFolder(thisPath):
551 # if not isRadarFolder(thisPath):
462 continue
552 # continue
463
553 #
464 dirList.append(thisPath)
554 # dirList.append(thisPath)
465
555 #
466 if not(dirList):
556 # if not(dirList):
467 return None, None
557 # return None, None
468
558 #
469 if startDate and endDate:
559 # if startDate and endDate:
470 thisDate = startDate
560 # thisDate = startDate
471
561 #
472 while(thisDate <= endDate):
562 # while(thisDate <= endDate):
473 year = thisDate.timetuple().tm_year
563 # year = thisDate.timetuple().tm_year
474 doy = thisDate.timetuple().tm_yday
564 # doy = thisDate.timetuple().tm_yday
475
565 #
476 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
566 # matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
477 if len(matchlist) == 0:
567 # if len(matchlist) == 0:
478 thisDate += datetime.timedelta(1)
568 # thisDate += datetime.timedelta(1)
479 continue
569 # continue
480 for match in matchlist:
570 # for match in matchlist:
481 pathList.append(os.path.join(single_path,match,expLabel))
571 # pathList.append(os.path.join(single_path,match,expLabel))
482
572 #
483 thisDate += datetime.timedelta(1)
573 # thisDate += datetime.timedelta(1)
484 else:
574 # else:
485 for thiDir in dirList:
575 # for thiDir in dirList:
486 pathList.append(os.path.join(single_path,thiDir,expLabel))
576 # pathList.append(os.path.join(single_path,thiDir,expLabel))
487
577
488 if pathList == []:
578 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
579
580 if dateList == []:
489 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
581 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
490 return None, None
582 return None, None
491
583
492 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
584 if len(dateList) > 1:
493
585 print "%d dates with data were found for the date range: %s - %s" %(len(dateList), startDate, endDate)
586 else:
587 print "data was found for the date %s" %(dateList[0])
588
494 filenameList = []
589 filenameList = []
495 datetimeList = []
590 datetimeList = []
496 pathDict = {}
591 # pathDict = {}
497 filenameList_to_sort = []
592 # filenameList_to_sort = []
498
593 #
499 for i in range(len(pathList)):
594 # for i in range(len(pathList)):
500
595 #
501 thisPath = pathList[i]
596 # thisPath = pathList[i]
502
597 #
503 fileList = glob.glob1(thisPath, "*%s" %ext)
598 # fileList = glob.glob1(thisPath, "*%s" %ext)
504 if len(fileList) < 1:
599 # if len(fileList) < 1:
505 continue
600 # continue
506 fileList.sort()
601 # fileList.sort()
507 pathDict.setdefault(fileList[0])
602 # pathDict.setdefault(fileList[0])
508 pathDict[fileList[0]] = i
603 # pathDict[fileList[0]] = i
509 filenameList_to_sort.append(fileList[0])
604 # filenameList_to_sort.append(fileList[0])
510
605 #
511 filenameList_to_sort.sort()
606 # filenameList_to_sort.sort()
512
607
513 for file in filenameList_to_sort:
608 for thisPath in pathList:
514 thisPath = pathList[pathDict[file]]
609 # thisPath = pathList[pathDict[file]]
515
610
516 fileList = glob.glob1(thisPath, "*%s" %ext)
611 fileList = glob.glob1(thisPath, "*%s" %ext)
517 fileList.sort()
612 fileList.sort()
518
613
519 for file in fileList:
614 for file in fileList:
520
615
521 filename = os.path.join(thisPath,file)
616 filename = os.path.join(thisPath,file)
522 thisDatetime = isFileinThisTime(filename, startTime, endTime)
617
618 if not isFileInDateRange(filename, startDate, endDate):
619 continue
620
621 thisDatetime = isFileInTimeRange(filename, startTime, endTime)
523
622
524 if not(thisDatetime):
623 if not(thisDatetime):
525 continue
624 continue
526
625
527 filenameList.append(filename)
626 filenameList.append(filename)
528 datetimeList.append(thisDatetime)
627 datetimeList.append(thisDatetime)
529
628
530 if not(filenameList):
629 if not(filenameList):
531 print "Any file was found for the time range %s - %s" %(startTime, endTime)
630 print "Any file was found for the time range %s - %s" %(startTime, endTime)
532 return None, None
631 return None, None
533
632
534 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
633 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
535 print
634 print
536
635
537 for i in range(len(filenameList)):
636 for i in range(len(filenameList)):
538 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
637 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
539
638
540 self.filenameList = filenameList
639 self.filenameList = filenameList
541 self.datetimeList = datetimeList
640 self.datetimeList = datetimeList
542
641
543 return pathList, filenameList
642 return pathList, filenameList
544
643
545 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
644 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
546
645
547 """
646 """
548 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
647 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
549 devuelve el archivo encontrado ademas de otros datos.
648 devuelve el archivo encontrado ademas de otros datos.
550
649
551 Input:
650 Input:
552 path : carpeta donde estan contenidos los files que contiene data
651 path : carpeta donde estan contenidos los files que contiene data
553
652
554 expLabel : Nombre del subexperimento (subfolder)
653 expLabel : Nombre del subexperimento (subfolder)
555
654
556 ext : extension de los files
655 ext : extension de los files
557
656
558 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
657 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
559
658
560 Return:
659 Return:
561 directory : eL directorio donde esta el file encontrado
660 directory : eL directorio donde esta el file encontrado
562 filename : el ultimo file de una determinada carpeta
661 filename : el ultimo file de una determinada carpeta
563 year : el anho
662 year : el anho
564 doy : el numero de dia del anho
663 doy : el numero de dia del anho
565 set : el set del archivo
664 set : el set del archivo
566
665
567
666
568 """
667 """
569 dirList = []
668 dirList = []
570
669
571 if not walk:
670 if not walk:
572 fullpath = path
671 fullpath = path
573 foldercounter = 0
672 foldercounter = 0
574 else:
673 else:
575 #Filtra solo los directorios
674 #Filtra solo los directorios
576 for thisPath in os.listdir(path):
675 for thisPath in os.listdir(path):
577 if not os.path.isdir(os.path.join(path,thisPath)):
676 if not os.path.isdir(os.path.join(path,thisPath)):
578 continue
677 continue
579 if not isRadarFolder(thisPath):
678 if not isRadarFolder(thisPath):
580 continue
679 continue
581
680
582 dirList.append(thisPath)
681 dirList.append(thisPath)
583
682
584 if not(dirList):
683 if not(dirList):
585 return None, None, None, None, None, None
684 return None, None, None, None, None, None
586
685
587 dirList = sorted( dirList, key=str.lower )
686 dirList = sorted( dirList, key=str.lower )
588
687
589 doypath = dirList[-1]
688 doypath = dirList[-1]
590 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
689 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
591 fullpath = os.path.join(path, doypath, expLabel)
690 fullpath = os.path.join(path, doypath, expLabel)
592
691
593
692
594 print "%s folder was found: " %(fullpath )
693 print "%s folder was found: " %(fullpath )
595
694
596 if set == None:
695 if set == None:
597 filename = getlastFileFromPath(fullpath, ext)
696 filename = getlastFileFromPath(fullpath, ext)
598 else:
697 else:
599 filename = getFileFromSet(fullpath, ext, set)
698 filename = getFileFromSet(fullpath, ext, set)
600
699
601 if not(filename):
700 if not(filename):
602 return None, None, None, None, None, None
701 return None, None, None, None, None, None
603
702
604 print "%s file was found" %(filename)
703 print "%s file was found" %(filename)
605
704
606 if not(self.__verifyFile(os.path.join(fullpath, filename))):
705 if not(self.__verifyFile(os.path.join(fullpath, filename))):
607 return None, None, None, None, None, None
706 return None, None, None, None, None, None
608
707
609 year = int( filename[1:5] )
708 year = int( filename[1:5] )
610 doy = int( filename[5:8] )
709 doy = int( filename[5:8] )
611 set = int( filename[8:11] )
710 set = int( filename[8:11] )
612
711
613 return fullpath, foldercounter, filename, year, doy, set
712 return fullpath, foldercounter, filename, year, doy, set
614
713
615 def __setNextFileOffline(self):
714 def __setNextFileOffline(self):
616
715
617 idFile = self.fileIndex
716 idFile = self.fileIndex
618
717
619 while (True):
718 while (True):
620 idFile += 1
719 idFile += 1
621 if not(idFile < len(self.filenameList)):
720 if not(idFile < len(self.filenameList)):
622 self.flagNoMoreFiles = 1
721 self.flagNoMoreFiles = 1
623 # print "[Reading] No more Files"
722 # print "[Reading] No more Files"
624 return 0
723 return 0
625
724
626 filename = self.filenameList[idFile]
725 filename = self.filenameList[idFile]
627
726
628 if not(self.__verifyFile(filename)):
727 if not(self.__verifyFile(filename)):
629 continue
728 continue
630
729
631 fileSize = os.path.getsize(filename)
730 fileSize = os.path.getsize(filename)
632 fp = open(filename,'rb')
731 fp = open(filename,'rb')
633 break
732 break
634
733
635 self.flagIsNewFile = 1
734 self.flagIsNewFile = 1
636 self.fileIndex = idFile
735 self.fileIndex = idFile
637 self.filename = filename
736 self.filename = filename
638 self.fileSize = fileSize
737 self.fileSize = fileSize
639 self.fp = fp
738 self.fp = fp
640
739
641 # print "[Reading] Setting the file: %s"%self.filename
740 # print "[Reading] Setting the file: %s"%self.filename
642
741
643 return 1
742 return 1
644
743
645 def __setNextFileOnline(self):
744 def __setNextFileOnline(self):
646 """
745 """
647 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
746 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
648 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
747 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
649 siguientes.
748 siguientes.
650
749
651 Affected:
750 Affected:
652 self.flagIsNewFile
751 self.flagIsNewFile
653 self.filename
752 self.filename
654 self.fileSize
753 self.fileSize
655 self.fp
754 self.fp
656 self.set
755 self.set
657 self.flagNoMoreFiles
756 self.flagNoMoreFiles
658
757
659 Return:
758 Return:
660 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
759 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
661 1 : si el file fue abierto con exito y esta listo a ser leido
760 1 : si el file fue abierto con exito y esta listo a ser leido
662
761
663 Excepciones:
762 Excepciones:
664 Si un determinado file no puede ser abierto
763 Si un determinado file no puede ser abierto
665 """
764 """
666 nFiles = 0
765 nFiles = 0
667 fileOk_flag = False
766 fileOk_flag = False
668 firstTime_flag = True
767 firstTime_flag = True
669
768
670 self.set += 1
769 self.set += 1
671
770
672 if self.set > 999:
771 if self.set > 999:
673 self.set = 0
772 self.set = 0
674 self.foldercounter += 1
773 self.foldercounter += 1
675
774
676 #busca el 1er file disponible
775 #busca el 1er file disponible
677 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
776 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
678 if fullfilename:
777 if fullfilename:
679 if self.__verifyFile(fullfilename, False):
778 if self.__verifyFile(fullfilename, False):
680 fileOk_flag = True
779 fileOk_flag = True
681
780
682 #si no encuentra un file entonces espera y vuelve a buscar
781 #si no encuentra un file entonces espera y vuelve a buscar
683 if not(fileOk_flag):
782 if not(fileOk_flag):
684 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
783 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
685
784
686 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
785 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
687 tries = self.nTries
786 tries = self.nTries
688 else:
787 else:
689 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
788 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
690
789
691 for nTries in range( tries ):
790 for nTries in range( tries ):
692 if firstTime_flag:
791 if firstTime_flag:
693 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
792 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
694 sleep( self.delay )
793 sleep( self.delay )
695 else:
794 else:
696 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
795 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
697
796
698 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
797 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
699 if fullfilename:
798 if fullfilename:
700 if self.__verifyFile(fullfilename):
799 if self.__verifyFile(fullfilename):
701 fileOk_flag = True
800 fileOk_flag = True
702 break
801 break
703
802
704 if fileOk_flag:
803 if fileOk_flag:
705 break
804 break
706
805
707 firstTime_flag = False
806 firstTime_flag = False
708
807
709 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
808 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
710 self.set += 1
809 self.set += 1
711
810
712 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
811 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
713 self.set = 0
812 self.set = 0
714 self.doy += 1
813 self.doy += 1
715 self.foldercounter = 0
814 self.foldercounter = 0
716
815
717 if fileOk_flag:
816 if fileOk_flag:
718 self.fileSize = os.path.getsize( fullfilename )
817 self.fileSize = os.path.getsize( fullfilename )
719 self.filename = fullfilename
818 self.filename = fullfilename
720 self.flagIsNewFile = 1
819 self.flagIsNewFile = 1
721 if self.fp != None: self.fp.close()
820 if self.fp != None: self.fp.close()
722 self.fp = open(fullfilename, 'rb')
821 self.fp = open(fullfilename, 'rb')
723 self.flagNoMoreFiles = 0
822 self.flagNoMoreFiles = 0
724 # print '[Reading] Setting the file: %s' % fullfilename
823 # print '[Reading] Setting the file: %s' % fullfilename
725 else:
824 else:
726 self.fileSize = 0
825 self.fileSize = 0
727 self.filename = None
826 self.filename = None
728 self.flagIsNewFile = 0
827 self.flagIsNewFile = 0
729 self.fp = None
828 self.fp = None
730 self.flagNoMoreFiles = 1
829 self.flagNoMoreFiles = 1
731 # print '[Reading] No more files to read'
830 # print '[Reading] No more files to read'
732
831
733 return fileOk_flag
832 return fileOk_flag
734
833
735 def setNextFile(self):
834 def setNextFile(self):
736 if self.fp != None:
835 if self.fp != None:
737 self.fp.close()
836 self.fp.close()
738
837
739 if self.online:
838 if self.online:
740 newFile = self.__setNextFileOnline()
839 newFile = self.__setNextFileOnline()
741 else:
840 else:
742 newFile = self.__setNextFileOffline()
841 newFile = self.__setNextFileOffline()
743
842
744 if not(newFile):
843 if not(newFile):
745 print '[Reading] No more files to read'
844 print '[Reading] No more files to read'
746 return 0
845 return 0
747
846
748 print '[Reading] Setting the file: %s' % self.filename
847 print '[Reading] Setting the file: %s' % self.filename
749
848
750 self.__readFirstHeader()
849 self.__readFirstHeader()
751 self.nReadBlocks = 0
850 self.nReadBlocks = 0
752 return 1
851 return 1
753
852
754 def __waitNewBlock(self):
853 def __waitNewBlock(self):
755 """
854 """
756 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
855 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
757
856
758 Si el modo de lectura es OffLine siempre retorn 0
857 Si el modo de lectura es OffLine siempre retorn 0
759 """
858 """
760 if not self.online:
859 if not self.online:
761 return 0
860 return 0
762
861
763 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
862 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
764 return 0
863 return 0
765
864
766 currentPointer = self.fp.tell()
865 currentPointer = self.fp.tell()
767
866
768 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
867 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
769
868
770 for nTries in range( self.nTries ):
869 for nTries in range( self.nTries ):
771
870
772 self.fp.close()
871 self.fp.close()
773 self.fp = open( self.filename, 'rb' )
872 self.fp = open( self.filename, 'rb' )
774 self.fp.seek( currentPointer )
873 self.fp.seek( currentPointer )
775
874
776 self.fileSize = os.path.getsize( self.filename )
875 self.fileSize = os.path.getsize( self.filename )
777 currentSize = self.fileSize - currentPointer
876 currentSize = self.fileSize - currentPointer
778
877
779 if ( currentSize >= neededSize ):
878 if ( currentSize >= neededSize ):
780 self.basicHeaderObj.read(self.fp)
879 self.basicHeaderObj.read(self.fp)
781 return 1
880 return 1
782
881
783 if self.fileSize == self.fileSizeByHeader:
882 if self.fileSize == self.fileSizeByHeader:
784 # self.flagEoF = True
883 # self.flagEoF = True
785 return 0
884 return 0
786
885
787 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
886 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
788 sleep( self.delay )
887 sleep( self.delay )
789
888
790
889
791 return 0
890 return 0
792
891
793 def waitDataBlock(self,pointer_location):
892 def waitDataBlock(self,pointer_location):
794
893
795 currentPointer = pointer_location
894 currentPointer = pointer_location
796
895
797 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
896 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
798
897
799 for nTries in range( self.nTries ):
898 for nTries in range( self.nTries ):
800 self.fp.close()
899 self.fp.close()
801 self.fp = open( self.filename, 'rb' )
900 self.fp = open( self.filename, 'rb' )
802 self.fp.seek( currentPointer )
901 self.fp.seek( currentPointer )
803
902
804 self.fileSize = os.path.getsize( self.filename )
903 self.fileSize = os.path.getsize( self.filename )
805 currentSize = self.fileSize - currentPointer
904 currentSize = self.fileSize - currentPointer
806
905
807 if ( currentSize >= neededSize ):
906 if ( currentSize >= neededSize ):
808 return 1
907 return 1
809
908
810 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
909 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
811 sleep( self.delay )
910 sleep( self.delay )
812
911
813 return 0
912 return 0
814
913
815 def __jumpToLastBlock(self):
914 def __jumpToLastBlock(self):
816
915
817 if not(self.__isFirstTimeOnline):
916 if not(self.__isFirstTimeOnline):
818 return
917 return
819
918
820 csize = self.fileSize - self.fp.tell()
919 csize = self.fileSize - self.fp.tell()
821 blocksize = self.processingHeaderObj.blockSize
920 blocksize = self.processingHeaderObj.blockSize
822
921
823 #salta el primer bloque de datos
922 #salta el primer bloque de datos
824 if csize > self.processingHeaderObj.blockSize:
923 if csize > self.processingHeaderObj.blockSize:
825 self.fp.seek(self.fp.tell() + blocksize)
924 self.fp.seek(self.fp.tell() + blocksize)
826 else:
925 else:
827 return
926 return
828
927
829 csize = self.fileSize - self.fp.tell()
928 csize = self.fileSize - self.fp.tell()
830 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
929 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
831 while True:
930 while True:
832
931
833 if self.fp.tell()<self.fileSize:
932 if self.fp.tell()<self.fileSize:
834 self.fp.seek(self.fp.tell() + neededsize)
933 self.fp.seek(self.fp.tell() + neededsize)
835 else:
934 else:
836 self.fp.seek(self.fp.tell() - neededsize)
935 self.fp.seek(self.fp.tell() - neededsize)
837 break
936 break
838
937
839 # csize = self.fileSize - self.fp.tell()
938 # csize = self.fileSize - self.fp.tell()
840 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
939 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
841 # factor = int(csize/neededsize)
940 # factor = int(csize/neededsize)
842 # if factor > 0:
941 # if factor > 0:
843 # self.fp.seek(self.fp.tell() + factor*neededsize)
942 # self.fp.seek(self.fp.tell() + factor*neededsize)
844
943
845 self.flagIsNewFile = 0
944 self.flagIsNewFile = 0
846 self.__isFirstTimeOnline = 0
945 self.__isFirstTimeOnline = 0
847
946
848 def __setNewBlock(self):
947 def __setNewBlock(self):
849
948
850 if self.fp == None:
949 if self.fp == None:
851 return 0
950 return 0
852
951
853 if self.online:
952 if self.online:
854 self.__jumpToLastBlock()
953 self.__jumpToLastBlock()
855
954
856 if self.flagIsNewFile:
955 if self.flagIsNewFile:
857 return 1
956 return 1
858
957
859 self.lastUTTime = self.basicHeaderObj.utc
958 self.lastUTTime = self.basicHeaderObj.utc
860 currentSize = self.fileSize - self.fp.tell()
959 currentSize = self.fileSize - self.fp.tell()
861 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
960 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
862
961
863 if (currentSize >= neededSize):
962 if (currentSize >= neededSize):
864 self.basicHeaderObj.read(self.fp)
963 self.basicHeaderObj.read(self.fp)
865 return 1
964 return 1
866
965
867 if self.__waitNewBlock():
966 if self.__waitNewBlock():
868 return 1
967 return 1
869
968
870 if not(self.setNextFile()):
969 if not(self.setNextFile()):
871 return 0
970 return 0
872
971
873 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
972 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
874
973
875 self.flagDiscontinuousBlock = 0
974 self.flagDiscontinuousBlock = 0
876
975
877 if deltaTime > self.maxTimeStep:
976 if deltaTime > self.maxTimeStep:
878 self.flagDiscontinuousBlock = 1
977 self.flagDiscontinuousBlock = 1
879
978
880 return 1
979 return 1
881
980
882 def readNextBlock(self):
981 def readNextBlock(self):
883
982
884 if not(self.__setNewBlock()):
983 if not(self.__setNewBlock()):
885 return 0
984 return 0
886
985
887 if not(self.readBlock()):
986 if not(self.readBlock()):
888 return 0
987 return 0
889
988
890 self.getBasicHeader()
989 self.getBasicHeader()
891
990
892 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
991 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
893 self.processingHeaderObj.dataBlocksPerFile,
992 self.processingHeaderObj.dataBlocksPerFile,
894 self.dataOut.datatime.ctime())
993 self.dataOut.datatime.ctime())
895 return 1
994 return 1
896
995
897 def __readFirstHeader(self):
996 def __readFirstHeader(self):
898
997
899 self.basicHeaderObj.read(self.fp)
998 self.basicHeaderObj.read(self.fp)
900 self.systemHeaderObj.read(self.fp)
999 self.systemHeaderObj.read(self.fp)
901 self.radarControllerHeaderObj.read(self.fp)
1000 self.radarControllerHeaderObj.read(self.fp)
902 self.processingHeaderObj.read(self.fp)
1001 self.processingHeaderObj.read(self.fp)
903
1002
904 self.firstHeaderSize = self.basicHeaderObj.size
1003 self.firstHeaderSize = self.basicHeaderObj.size
905
1004
906 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1005 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
907 if datatype == 0:
1006 if datatype == 0:
908 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1007 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
909 elif datatype == 1:
1008 elif datatype == 1:
910 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1009 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
911 elif datatype == 2:
1010 elif datatype == 2:
912 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1011 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
913 elif datatype == 3:
1012 elif datatype == 3:
914 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1013 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
915 elif datatype == 4:
1014 elif datatype == 4:
916 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1015 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
917 elif datatype == 5:
1016 elif datatype == 5:
918 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1017 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
919 else:
1018 else:
920 raise ValueError, 'Data type was not defined'
1019 raise ValueError, 'Data type was not defined'
921
1020
922 self.dtype = datatype_str
1021 self.dtype = datatype_str
923 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1022 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
924 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1023 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
925 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1024 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
926 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1025 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
927 self.getBlockDimension()
1026 self.getBlockDimension()
928
1027
929 def __verifyFile(self, filename, msgFlag=True):
1028 def __verifyFile(self, filename, msgFlag=True):
930 msg = None
1029 msg = None
931 try:
1030 try:
932 fp = open(filename, 'rb')
1031 fp = open(filename, 'rb')
933 currentPosition = fp.tell()
1032 currentPosition = fp.tell()
934 except IOError:
1033 except IOError:
935 traceback.print_exc()
1034 traceback.print_exc()
936 if msgFlag:
1035 if msgFlag:
937 print "[Reading] The file %s can't be opened" % (filename)
1036 print "[Reading] The file %s can't be opened" % (filename)
938 return False
1037 return False
939
1038
940 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1039 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
941
1040
942 if neededSize == 0:
1041 if neededSize == 0:
943 basicHeaderObj = BasicHeader(LOCALTIME)
1042 basicHeaderObj = BasicHeader(LOCALTIME)
944 systemHeaderObj = SystemHeader()
1043 systemHeaderObj = SystemHeader()
945 radarControllerHeaderObj = RadarControllerHeader()
1044 radarControllerHeaderObj = RadarControllerHeader()
946 processingHeaderObj = ProcessingHeader()
1045 processingHeaderObj = ProcessingHeader()
947
1046
948 try:
1047 try:
949 if not( basicHeaderObj.read(fp) ): raise IOError
1048 if not( basicHeaderObj.read(fp) ): raise IOError
950 if not( systemHeaderObj.read(fp) ): raise IOError
1049 if not( systemHeaderObj.read(fp) ): raise IOError
951 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1050 if not( radarControllerHeaderObj.read(fp) ): raise IOError
952 if not( processingHeaderObj.read(fp) ): raise IOError
1051 if not( processingHeaderObj.read(fp) ): raise IOError
953 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1052 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
954
1053
955 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1054 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
956
1055
957 except IOError:
1056 except IOError:
958 traceback.print_exc()
1057 traceback.print_exc()
959 sys.exit(0)
1058 sys.exit(0)
960
1059
961 if msgFlag:
1060 if msgFlag:
962 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1061 print "[Reading] The file %s is empty or it hasn't enough data" % filename
963
1062
964 fp.close()
1063 fp.close()
965 return False
1064 return False
966 else:
1065 else:
967 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1066 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
968
1067
969 fp.close()
1068 fp.close()
970 fileSize = os.path.getsize(filename)
1069 fileSize = os.path.getsize(filename)
971 currentSize = fileSize - currentPosition
1070 currentSize = fileSize - currentPosition
972 if currentSize < neededSize:
1071 if currentSize < neededSize:
973 if msgFlag and (msg != None):
1072 if msgFlag and (msg != None):
974 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1073 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
975 return False
1074 return False
976
1075
977 return True
1076 return True
978
1077
979 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True):
1078 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
980
1079
981 dateList = []
1080 dateList = []
982 pathList = []
1081 pathList = []
983
1082
1083 multi_path = path.split(',')
1084
984 if not walk:
1085 if not walk:
985 #pathList.append(path)
1086
986 multi_path = path.split(',')
987 for single_path in multi_path:
1087 for single_path in multi_path:
988
1088
989 if not os.path.isdir(single_path):
1089 if not os.path.isdir(single_path):
990 continue
1090 continue
991
1091
992 ok = False
993 fileList = glob.glob1(single_path, "*"+ext)
1092 fileList = glob.glob1(single_path, "*"+ext)
994
1093
995 for thisFile in fileList:
1094 for thisFile in fileList:
996
1095
997 if not os.path.isfile(os.path.join(single_path, thisFile)):
1096 if not os.path.isfile(os.path.join(single_path, thisFile)):
998 continue
1097 continue
999
1098
1000 if not isRadarFile(thisFile):
1099 if not isRadarFile(thisFile):
1001 continue
1100 continue
1002
1101
1003 ok = True
1102 if not isFileInDateRange(thisFile, startDate, endDate):
1103 continue
1104
1004 thisDate = getDateFromRadarFile(thisFile)
1105 thisDate = getDateFromRadarFile(thisFile)
1005
1106
1006 if thisDate not in dateList:
1107 if thisDate in dateList:
1007 dateList.append(thisDate)
1108 continue
1008
1109
1009 if ok:
1110 dateList.append(thisDate)
1010 pathList.append(single_path)
1111 pathList.append(single_path)
1011
1012 return dateList
1013
1112
1014 multi_path = path.split(',')
1113 else:
1015 for single_path in multi_path:
1114 for single_path in multi_path:
1016
1017 if not os.path.isdir(single_path):
1018 continue
1019
1020 dirList = []
1021
1022 for thisPath in os.listdir(single_path):
1023
1115
1024 if not os.path.isdir(os.path.join(single_path,thisPath)):
1116 if not os.path.isdir(single_path):
1025 continue
1117 continue
1118
1119 dirList = []
1026
1120
1027 if not isRadarFolder(thisPath):
1121 for thisPath in os.listdir(single_path):
1028 continue
1029
1030 dirList.append(thisPath)
1031
1032 if not dirList:
1033 return dateList
1034
1035 if startDate and endDate:
1036 thisDate = startDate
1037
1038 while(thisDate <= endDate):
1039 year = thisDate.timetuple().tm_year
1040 doy = thisDate.timetuple().tm_yday
1041
1122
1042 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
1123 if not os.path.isdir(os.path.join(single_path,thisPath)):
1043 if len(matchlist) == 0:
1044 thisDate += datetime.timedelta(1)
1045 continue
1124 continue
1046
1125
1047 for match in matchlist:
1126 if not isRadarFolder(thisPath):
1048
1127 continue
1049 datapath = os.path.join(single_path, match, expLabel)
1050 fileList = glob.glob1(datapath, "*"+ext)
1051
1052 if len(fileList) < 1:
1053 continue
1054
1055 pathList.append(datapath)
1056 dateList.append(thisDate)
1057
1128
1058 thisDate += datetime.timedelta(1)
1129 if not isFolderInDateRange(thisPath, startDate, endDate):
1059 else:
1130 continue
1131
1132 dirList.append(thisPath)
1133
1134 if not dirList:
1135 continue
1136
1060 for thisDir in dirList:
1137 for thisDir in dirList:
1061
1138
1062 datapath = os.path.join(single_path, thisDir, expLabel)
1139 datapath = os.path.join(single_path, thisDir, expLabel)
1063 fileList = glob.glob1(datapath, "*"+ext)
1140 fileList = glob.glob1(datapath, "*"+ext)
1064
1141
1065 if len(fileList) < 1:
1142 if len(fileList) < 1:
1066 continue
1143 continue
1067
1144
1068 year = int(thisDir[1:5])
1145 thisDate = getDateFromRadarFolder(thisDir)
1069 doy = int(thisDir[5:8])
1070 thisDate = datetime.date(year,1,1) + datetime.timedelta(doy-1)
1071
1146
1072 pathList.append(datapath)
1147 pathList.append(datapath)
1073 dateList.append(thisDate)
1148 dateList.append(thisDate)
1074
1075 return dateList
1076
1149
1150 dateList.sort()
1151
1152 if include_path:
1153 return dateList, pathList
1154
1155 return dateList
1077
1156
1078 def setup(self,
1157 def setup(self,
1079 path=None,
1158 path=None,
1080 startDate=None,
1159 startDate=None,
1081 endDate=None,
1160 endDate=None,
1082 startTime=datetime.time(0,0,0),
1161 startTime=datetime.time(0,0,0),
1083 endTime=datetime.time(23,59,59),
1162 endTime=datetime.time(23,59,59),
1084 set=None,
1163 set=None,
1085 expLabel = "",
1164 expLabel = "",
1086 ext = None,
1165 ext = None,
1087 online = False,
1166 online = False,
1088 delay = 60,
1167 delay = 60,
1089 walk = True,
1168 walk = True,
1090 getblock = False,
1169 getblock = False,
1091 nTxs = 1):
1170 nTxs = 1):
1092
1171
1093 if path == None:
1172 if path == None:
1094 raise ValueError, "[Reading] The path is not valid"
1173 raise ValueError, "[Reading] The path is not valid"
1095
1174
1096 if ext == None:
1175 if ext == None:
1097 ext = self.ext
1176 ext = self.ext
1098
1177
1099 if online:
1178 if online:
1100 print "[Reading] Searching files in online mode..."
1179 print "[Reading] Searching files in online mode..."
1101
1180
1102 for nTries in range( self.nTries ):
1181 for nTries in range( self.nTries ):
1103 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1182 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1104
1183
1105 if fullpath:
1184 if fullpath:
1106 break
1185 break
1107
1186
1108 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1187 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1109 sleep( self.delay )
1188 sleep( self.delay )
1110
1189
1111 if not(fullpath):
1190 if not(fullpath):
1112 print "[Reading] There 'isn't any valid file in %s" % path
1191 print "[Reading] There 'isn't any valid file in %s" % path
1113 return None
1192 return None
1114
1193
1115 self.year = year
1194 self.year = year
1116 self.doy = doy
1195 self.doy = doy
1117 self.set = set - 1
1196 self.set = set - 1
1118 self.path = path
1197 self.path = path
1119 self.foldercounter = foldercounter
1198 self.foldercounter = foldercounter
1120 last_set = None
1199 last_set = None
1121
1200
1122 else:
1201 else:
1123 print "[Reading] Searching files in offline mode ..."
1202 print "[Reading] Searching files in offline mode ..."
1124 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1203 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1125 startTime=startTime, endTime=endTime,
1204 startTime=startTime, endTime=endTime,
1126 set=set, expLabel=expLabel, ext=ext,
1205 set=set, expLabel=expLabel, ext=ext,
1127 walk=walk)
1206 walk=walk)
1128
1207
1129 if not(pathList):
1208 if not(pathList):
1130 print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
1209 print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
1131 datetime.datetime.combine(startDate,startTime).ctime(),
1210 datetime.datetime.combine(startDate,startTime).ctime(),
1132 datetime.datetime.combine(endDate,endTime).ctime())
1211 datetime.datetime.combine(endDate,endTime).ctime())
1133
1212
1134 sys.exit(-1)
1213 sys.exit(-1)
1135
1214
1136
1215
1137 self.fileIndex = -1
1216 self.fileIndex = -1
1138 self.pathList = pathList
1217 self.pathList = pathList
1139 self.filenameList = filenameList
1218 self.filenameList = filenameList
1140 file_name = os.path.basename(filenameList[-1])
1219 file_name = os.path.basename(filenameList[-1])
1141 basename, ext = os.path.splitext(file_name)
1220 basename, ext = os.path.splitext(file_name)
1142 last_set = int(basename[-3:])
1221 last_set = int(basename[-3:])
1143
1222
1144 self.online = online
1223 self.online = online
1145 self.delay = delay
1224 self.delay = delay
1146 ext = ext.lower()
1225 ext = ext.lower()
1147 self.ext = ext
1226 self.ext = ext
1148 self.getByBlock = getblock
1227 self.getByBlock = getblock
1149 self.nTxs = int(nTxs)
1228 self.nTxs = int(nTxs)
1150
1229
1151 if not(self.setNextFile()):
1230 if not(self.setNextFile()):
1152 if (startDate!=None) and (endDate!=None):
1231 if (startDate!=None) and (endDate!=None):
1153 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1232 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1154 elif startDate != None:
1233 elif startDate != None:
1155 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1234 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1156 else:
1235 else:
1157 print "[Reading] No files"
1236 print "[Reading] No files"
1158
1237
1159 sys.exit(-1)
1238 sys.exit(-1)
1160
1239
1161 # self.getBasicHeader()
1240 # self.getBasicHeader()
1162
1241
1163 if last_set != None:
1242 if last_set != None:
1164 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1243 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1165 return
1244 return
1166
1245
1167 def getBasicHeader(self):
1246 def getBasicHeader(self):
1168
1247
1169 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1248 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1170
1249
1171 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1250 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1172
1251
1173 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1252 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1174
1253
1175 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1254 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1176
1255
1177 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1256 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1178
1257
1179 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1258 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1180
1259
1181 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1260 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1182
1261
1183 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1262 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1184
1263
1185
1264
1186 def getFirstHeader(self):
1265 def getFirstHeader(self):
1187
1266
1188 raise ValueError, "This method has not been implemented"
1267 raise ValueError, "This method has not been implemented"
1189
1268
1190 def getData(self):
1269 def getData(self):
1191
1270
1192 raise ValueError, "This method has not been implemented"
1271 raise ValueError, "This method has not been implemented"
1193
1272
1194 def hasNotDataInBuffer(self):
1273 def hasNotDataInBuffer(self):
1195
1274
1196 raise ValueError, "This method has not been implemented"
1275 raise ValueError, "This method has not been implemented"
1197
1276
1198 def readBlock(self):
1277 def readBlock(self):
1199
1278
1200 raise ValueError, "This method has not been implemented"
1279 raise ValueError, "This method has not been implemented"
1201
1280
1202 def isEndProcess(self):
1281 def isEndProcess(self):
1203
1282
1204 return self.flagNoMoreFiles
1283 return self.flagNoMoreFiles
1205
1284
1206 def printReadBlocks(self):
1285 def printReadBlocks(self):
1207
1286
1208 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1287 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1209
1288
1210 def printTotalBlocks(self):
1289 def printTotalBlocks(self):
1211
1290
1212 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1291 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1213
1292
1214 def printNumberOfBlock(self):
1293 def printNumberOfBlock(self):
1215
1294
1216 if self.flagIsNewBlock:
1295 if self.flagIsNewBlock:
1217 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1296 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1218 self.processingHeaderObj.dataBlocksPerFile,
1297 self.processingHeaderObj.dataBlocksPerFile,
1219 self.dataOut.datatime.ctime())
1298 self.dataOut.datatime.ctime())
1220
1299
1221 def printInfo(self):
1300 def printInfo(self):
1222
1301
1223 if self.__printInfo == False:
1302 if self.__printInfo == False:
1224 return
1303 return
1225
1304
1226 self.basicHeaderObj.printInfo()
1305 self.basicHeaderObj.printInfo()
1227 self.systemHeaderObj.printInfo()
1306 self.systemHeaderObj.printInfo()
1228 self.radarControllerHeaderObj.printInfo()
1307 self.radarControllerHeaderObj.printInfo()
1229 self.processingHeaderObj.printInfo()
1308 self.processingHeaderObj.printInfo()
1230
1309
1231 self.__printInfo = False
1310 self.__printInfo = False
1232
1311
1233
1312
1234 def run(self, **kwargs):
1313 def run(self, **kwargs):
1235
1314
1236 if not(self.isConfig):
1315 if not(self.isConfig):
1237
1316
1238 # self.dataOut = dataOut
1317 # self.dataOut = dataOut
1239 self.setup(**kwargs)
1318 self.setup(**kwargs)
1240 self.isConfig = True
1319 self.isConfig = True
1241
1320
1242 self.getData()
1321 self.getData()
1243
1322
1244 class JRODataWriter(JRODataIO):
1323 class JRODataWriter(JRODataIO):
1245
1324
1246 """
1325 """
1247 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1326 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1248 de los datos siempre se realiza por bloques.
1327 de los datos siempre se realiza por bloques.
1249 """
1328 """
1250
1329
1251 blockIndex = 0
1330 blockIndex = 0
1252
1331
1253 path = None
1332 path = None
1254
1333
1255 setFile = None
1334 setFile = None
1256
1335
1257 profilesPerBlock = None
1336 profilesPerBlock = None
1258
1337
1259 blocksPerFile = None
1338 blocksPerFile = None
1260
1339
1261 nWriteBlocks = 0
1340 nWriteBlocks = 0
1262
1341
1263 fileDate = None
1342 fileDate = None
1264
1343
1265 def __init__(self, dataOut=None):
1344 def __init__(self, dataOut=None):
1266 raise ValueError, "Not implemented"
1345 raise ValueError, "Not implemented"
1267
1346
1268
1347
1269 def hasAllDataInBuffer(self):
1348 def hasAllDataInBuffer(self):
1270 raise ValueError, "Not implemented"
1349 raise ValueError, "Not implemented"
1271
1350
1272
1351
1273 def setBlockDimension(self):
1352 def setBlockDimension(self):
1274 raise ValueError, "Not implemented"
1353 raise ValueError, "Not implemented"
1275
1354
1276
1355
1277 def writeBlock(self):
1356 def writeBlock(self):
1278 raise ValueError, "No implemented"
1357 raise ValueError, "No implemented"
1279
1358
1280
1359
1281 def putData(self):
1360 def putData(self):
1282 raise ValueError, "No implemented"
1361 raise ValueError, "No implemented"
1283
1362
1284
1363
1285 def getProcessFlags(self):
1364 def getProcessFlags(self):
1286
1365
1287 processFlags = 0
1366 processFlags = 0
1288
1367
1289 dtype_index = get_dtype_index(self.dtype)
1368 dtype_index = get_dtype_index(self.dtype)
1290 procflag_dtype = get_procflag_dtype(dtype_index)
1369 procflag_dtype = get_procflag_dtype(dtype_index)
1291
1370
1292 processFlags += procflag_dtype
1371 processFlags += procflag_dtype
1293
1372
1294 if self.dataOut.flagDecodeData:
1373 if self.dataOut.flagDecodeData:
1295 processFlags += PROCFLAG.DECODE_DATA
1374 processFlags += PROCFLAG.DECODE_DATA
1296
1375
1297 if self.dataOut.flagDeflipData:
1376 if self.dataOut.flagDeflipData:
1298 processFlags += PROCFLAG.DEFLIP_DATA
1377 processFlags += PROCFLAG.DEFLIP_DATA
1299
1378
1300 if self.dataOut.code is not None:
1379 if self.dataOut.code is not None:
1301 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1380 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1302
1381
1303 if self.dataOut.nCohInt > 1:
1382 if self.dataOut.nCohInt > 1:
1304 processFlags += PROCFLAG.COHERENT_INTEGRATION
1383 processFlags += PROCFLAG.COHERENT_INTEGRATION
1305
1384
1306 if self.dataOut.type == "Spectra":
1385 if self.dataOut.type == "Spectra":
1307 if self.dataOut.nIncohInt > 1:
1386 if self.dataOut.nIncohInt > 1:
1308 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1387 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1309
1388
1310 if self.dataOut.data_dc is not None:
1389 if self.dataOut.data_dc is not None:
1311 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1390 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1312
1391
1313 if self.dataOut.flagShiftFFT:
1392 if self.dataOut.flagShiftFFT:
1314 processFlags += PROCFLAG.SHIFT_FFT_DATA
1393 processFlags += PROCFLAG.SHIFT_FFT_DATA
1315
1394
1316 return processFlags
1395 return processFlags
1317
1396
1318 def setBasicHeader(self):
1397 def setBasicHeader(self):
1319
1398
1320 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1399 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1321 self.basicHeaderObj.version = self.versionFile
1400 self.basicHeaderObj.version = self.versionFile
1322 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1401 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1323
1402
1324 utc = numpy.floor(self.dataOut.utctime)
1403 utc = numpy.floor(self.dataOut.utctime)
1325 milisecond = (self.dataOut.utctime - utc)* 1000.0
1404 milisecond = (self.dataOut.utctime - utc)* 1000.0
1326
1405
1327 self.basicHeaderObj.utc = utc
1406 self.basicHeaderObj.utc = utc
1328 self.basicHeaderObj.miliSecond = milisecond
1407 self.basicHeaderObj.miliSecond = milisecond
1329 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1408 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1330 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1409 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1331 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1410 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1332
1411
1333 def setFirstHeader(self):
1412 def setFirstHeader(self):
1334 """
1413 """
1335 Obtiene una copia del First Header
1414 Obtiene una copia del First Header
1336
1415
1337 Affected:
1416 Affected:
1338
1417
1339 self.basicHeaderObj
1418 self.basicHeaderObj
1340 self.systemHeaderObj
1419 self.systemHeaderObj
1341 self.radarControllerHeaderObj
1420 self.radarControllerHeaderObj
1342 self.processingHeaderObj self.
1421 self.processingHeaderObj self.
1343
1422
1344 Return:
1423 Return:
1345 None
1424 None
1346 """
1425 """
1347
1426
1348 raise ValueError, "No implemented"
1427 raise ValueError, "No implemented"
1349
1428
1350 def __writeFirstHeader(self):
1429 def __writeFirstHeader(self):
1351 """
1430 """
1352 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1431 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1353
1432
1354 Affected:
1433 Affected:
1355 __dataType
1434 __dataType
1356
1435
1357 Return:
1436 Return:
1358 None
1437 None
1359 """
1438 """
1360
1439
1361 # CALCULAR PARAMETROS
1440 # CALCULAR PARAMETROS
1362
1441
1363 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1442 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1364 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1443 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1365
1444
1366 self.basicHeaderObj.write(self.fp)
1445 self.basicHeaderObj.write(self.fp)
1367 self.systemHeaderObj.write(self.fp)
1446 self.systemHeaderObj.write(self.fp)
1368 self.radarControllerHeaderObj.write(self.fp)
1447 self.radarControllerHeaderObj.write(self.fp)
1369 self.processingHeaderObj.write(self.fp)
1448 self.processingHeaderObj.write(self.fp)
1370
1449
1371 def __setNewBlock(self):
1450 def __setNewBlock(self):
1372 """
1451 """
1373 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1452 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1374
1453
1375 Return:
1454 Return:
1376 0 : si no pudo escribir nada
1455 0 : si no pudo escribir nada
1377 1 : Si escribio el Basic el First Header
1456 1 : Si escribio el Basic el First Header
1378 """
1457 """
1379 if self.fp == None:
1458 if self.fp == None:
1380 self.setNextFile()
1459 self.setNextFile()
1381
1460
1382 if self.flagIsNewFile:
1461 if self.flagIsNewFile:
1383 return 1
1462 return 1
1384
1463
1385 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1464 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1386 self.basicHeaderObj.write(self.fp)
1465 self.basicHeaderObj.write(self.fp)
1387 return 1
1466 return 1
1388
1467
1389 if not( self.setNextFile() ):
1468 if not( self.setNextFile() ):
1390 return 0
1469 return 0
1391
1470
1392 return 1
1471 return 1
1393
1472
1394
1473
1395 def writeNextBlock(self):
1474 def writeNextBlock(self):
1396 """
1475 """
1397 Selecciona el bloque siguiente de datos y los escribe en un file
1476 Selecciona el bloque siguiente de datos y los escribe en un file
1398
1477
1399 Return:
1478 Return:
1400 0 : Si no hizo pudo escribir el bloque de datos
1479 0 : Si no hizo pudo escribir el bloque de datos
1401 1 : Si no pudo escribir el bloque de datos
1480 1 : Si no pudo escribir el bloque de datos
1402 """
1481 """
1403 if not( self.__setNewBlock() ):
1482 if not( self.__setNewBlock() ):
1404 return 0
1483 return 0
1405
1484
1406 self.writeBlock()
1485 self.writeBlock()
1407
1486
1408 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1487 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1409 self.processingHeaderObj.dataBlocksPerFile)
1488 self.processingHeaderObj.dataBlocksPerFile)
1410
1489
1411 return 1
1490 return 1
1412
1491
1413 def setNextFile(self):
1492 def setNextFile(self):
1414 """
1493 """
1415 Determina el siguiente file que sera escrito
1494 Determina el siguiente file que sera escrito
1416
1495
1417 Affected:
1496 Affected:
1418 self.filename
1497 self.filename
1419 self.subfolder
1498 self.subfolder
1420 self.fp
1499 self.fp
1421 self.setFile
1500 self.setFile
1422 self.flagIsNewFile
1501 self.flagIsNewFile
1423
1502
1424 Return:
1503 Return:
1425 0 : Si el archivo no puede ser escrito
1504 0 : Si el archivo no puede ser escrito
1426 1 : Si el archivo esta listo para ser escrito
1505 1 : Si el archivo esta listo para ser escrito
1427 """
1506 """
1428 ext = self.ext
1507 ext = self.ext
1429 path = self.path
1508 path = self.path
1430
1509
1431 if self.fp != None:
1510 if self.fp != None:
1432 self.fp.close()
1511 self.fp.close()
1433
1512
1434 timeTuple = time.localtime( self.dataOut.utctime)
1513 timeTuple = time.localtime( self.dataOut.utctime)
1435 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1514 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1436
1515
1437 fullpath = os.path.join( path, subfolder )
1516 fullpath = os.path.join( path, subfolder )
1438 setFile = self.setFile
1517 setFile = self.setFile
1439
1518
1440 if not( os.path.exists(fullpath) ):
1519 if not( os.path.exists(fullpath) ):
1441 os.mkdir(fullpath)
1520 os.mkdir(fullpath)
1442 setFile = -1 #inicializo mi contador de seteo
1521 setFile = -1 #inicializo mi contador de seteo
1443 else:
1522 else:
1444 filesList = os.listdir( fullpath )
1523 filesList = os.listdir( fullpath )
1445 if len( filesList ) > 0:
1524 if len( filesList ) > 0:
1446 filesList = sorted( filesList, key=str.lower )
1525 filesList = sorted( filesList, key=str.lower )
1447 filen = filesList[-1]
1526 filen = filesList[-1]
1448 # el filename debera tener el siguiente formato
1527 # el filename debera tener el siguiente formato
1449 # 0 1234 567 89A BCDE (hex)
1528 # 0 1234 567 89A BCDE (hex)
1450 # x YYYY DDD SSS .ext
1529 # x YYYY DDD SSS .ext
1451 if isNumber( filen[8:11] ):
1530 if isNumber( filen[8:11] ):
1452 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1531 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1453 else:
1532 else:
1454 setFile = -1
1533 setFile = -1
1455 else:
1534 else:
1456 setFile = -1 #inicializo mi contador de seteo
1535 setFile = -1 #inicializo mi contador de seteo
1457
1536
1458 setFile += 1
1537 setFile += 1
1459
1538
1460 #If this is a new day it resets some values
1539 #If this is a new day it resets some values
1461 if self.dataOut.datatime.date() > self.fileDate:
1540 if self.dataOut.datatime.date() > self.fileDate:
1462 setFile = 0
1541 setFile = 0
1463 self.nTotalBlocks = 0
1542 self.nTotalBlocks = 0
1464
1543
1465 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1544 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1466
1545
1467 filename = os.path.join( path, subfolder, filen )
1546 filename = os.path.join( path, subfolder, filen )
1468
1547
1469 fp = open( filename,'wb' )
1548 fp = open( filename,'wb' )
1470
1549
1471 self.blockIndex = 0
1550 self.blockIndex = 0
1472
1551
1473 #guardando atributos
1552 #guardando atributos
1474 self.filename = filename
1553 self.filename = filename
1475 self.subfolder = subfolder
1554 self.subfolder = subfolder
1476 self.fp = fp
1555 self.fp = fp
1477 self.setFile = setFile
1556 self.setFile = setFile
1478 self.flagIsNewFile = 1
1557 self.flagIsNewFile = 1
1479 self.fileDate = self.dataOut.datatime.date()
1558 self.fileDate = self.dataOut.datatime.date()
1480
1559
1481 self.setFirstHeader()
1560 self.setFirstHeader()
1482
1561
1483 print '[Writing] Opening file: %s'%self.filename
1562 print '[Writing] Opening file: %s'%self.filename
1484
1563
1485 self.__writeFirstHeader()
1564 self.__writeFirstHeader()
1486
1565
1487 return 1
1566 return 1
1488
1567
1489 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1568 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1490 """
1569 """
1491 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1570 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1492
1571
1493 Inputs:
1572 Inputs:
1494 path : directory where data will be saved
1573 path : directory where data will be saved
1495 profilesPerBlock : number of profiles per block
1574 profilesPerBlock : number of profiles per block
1496 set : initial file set
1575 set : initial file set
1497 datatype : An integer number that defines data type:
1576 datatype : An integer number that defines data type:
1498 0 : int8 (1 byte)
1577 0 : int8 (1 byte)
1499 1 : int16 (2 bytes)
1578 1 : int16 (2 bytes)
1500 2 : int32 (4 bytes)
1579 2 : int32 (4 bytes)
1501 3 : int64 (8 bytes)
1580 3 : int64 (8 bytes)
1502 4 : float32 (4 bytes)
1581 4 : float32 (4 bytes)
1503 5 : double64 (8 bytes)
1582 5 : double64 (8 bytes)
1504
1583
1505 Return:
1584 Return:
1506 0 : Si no realizo un buen seteo
1585 0 : Si no realizo un buen seteo
1507 1 : Si realizo un buen seteo
1586 1 : Si realizo un buen seteo
1508 """
1587 """
1509
1588
1510 if ext == None:
1589 if ext == None:
1511 ext = self.ext
1590 ext = self.ext
1512
1591
1513 self.ext = ext.lower()
1592 self.ext = ext.lower()
1514
1593
1515 self.path = path
1594 self.path = path
1516
1595
1517 if set is None:
1596 if set is None:
1518 self.setFile = -1
1597 self.setFile = -1
1519 else:
1598 else:
1520 self.setFile = set - 1
1599 self.setFile = set - 1
1521
1600
1522 self.blocksPerFile = blocksPerFile
1601 self.blocksPerFile = blocksPerFile
1523
1602
1524 self.profilesPerBlock = profilesPerBlock
1603 self.profilesPerBlock = profilesPerBlock
1525
1604
1526 self.dataOut = dataOut
1605 self.dataOut = dataOut
1527 self.fileDate = self.dataOut.datatime.date()
1606 self.fileDate = self.dataOut.datatime.date()
1528 #By default
1607 #By default
1529 self.dtype = self.dataOut.dtype
1608 self.dtype = self.dataOut.dtype
1530
1609
1531 if datatype is not None:
1610 if datatype is not None:
1532 self.dtype = get_numpy_dtype(datatype)
1611 self.dtype = get_numpy_dtype(datatype)
1533
1612
1534 if not(self.setNextFile()):
1613 if not(self.setNextFile()):
1535 print "[Writing] There isn't a next file"
1614 print "[Writing] There isn't a next file"
1536 return 0
1615 return 0
1537
1616
1538 self.setBlockDimension()
1617 self.setBlockDimension()
1539
1618
1540 return 1
1619 return 1
1541
1620
1542 def run(self, dataOut, **kwargs):
1621 def run(self, dataOut, **kwargs):
1543
1622
1544 if not(self.isConfig):
1623 if not(self.isConfig):
1545
1624
1546 self.setup(dataOut, **kwargs)
1625 self.setup(dataOut, **kwargs)
1547 self.isConfig = True
1626 self.isConfig = True
1548
1627
1549 self.putData()
1628 self.putData()
1550
1629
General Comments 0
You need to be logged in to leave comments. Login now