##// END OF EJS Templates
schain
RSS
Bug fixed in jroIO_base: date and time filters were not working when walk=0 and there were many days inside one directory.
Miguel Valdez -
r640:4d492e909122
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1550 +1,1629
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 traceback.print_exc()
72 72 raise IOError, "The file %s can't be opened" %(filename)
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 def isFileinThisTime(filename, startTime, endTime):
86 def isFileInTimeRange(filename, startTime, endTime):
87 87 """
88 88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89 89
90 90 Inputs:
91 91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92 92
93 93 startTime : tiempo inicial del rango seleccionado en formato datetime.time
94 94
95 95 endTime : tiempo final del rango seleccionado en formato datetime.time
96 96
97 97 Return:
98 98 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
99 99 fecha especificado, de lo contrario retorna False.
100 100
101 101 Excepciones:
102 102 Si el archivo no existe o no puede ser abierto
103 103 Si la cabecera no puede ser leida.
104 104
105 105 """
106 106
107 107
108 108 try:
109 109 fp = open(filename,'rb')
110 110 except IOError:
111 111 traceback.print_exc()
112 112 raise IOError, "The file %s can't be opened" %(filename)
113 113
114 114 basicHeaderObj = BasicHeader(LOCALTIME)
115 115 sts = basicHeaderObj.read(fp)
116 116 fp.close()
117 117
118 118 thisDatetime = basicHeaderObj.datatime
119 119 thisTime = thisDatetime.time()
120 120
121 121 if not(sts):
122 122 print "Skipping the file %s because it has not a valid header" %(filename)
123 123 return None
124 124
125 125 if not ((startTime <= thisTime) and (endTime > thisTime)):
126 126 return None
127 127
128 128 return thisDatetime
129 129
130 def isFolderInDateRange(folder, startDate=None, endDate=None):
131 """
132 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
133
134 Inputs:
135 folder : nombre completo del directorio.
136 Su formato deberia ser "/path_root/?YYYYDDD"
137
138 siendo:
139 YYYY : Anio (ejemplo 2015)
140 DDD : Dia del anio (ejemplo 305)
141
142 startDate : fecha inicial del rango seleccionado en formato datetime.date
143
144 endDate : fecha final del rango seleccionado en formato datetime.date
145
146 Return:
147 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
148 fecha especificado, de lo contrario retorna False.
149 Excepciones:
150 Si el directorio no tiene el formato adecuado
151 """
152
153 basename = os.path.basename(folder)
154
155 if not isRadarFolder(basename):
156 raise IOError, "The folder %s has not the rigth format" %folder
157
158 if startDate and endDate:
159 thisDate = getDateFromRadarFolder(basename)
160
161 if thisDate < startDate:
162 return 0
163
164 if thisDate > endDate:
165 return 0
166
167 return 1
168
169 def isFileInDateRange(filename, startDate=None, endDate=None):
170 """
171 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
172
173 Inputs:
174 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
175
176 Su formato deberia ser "?YYYYDDDsss"
177
178 siendo:
179 YYYY : Anio (ejemplo 2015)
180 DDD : Dia del anio (ejemplo 305)
181 sss : set
182
183 startDate : fecha inicial del rango seleccionado en formato datetime.date
184
185 endDate : fecha final del rango seleccionado en formato datetime.date
186
187 Return:
188 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
189 fecha especificado, de lo contrario retorna False.
190 Excepciones:
191 Si el archivo no tiene el formato adecuado
192 """
193
194 basename = os.path.basename(filename)
195
196 if not isRadarFile(basename):
197 raise IOError, "The filename %s has not the rigth format" %filename
198
199 if startDate and endDate:
200 thisDate = getDateFromRadarFile(basename)
201
202 if thisDate < startDate:
203 return 0
204
205 if thisDate > endDate:
206 return 0
207
208 return 1
209
130 210 def getFileFromSet(path, ext, set):
131 211 validFilelist = []
132 212 fileList = os.listdir(path)
133 213
134 214 # 0 1234 567 89A BCDE
135 215 # H YYYY DDD SSS .ext
136 216
137 217 for thisFile in fileList:
138 218 try:
139 219 year = int(thisFile[1:5])
140 220 doy = int(thisFile[5:8])
141 221 except:
142 222 continue
143 223
144 224 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
145 225 continue
146 226
147 227 validFilelist.append(thisFile)
148 228
149 229 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
150 230
151 231 if len(myfile)!= 0:
152 232 return myfile[0]
153 233 else:
154 234 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
155 235 print 'the filename %s does not exist'%filename
156 236 print '...going to the last file: '
157 237
158 238 if validFilelist:
159 239 validFilelist = sorted( validFilelist, key=str.lower )
160 240 return validFilelist[-1]
161 241
162 242 return None
163 243
164 244 def getlastFileFromPath(path, ext):
165 245 """
166 246 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
167 247 al final de la depuracion devuelve el ultimo file de la lista que quedo.
168 248
169 249 Input:
170 250 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
171 251 ext : extension de los files contenidos en una carpeta
172 252
173 253 Return:
174 254 El ultimo file de una determinada carpeta, no se considera el path.
175 255 """
176 256 validFilelist = []
177 257 fileList = os.listdir(path)
178 258
179 259 # 0 1234 567 89A BCDE
180 260 # H YYYY DDD SSS .ext
181 261
182 262 for thisFile in fileList:
183 263
184 264 year = thisFile[1:5]
185 265 if not isNumber(year):
186 266 continue
187 267
188 268 doy = thisFile[5:8]
189 269 if not isNumber(doy):
190 270 continue
191 271
192 272 year = int(year)
193 273 doy = int(doy)
194 274
195 275 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
196 276 continue
197 277
198 278 validFilelist.append(thisFile)
199 279
200 280 if validFilelist:
201 281 validFilelist = sorted( validFilelist, key=str.lower )
202 282 return validFilelist[-1]
203 283
204 284 return None
205 285
206 286 def checkForRealPath(path, foldercounter, year, doy, set, ext):
207 287 """
208 288 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
209 289 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
210 290 el path exacto de un determinado file.
211 291
212 292 Example :
213 293 nombre correcto del file es .../.../D2009307/P2009307367.ext
214 294
215 295 Entonces la funcion prueba con las siguientes combinaciones
216 296 .../.../y2009307367.ext
217 297 .../.../Y2009307367.ext
218 298 .../.../x2009307/y2009307367.ext
219 299 .../.../x2009307/Y2009307367.ext
220 300 .../.../X2009307/y2009307367.ext
221 301 .../.../X2009307/Y2009307367.ext
222 302 siendo para este caso, la ultima combinacion de letras, identica al file buscado
223 303
224 304 Return:
225 305 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
226 306 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
227 307 para el filename
228 308 """
229 309 fullfilename = None
230 310 find_flag = False
231 311 filename = None
232 312
233 313 prefixDirList = [None,'d','D']
234 314 if ext.lower() == ".r": #voltage
235 315 prefixFileList = ['d','D']
236 316 elif ext.lower() == ".pdata": #spectra
237 317 prefixFileList = ['p','P']
238 318 else:
239 319 return None, filename
240 320
241 321 #barrido por las combinaciones posibles
242 322 for prefixDir in prefixDirList:
243 323 thispath = path
244 324 if prefixDir != None:
245 325 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
246 326 if foldercounter == 0:
247 327 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
248 328 else:
249 329 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
250 330 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
251 331 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
252 332 fullfilename = os.path.join( thispath, filename ) #formo el path completo
253 333
254 334 if os.path.exists( fullfilename ): #verifico que exista
255 335 find_flag = True
256 336 break
257 337 if find_flag:
258 338 break
259 339
260 340 if not(find_flag):
261 341 return None, filename
262 342
263 343 return fullfilename, filename
264 344
265 345 def isRadarFolder(folder):
266 346 try:
267 347 year = int(folder[1:5])
268 348 doy = int(folder[5:8])
269 349 except:
270 350 return 0
271 351
272 352 return 1
273 353
274 354 def isRadarFile(file):
275 355 try:
276 356 year = int(file[1:5])
277 357 doy = int(file[5:8])
278 358 set = int(file[8:11])
279 359 except:
280 360 return 0
281 361
282 362 return 1
283 363
284 364 def getDateFromRadarFile(file):
285 try:
286 year = int(file[1:5])
287 doy = int(file[5:8])
288 set = int(file[8:11])
289 except:
290 return None
291
292 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
293 return thisDate
294
365 try:
366 year = int(file[1:5])
367 doy = int(file[5:8])
368 set = int(file[8:11])
369 except:
370 return None
371
372 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
373 return thisDate
374
375 def getDateFromRadarFolder(folder):
376 try:
377 year = int(folder[1:5])
378 doy = int(folder[5:8])
379 except:
380 return None
381
382 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
383 return thisDate
384
295 385 class JRODataIO:
296 386
297 387 c = 3E8
298 388
299 389 isConfig = False
300 390
301 391 basicHeaderObj = None
302 392
303 393 systemHeaderObj = None
304 394
305 395 radarControllerHeaderObj = None
306 396
307 397 processingHeaderObj = None
308 398
309 399 online = 0
310 400
311 401 dtype = None
312 402
313 403 pathList = []
314 404
315 405 filenameList = []
316 406
317 407 filename = None
318 408
319 409 ext = None
320 410
321 411 flagIsNewFile = 1
322 412
323 413 flagDiscontinuousBlock = 0
324 414
325 415 flagIsNewBlock = 0
326 416
327 417 fp = None
328 418
329 419 firstHeaderSize = 0
330 420
331 421 basicHeaderSize = 24
332 422
333 423 versionFile = 1103
334 424
335 425 fileSize = None
336 426
337 427 # ippSeconds = None
338 428
339 429 fileSizeByHeader = None
340 430
341 431 fileIndex = None
342 432
343 433 profileIndex = None
344 434
345 435 blockIndex = None
346 436
347 437 nTotalBlocks = None
348 438
349 439 maxTimeStep = 30
350 440
351 441 lastUTTime = None
352 442
353 443 datablock = None
354 444
355 445 dataOut = None
356 446
357 447 blocksize = None
358 448
359 449 getByBlock = False
360 450
361 451 def __init__(self):
362 452
363 453 raise ValueError, "Not implemented"
364 454
365 455 def run(self):
366 456
367 457 raise ValueError, "Not implemented"
368 458
369 459 def getDtypeWidth(self):
370 460
371 461 dtype_index = get_dtype_index(self.dtype)
372 462 dtype_width = get_dtype_width(dtype_index)
373 463
374 464 return dtype_width
375 465
376 466 class JRODataReader(JRODataIO):
377 467
378 468 nReadBlocks = 0
379 469
380 470 delay = 10 #number of seconds waiting a new file
381 471
382 472 nTries = 3 #quantity tries
383 473
384 474 nFiles = 3 #number of files for searching
385 475
386 476 path = None
387 477
388 478 foldercounter = 0
389 479
390 480 flagNoMoreFiles = 0
391 481
392 482 datetimeList = []
393 483
394 484 __isFirstTimeOnline = 1
395 485
396 486 __printInfo = True
397 487
398 488 profileIndex = None
399 489
400 490 nTxs = 1
401 491
402 492 txIndex = None
403 493
404 494 def __init__(self):
405 495
406 496 """
407 497
408 498 """
409 499
410 500 # raise NotImplementedError, "This method has not been implemented"
411 501
412 502
413 503 def createObjByDefault(self):
414 504 """
415 505
416 506 """
417 507 raise NotImplementedError, "This method has not been implemented"
418 508
419 509 def getBlockDimension(self):
420 510
421 511 raise NotImplementedError, "No implemented"
422 512
423 513 def __searchFilesOffLine(self,
424 514 path,
425 515 startDate=None,
426 516 endDate=None,
427 517 startTime=datetime.time(0,0,0),
428 518 endTime=datetime.time(23,59,59),
429 519 set=None,
430 520 expLabel='',
431 521 ext='.r',
432 522 walk=True):
433 523
434 524 self.filenameList = []
435 525 self.datetimeList = []
436 526
437 527 pathList = []
438 528
439 if not walk:
440 #pathList.append(path)
441 multi_path = path.split(',')
442 for single_path in multi_path:
443
444 if not os.path.isdir(single_path):
445 continue
446
447 pathList.append(single_path)
448
449 else:
450 #dirList = []
451 multi_path = path.split(',')
452 for single_path in multi_path:
453
454 if not os.path.isdir(single_path):
455 continue
456
457 dirList = []
458 for thisPath in os.listdir(single_path):
459 if not os.path.isdir(os.path.join(single_path,thisPath)):
460 continue
461 if not isRadarFolder(thisPath):
462 continue
463
464 dirList.append(thisPath)
465
466 if not(dirList):
467 return None, None
468
469 if startDate and endDate:
470 thisDate = startDate
471
472 while(thisDate <= endDate):
473 year = thisDate.timetuple().tm_year
474 doy = thisDate.timetuple().tm_yday
475
476 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
477 if len(matchlist) == 0:
478 thisDate += datetime.timedelta(1)
479 continue
480 for match in matchlist:
481 pathList.append(os.path.join(single_path,match,expLabel))
482
483 thisDate += datetime.timedelta(1)
484 else:
485 for thiDir in dirList:
486 pathList.append(os.path.join(single_path,thiDir,expLabel))
487
488 if pathList == []:
529 # if not walk:
530 # #pathList.append(path)
531 # multi_path = path.split(',')
532 # for single_path in multi_path:
533 #
534 # if not os.path.isdir(single_path):
535 # continue
536 #
537 # pathList.append(single_path)
538 #
539 # else:
540 # #dirList = []
541 # multi_path = path.split(',')
542 # for single_path in multi_path:
543 #
544 # if not os.path.isdir(single_path):
545 # continue
546 #
547 # dirList = []
548 # for thisPath in os.listdir(single_path):
549 # if not os.path.isdir(os.path.join(single_path,thisPath)):
550 # continue
551 # if not isRadarFolder(thisPath):
552 # continue
553 #
554 # dirList.append(thisPath)
555 #
556 # if not(dirList):
557 # return None, None
558 #
559 # if startDate and endDate:
560 # thisDate = startDate
561 #
562 # while(thisDate <= endDate):
563 # year = thisDate.timetuple().tm_year
564 # doy = thisDate.timetuple().tm_yday
565 #
566 # matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
567 # if len(matchlist) == 0:
568 # thisDate += datetime.timedelta(1)
569 # continue
570 # for match in matchlist:
571 # pathList.append(os.path.join(single_path,match,expLabel))
572 #
573 # thisDate += datetime.timedelta(1)
574 # else:
575 # for thiDir in dirList:
576 # pathList.append(os.path.join(single_path,thiDir,expLabel))
577
578 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
579
580 if dateList == []:
489 581 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
490 582 return None, None
491 583
492 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
493
584 if len(dateList) > 1:
585 print "%d dates with data were found for the date range: %s - %s" %(len(dateList), startDate, endDate)
586 else:
587 print "data was found for the date %s" %(dateList[0])
588
494 589 filenameList = []
495 590 datetimeList = []
496 pathDict = {}
497 filenameList_to_sort = []
498
499 for i in range(len(pathList)):
500
501 thisPath = pathList[i]
502
503 fileList = glob.glob1(thisPath, "*%s" %ext)
504 if len(fileList) < 1:
505 continue
506 fileList.sort()
507 pathDict.setdefault(fileList[0])
508 pathDict[fileList[0]] = i
509 filenameList_to_sort.append(fileList[0])
510
511 filenameList_to_sort.sort()
512
513 for file in filenameList_to_sort:
514 thisPath = pathList[pathDict[file]]
591 # pathDict = {}
592 # filenameList_to_sort = []
593 #
594 # for i in range(len(pathList)):
595 #
596 # thisPath = pathList[i]
597 #
598 # fileList = glob.glob1(thisPath, "*%s" %ext)
599 # if len(fileList) < 1:
600 # continue
601 # fileList.sort()
602 # pathDict.setdefault(fileList[0])
603 # pathDict[fileList[0]] = i
604 # filenameList_to_sort.append(fileList[0])
605 #
606 # filenameList_to_sort.sort()
607
608 for thisPath in pathList:
609 # thisPath = pathList[pathDict[file]]
515 610
516 611 fileList = glob.glob1(thisPath, "*%s" %ext)
517 612 fileList.sort()
518 613
519 614 for file in fileList:
520 615
521 616 filename = os.path.join(thisPath,file)
522 thisDatetime = isFileinThisTime(filename, startTime, endTime)
617
618 if not isFileInDateRange(filename, startDate, endDate):
619 continue
620
621 thisDatetime = isFileInTimeRange(filename, startTime, endTime)
523 622
524 623 if not(thisDatetime):
525 624 continue
526 625
527 626 filenameList.append(filename)
528 627 datetimeList.append(thisDatetime)
529 628
530 629 if not(filenameList):
531 630 print "Any file was found for the time range %s - %s" %(startTime, endTime)
532 631 return None, None
533 632
534 633 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
535 634 print
536 635
537 636 for i in range(len(filenameList)):
538 637 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
539 638
540 639 self.filenameList = filenameList
541 640 self.datetimeList = datetimeList
542 641
543 642 return pathList, filenameList
544 643
545 644 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
546 645
547 646 """
548 647 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
549 648 devuelve el archivo encontrado ademas de otros datos.
550 649
551 650 Input:
552 651 path : carpeta donde estan contenidos los files que contiene data
553 652
554 653 expLabel : Nombre del subexperimento (subfolder)
555 654
556 655 ext : extension de los files
557 656
558 657 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
559 658
560 659 Return:
561 660 directory : eL directorio donde esta el file encontrado
562 661 filename : el ultimo file de una determinada carpeta
563 662 year : el anho
564 663 doy : el numero de dia del anho
565 664 set : el set del archivo
566 665
567 666
568 667 """
569 668 dirList = []
570 669
571 670 if not walk:
572 671 fullpath = path
573 672 foldercounter = 0
574 673 else:
575 674 #Filtra solo los directorios
576 675 for thisPath in os.listdir(path):
577 676 if not os.path.isdir(os.path.join(path,thisPath)):
578 677 continue
579 678 if not isRadarFolder(thisPath):
580 679 continue
581 680
582 681 dirList.append(thisPath)
583 682
584 683 if not(dirList):
585 684 return None, None, None, None, None, None
586 685
587 686 dirList = sorted( dirList, key=str.lower )
588 687
589 688 doypath = dirList[-1]
590 689 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
591 690 fullpath = os.path.join(path, doypath, expLabel)
592 691
593 692
594 693 print "%s folder was found: " %(fullpath )
595 694
596 695 if set == None:
597 696 filename = getlastFileFromPath(fullpath, ext)
598 697 else:
599 698 filename = getFileFromSet(fullpath, ext, set)
600 699
601 700 if not(filename):
602 701 return None, None, None, None, None, None
603 702
604 703 print "%s file was found" %(filename)
605 704
606 705 if not(self.__verifyFile(os.path.join(fullpath, filename))):
607 706 return None, None, None, None, None, None
608 707
609 708 year = int( filename[1:5] )
610 709 doy = int( filename[5:8] )
611 710 set = int( filename[8:11] )
612 711
613 712 return fullpath, foldercounter, filename, year, doy, set
614 713
615 714 def __setNextFileOffline(self):
616 715
617 716 idFile = self.fileIndex
618 717
619 718 while (True):
620 719 idFile += 1
621 720 if not(idFile < len(self.filenameList)):
622 721 self.flagNoMoreFiles = 1
623 722 # print "[Reading] No more Files"
624 723 return 0
625 724
626 725 filename = self.filenameList[idFile]
627 726
628 727 if not(self.__verifyFile(filename)):
629 728 continue
630 729
631 730 fileSize = os.path.getsize(filename)
632 731 fp = open(filename,'rb')
633 732 break
634 733
635 734 self.flagIsNewFile = 1
636 735 self.fileIndex = idFile
637 736 self.filename = filename
638 737 self.fileSize = fileSize
639 738 self.fp = fp
640 739
641 740 # print "[Reading] Setting the file: %s"%self.filename
642 741
643 742 return 1
644 743
645 744 def __setNextFileOnline(self):
646 745 """
647 746 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
648 747 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
649 748 siguientes.
650 749
651 750 Affected:
652 751 self.flagIsNewFile
653 752 self.filename
654 753 self.fileSize
655 754 self.fp
656 755 self.set
657 756 self.flagNoMoreFiles
658 757
659 758 Return:
660 759 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
661 760 1 : si el file fue abierto con exito y esta listo a ser leido
662 761
663 762 Excepciones:
664 763 Si un determinado file no puede ser abierto
665 764 """
666 765 nFiles = 0
667 766 fileOk_flag = False
668 767 firstTime_flag = True
669 768
670 769 self.set += 1
671 770
672 771 if self.set > 999:
673 772 self.set = 0
674 773 self.foldercounter += 1
675 774
676 775 #busca el 1er file disponible
677 776 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
678 777 if fullfilename:
679 778 if self.__verifyFile(fullfilename, False):
680 779 fileOk_flag = True
681 780
682 781 #si no encuentra un file entonces espera y vuelve a buscar
683 782 if not(fileOk_flag):
684 783 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
685 784
686 785 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
687 786 tries = self.nTries
688 787 else:
689 788 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
690 789
691 790 for nTries in range( tries ):
692 791 if firstTime_flag:
693 792 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
694 793 sleep( self.delay )
695 794 else:
696 795 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
697 796
698 797 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
699 798 if fullfilename:
700 799 if self.__verifyFile(fullfilename):
701 800 fileOk_flag = True
702 801 break
703 802
704 803 if fileOk_flag:
705 804 break
706 805
707 806 firstTime_flag = False
708 807
709 808 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
710 809 self.set += 1
711 810
712 811 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
713 812 self.set = 0
714 813 self.doy += 1
715 814 self.foldercounter = 0
716 815
717 816 if fileOk_flag:
718 817 self.fileSize = os.path.getsize( fullfilename )
719 818 self.filename = fullfilename
720 819 self.flagIsNewFile = 1
721 820 if self.fp != None: self.fp.close()
722 821 self.fp = open(fullfilename, 'rb')
723 822 self.flagNoMoreFiles = 0
724 823 # print '[Reading] Setting the file: %s' % fullfilename
725 824 else:
726 825 self.fileSize = 0
727 826 self.filename = None
728 827 self.flagIsNewFile = 0
729 828 self.fp = None
730 829 self.flagNoMoreFiles = 1
731 830 # print '[Reading] No more files to read'
732 831
733 832 return fileOk_flag
734 833
735 834 def setNextFile(self):
736 835 if self.fp != None:
737 836 self.fp.close()
738 837
739 838 if self.online:
740 839 newFile = self.__setNextFileOnline()
741 840 else:
742 841 newFile = self.__setNextFileOffline()
743 842
744 843 if not(newFile):
745 844 print '[Reading] No more files to read'
746 845 return 0
747 846
748 847 print '[Reading] Setting the file: %s' % self.filename
749 848
750 849 self.__readFirstHeader()
751 850 self.nReadBlocks = 0
752 851 return 1
753 852
754 853 def __waitNewBlock(self):
755 854 """
756 855 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
757 856
758 857 Si el modo de lectura es OffLine siempre retorn 0
759 858 """
760 859 if not self.online:
761 860 return 0
762 861
763 862 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
764 863 return 0
765 864
766 865 currentPointer = self.fp.tell()
767 866
768 867 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
769 868
770 869 for nTries in range( self.nTries ):
771 870
772 871 self.fp.close()
773 872 self.fp = open( self.filename, 'rb' )
774 873 self.fp.seek( currentPointer )
775 874
776 875 self.fileSize = os.path.getsize( self.filename )
777 876 currentSize = self.fileSize - currentPointer
778 877
779 878 if ( currentSize >= neededSize ):
780 879 self.basicHeaderObj.read(self.fp)
781 880 return 1
782 881
783 882 if self.fileSize == self.fileSizeByHeader:
784 883 # self.flagEoF = True
785 884 return 0
786 885
787 886 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
788 887 sleep( self.delay )
789 888
790 889
791 890 return 0
792 891
793 892 def waitDataBlock(self,pointer_location):
794 893
795 894 currentPointer = pointer_location
796 895
797 896 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
798 897
799 898 for nTries in range( self.nTries ):
800 899 self.fp.close()
801 900 self.fp = open( self.filename, 'rb' )
802 901 self.fp.seek( currentPointer )
803 902
804 903 self.fileSize = os.path.getsize( self.filename )
805 904 currentSize = self.fileSize - currentPointer
806 905
807 906 if ( currentSize >= neededSize ):
808 907 return 1
809 908
810 909 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
811 910 sleep( self.delay )
812 911
813 912 return 0
814 913
815 914 def __jumpToLastBlock(self):
816 915
817 916 if not(self.__isFirstTimeOnline):
818 917 return
819 918
820 919 csize = self.fileSize - self.fp.tell()
821 920 blocksize = self.processingHeaderObj.blockSize
822 921
823 922 #salta el primer bloque de datos
824 923 if csize > self.processingHeaderObj.blockSize:
825 924 self.fp.seek(self.fp.tell() + blocksize)
826 925 else:
827 926 return
828 927
829 928 csize = self.fileSize - self.fp.tell()
830 929 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
831 930 while True:
832 931
833 932 if self.fp.tell()<self.fileSize:
834 933 self.fp.seek(self.fp.tell() + neededsize)
835 934 else:
836 935 self.fp.seek(self.fp.tell() - neededsize)
837 936 break
838 937
839 938 # csize = self.fileSize - self.fp.tell()
840 939 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
841 940 # factor = int(csize/neededsize)
842 941 # if factor > 0:
843 942 # self.fp.seek(self.fp.tell() + factor*neededsize)
844 943
845 944 self.flagIsNewFile = 0
846 945 self.__isFirstTimeOnline = 0
847 946
848 947 def __setNewBlock(self):
849 948
850 949 if self.fp == None:
851 950 return 0
852 951
853 952 if self.online:
854 953 self.__jumpToLastBlock()
855 954
856 955 if self.flagIsNewFile:
857 956 return 1
858 957
859 958 self.lastUTTime = self.basicHeaderObj.utc
860 959 currentSize = self.fileSize - self.fp.tell()
861 960 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
862 961
863 962 if (currentSize >= neededSize):
864 963 self.basicHeaderObj.read(self.fp)
865 964 return 1
866 965
867 966 if self.__waitNewBlock():
868 967 return 1
869 968
870 969 if not(self.setNextFile()):
871 970 return 0
872 971
873 972 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
874 973
875 974 self.flagDiscontinuousBlock = 0
876 975
877 976 if deltaTime > self.maxTimeStep:
878 977 self.flagDiscontinuousBlock = 1
879 978
880 979 return 1
881 980
882 981 def readNextBlock(self):
883 982
884 983 if not(self.__setNewBlock()):
885 984 return 0
886 985
887 986 if not(self.readBlock()):
888 987 return 0
889 988
890 989 self.getBasicHeader()
891 990
892 991 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
893 992 self.processingHeaderObj.dataBlocksPerFile,
894 993 self.dataOut.datatime.ctime())
895 994 return 1
896 995
897 996 def __readFirstHeader(self):
898 997
899 998 self.basicHeaderObj.read(self.fp)
900 999 self.systemHeaderObj.read(self.fp)
901 1000 self.radarControllerHeaderObj.read(self.fp)
902 1001 self.processingHeaderObj.read(self.fp)
903 1002
904 1003 self.firstHeaderSize = self.basicHeaderObj.size
905 1004
906 1005 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
907 1006 if datatype == 0:
908 1007 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
909 1008 elif datatype == 1:
910 1009 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
911 1010 elif datatype == 2:
912 1011 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
913 1012 elif datatype == 3:
914 1013 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
915 1014 elif datatype == 4:
916 1015 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
917 1016 elif datatype == 5:
918 1017 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
919 1018 else:
920 1019 raise ValueError, 'Data type was not defined'
921 1020
922 1021 self.dtype = datatype_str
923 1022 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
924 1023 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
925 1024 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
926 1025 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
927 1026 self.getBlockDimension()
928 1027
929 1028 def __verifyFile(self, filename, msgFlag=True):
930 1029 msg = None
931 1030 try:
932 1031 fp = open(filename, 'rb')
933 1032 currentPosition = fp.tell()
934 1033 except IOError:
935 1034 traceback.print_exc()
936 1035 if msgFlag:
937 1036 print "[Reading] The file %s can't be opened" % (filename)
938 1037 return False
939 1038
940 1039 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
941 1040
942 1041 if neededSize == 0:
943 1042 basicHeaderObj = BasicHeader(LOCALTIME)
944 1043 systemHeaderObj = SystemHeader()
945 1044 radarControllerHeaderObj = RadarControllerHeader()
946 1045 processingHeaderObj = ProcessingHeader()
947 1046
948 1047 try:
949 1048 if not( basicHeaderObj.read(fp) ): raise IOError
950 1049 if not( systemHeaderObj.read(fp) ): raise IOError
951 1050 if not( radarControllerHeaderObj.read(fp) ): raise IOError
952 1051 if not( processingHeaderObj.read(fp) ): raise IOError
953 1052 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
954 1053
955 1054 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
956 1055
957 1056 except IOError:
958 1057 traceback.print_exc()
959 1058 sys.exit(0)
960 1059
961 1060 if msgFlag:
962 1061 print "[Reading] The file %s is empty or it hasn't enough data" % filename
963 1062
964 1063 fp.close()
965 1064 return False
966 1065 else:
967 1066 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
968 1067
969 1068 fp.close()
970 1069 fileSize = os.path.getsize(filename)
971 1070 currentSize = fileSize - currentPosition
972 1071 if currentSize < neededSize:
973 1072 if msgFlag and (msg != None):
974 1073 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
975 1074 return False
976 1075
977 1076 return True
978 1077
979 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True):
1078 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
980 1079
981 1080 dateList = []
982 1081 pathList = []
983 1082
1083 multi_path = path.split(',')
1084
984 1085 if not walk:
985 #pathList.append(path)
986 multi_path = path.split(',')
1086
987 1087 for single_path in multi_path:
988 1088
989 1089 if not os.path.isdir(single_path):
990 1090 continue
991 1091
992 ok = False
993 1092 fileList = glob.glob1(single_path, "*"+ext)
994 1093
995 1094 for thisFile in fileList:
996 1095
997 1096 if not os.path.isfile(os.path.join(single_path, thisFile)):
998 1097 continue
999 1098
1000 1099 if not isRadarFile(thisFile):
1001 1100 continue
1002 1101
1003 ok = True
1102 if not isFileInDateRange(thisFile, startDate, endDate):
1103 continue
1104
1004 1105 thisDate = getDateFromRadarFile(thisFile)
1005 1106
1006 if thisDate not in dateList:
1007 dateList.append(thisDate)
1008
1009 if ok:
1107 if thisDate in dateList:
1108 continue
1109
1110 dateList.append(thisDate)
1010 1111 pathList.append(single_path)
1011
1012 return dateList
1013 1112
1014 multi_path = path.split(',')
1015 for single_path in multi_path:
1016
1017 if not os.path.isdir(single_path):
1018 continue
1019
1020 dirList = []
1021
1022 for thisPath in os.listdir(single_path):
1113 else:
1114 for single_path in multi_path:
1023 1115
1024 if not os.path.isdir(os.path.join(single_path,thisPath)):
1116 if not os.path.isdir(single_path):
1025 1117 continue
1118
1119 dirList = []
1026 1120
1027 if not isRadarFolder(thisPath):
1028 continue
1029
1030 dirList.append(thisPath)
1031
1032 if not dirList:
1033 return dateList
1034
1035 if startDate and endDate:
1036 thisDate = startDate
1037
1038 while(thisDate <= endDate):
1039 year = thisDate.timetuple().tm_year
1040 doy = thisDate.timetuple().tm_yday
1121 for thisPath in os.listdir(single_path):
1041 1122
1042 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
1043 if len(matchlist) == 0:
1044 thisDate += datetime.timedelta(1)
1123 if not os.path.isdir(os.path.join(single_path,thisPath)):
1045 1124 continue
1046 1125
1047 for match in matchlist:
1048
1049 datapath = os.path.join(single_path, match, expLabel)
1050 fileList = glob.glob1(datapath, "*"+ext)
1051
1052 if len(fileList) < 1:
1053 continue
1054
1055 pathList.append(datapath)
1056 dateList.append(thisDate)
1126 if not isRadarFolder(thisPath):
1127 continue
1057 1128
1058 thisDate += datetime.timedelta(1)
1059 else:
1129 if not isFolderInDateRange(thisPath, startDate, endDate):
1130 continue
1131
1132 dirList.append(thisPath)
1133
1134 if not dirList:
1135 continue
1136
1060 1137 for thisDir in dirList:
1061 1138
1062 1139 datapath = os.path.join(single_path, thisDir, expLabel)
1063 1140 fileList = glob.glob1(datapath, "*"+ext)
1064 1141
1065 1142 if len(fileList) < 1:
1066 1143 continue
1067 1144
1068 year = int(thisDir[1:5])
1069 doy = int(thisDir[5:8])
1070 thisDate = datetime.date(year,1,1) + datetime.timedelta(doy-1)
1145 thisDate = getDateFromRadarFolder(thisDir)
1071 1146
1072 1147 pathList.append(datapath)
1073 1148 dateList.append(thisDate)
1074
1075 return dateList
1076 1149
1150 dateList.sort()
1151
1152 if include_path:
1153 return dateList, pathList
1154
1155 return dateList
1077 1156
1078 1157 def setup(self,
1079 1158 path=None,
1080 1159 startDate=None,
1081 1160 endDate=None,
1082 1161 startTime=datetime.time(0,0,0),
1083 1162 endTime=datetime.time(23,59,59),
1084 1163 set=None,
1085 1164 expLabel = "",
1086 1165 ext = None,
1087 1166 online = False,
1088 1167 delay = 60,
1089 1168 walk = True,
1090 1169 getblock = False,
1091 1170 nTxs = 1):
1092 1171
1093 1172 if path == None:
1094 1173 raise ValueError, "[Reading] The path is not valid"
1095 1174
1096 1175 if ext == None:
1097 1176 ext = self.ext
1098 1177
1099 1178 if online:
1100 1179 print "[Reading] Searching files in online mode..."
1101 1180
1102 1181 for nTries in range( self.nTries ):
1103 1182 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1104 1183
1105 1184 if fullpath:
1106 1185 break
1107 1186
1108 1187 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1109 1188 sleep( self.delay )
1110 1189
1111 1190 if not(fullpath):
1112 1191 print "[Reading] There 'isn't any valid file in %s" % path
1113 1192 return None
1114 1193
1115 1194 self.year = year
1116 1195 self.doy = doy
1117 1196 self.set = set - 1
1118 1197 self.path = path
1119 1198 self.foldercounter = foldercounter
1120 1199 last_set = None
1121 1200
1122 1201 else:
1123 1202 print "[Reading] Searching files in offline mode ..."
1124 1203 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1125 1204 startTime=startTime, endTime=endTime,
1126 1205 set=set, expLabel=expLabel, ext=ext,
1127 1206 walk=walk)
1128 1207
1129 1208 if not(pathList):
1130 1209 print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
1131 1210 datetime.datetime.combine(startDate,startTime).ctime(),
1132 1211 datetime.datetime.combine(endDate,endTime).ctime())
1133 1212
1134 1213 sys.exit(-1)
1135 1214
1136 1215
1137 1216 self.fileIndex = -1
1138 1217 self.pathList = pathList
1139 1218 self.filenameList = filenameList
1140 1219 file_name = os.path.basename(filenameList[-1])
1141 1220 basename, ext = os.path.splitext(file_name)
1142 1221 last_set = int(basename[-3:])
1143 1222
1144 1223 self.online = online
1145 1224 self.delay = delay
1146 1225 ext = ext.lower()
1147 1226 self.ext = ext
1148 1227 self.getByBlock = getblock
1149 1228 self.nTxs = int(nTxs)
1150 1229
1151 1230 if not(self.setNextFile()):
1152 1231 if (startDate!=None) and (endDate!=None):
1153 1232 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1154 1233 elif startDate != None:
1155 1234 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1156 1235 else:
1157 1236 print "[Reading] No files"
1158 1237
1159 1238 sys.exit(-1)
1160 1239
1161 1240 # self.getBasicHeader()
1162 1241
1163 1242 if last_set != None:
1164 1243 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1165 1244 return
1166 1245
1167 1246 def getBasicHeader(self):
1168 1247
1169 1248 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1170 1249
1171 1250 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1172 1251
1173 1252 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1174 1253
1175 1254 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1176 1255
1177 1256 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1178 1257
1179 1258 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1180 1259
1181 1260 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1182 1261
1183 1262 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1184 1263
1185 1264
1186 1265 def getFirstHeader(self):
1187 1266
1188 1267 raise ValueError, "This method has not been implemented"
1189 1268
1190 1269 def getData(self):
1191 1270
1192 1271 raise ValueError, "This method has not been implemented"
1193 1272
1194 1273 def hasNotDataInBuffer(self):
1195 1274
1196 1275 raise ValueError, "This method has not been implemented"
1197 1276
1198 1277 def readBlock(self):
1199 1278
1200 1279 raise ValueError, "This method has not been implemented"
1201 1280
1202 1281 def isEndProcess(self):
1203 1282
1204 1283 return self.flagNoMoreFiles
1205 1284
1206 1285 def printReadBlocks(self):
1207 1286
1208 1287 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1209 1288
1210 1289 def printTotalBlocks(self):
1211 1290
1212 1291 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1213 1292
1214 1293 def printNumberOfBlock(self):
1215 1294
1216 1295 if self.flagIsNewBlock:
1217 1296 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1218 1297 self.processingHeaderObj.dataBlocksPerFile,
1219 1298 self.dataOut.datatime.ctime())
1220 1299
1221 1300 def printInfo(self):
1222 1301
1223 1302 if self.__printInfo == False:
1224 1303 return
1225 1304
1226 1305 self.basicHeaderObj.printInfo()
1227 1306 self.systemHeaderObj.printInfo()
1228 1307 self.radarControllerHeaderObj.printInfo()
1229 1308 self.processingHeaderObj.printInfo()
1230 1309
1231 1310 self.__printInfo = False
1232 1311
1233 1312
1234 1313 def run(self, **kwargs):
1235 1314
1236 1315 if not(self.isConfig):
1237 1316
1238 1317 # self.dataOut = dataOut
1239 1318 self.setup(**kwargs)
1240 1319 self.isConfig = True
1241 1320
1242 1321 self.getData()
1243 1322
1244 1323 class JRODataWriter(JRODataIO):
1245 1324
1246 1325 """
1247 1326 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1248 1327 de los datos siempre se realiza por bloques.
1249 1328 """
1250 1329
1251 1330 blockIndex = 0
1252 1331
1253 1332 path = None
1254 1333
1255 1334 setFile = None
1256 1335
1257 1336 profilesPerBlock = None
1258 1337
1259 1338 blocksPerFile = None
1260 1339
1261 1340 nWriteBlocks = 0
1262 1341
1263 1342 fileDate = None
1264 1343
1265 1344 def __init__(self, dataOut=None):
1266 1345 raise ValueError, "Not implemented"
1267 1346
1268 1347
1269 1348 def hasAllDataInBuffer(self):
1270 1349 raise ValueError, "Not implemented"
1271 1350
1272 1351
1273 1352 def setBlockDimension(self):
1274 1353 raise ValueError, "Not implemented"
1275 1354
1276 1355
1277 1356 def writeBlock(self):
1278 1357 raise ValueError, "No implemented"
1279 1358
1280 1359
1281 1360 def putData(self):
1282 1361 raise ValueError, "No implemented"
1283 1362
1284 1363
1285 1364 def getProcessFlags(self):
1286 1365
1287 1366 processFlags = 0
1288 1367
1289 1368 dtype_index = get_dtype_index(self.dtype)
1290 1369 procflag_dtype = get_procflag_dtype(dtype_index)
1291 1370
1292 1371 processFlags += procflag_dtype
1293 1372
1294 1373 if self.dataOut.flagDecodeData:
1295 1374 processFlags += PROCFLAG.DECODE_DATA
1296 1375
1297 1376 if self.dataOut.flagDeflipData:
1298 1377 processFlags += PROCFLAG.DEFLIP_DATA
1299 1378
1300 1379 if self.dataOut.code is not None:
1301 1380 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1302 1381
1303 1382 if self.dataOut.nCohInt > 1:
1304 1383 processFlags += PROCFLAG.COHERENT_INTEGRATION
1305 1384
1306 1385 if self.dataOut.type == "Spectra":
1307 1386 if self.dataOut.nIncohInt > 1:
1308 1387 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1309 1388
1310 1389 if self.dataOut.data_dc is not None:
1311 1390 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1312 1391
1313 1392 if self.dataOut.flagShiftFFT:
1314 1393 processFlags += PROCFLAG.SHIFT_FFT_DATA
1315 1394
1316 1395 return processFlags
1317 1396
1318 1397 def setBasicHeader(self):
1319 1398
1320 1399 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1321 1400 self.basicHeaderObj.version = self.versionFile
1322 1401 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1323 1402
1324 1403 utc = numpy.floor(self.dataOut.utctime)
1325 1404 milisecond = (self.dataOut.utctime - utc)* 1000.0
1326 1405
1327 1406 self.basicHeaderObj.utc = utc
1328 1407 self.basicHeaderObj.miliSecond = milisecond
1329 1408 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1330 1409 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1331 1410 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1332 1411
1333 1412 def setFirstHeader(self):
1334 1413 """
1335 1414 Obtiene una copia del First Header
1336 1415
1337 1416 Affected:
1338 1417
1339 1418 self.basicHeaderObj
1340 1419 self.systemHeaderObj
1341 1420 self.radarControllerHeaderObj
1342 1421 self.processingHeaderObj self.
1343 1422
1344 1423 Return:
1345 1424 None
1346 1425 """
1347 1426
1348 1427 raise ValueError, "No implemented"
1349 1428
1350 1429 def __writeFirstHeader(self):
1351 1430 """
1352 1431 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1353 1432
1354 1433 Affected:
1355 1434 __dataType
1356 1435
1357 1436 Return:
1358 1437 None
1359 1438 """
1360 1439
1361 1440 # CALCULAR PARAMETROS
1362 1441
1363 1442 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1364 1443 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1365 1444
1366 1445 self.basicHeaderObj.write(self.fp)
1367 1446 self.systemHeaderObj.write(self.fp)
1368 1447 self.radarControllerHeaderObj.write(self.fp)
1369 1448 self.processingHeaderObj.write(self.fp)
1370 1449
1371 1450 def __setNewBlock(self):
1372 1451 """
1373 1452 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1374 1453
1375 1454 Return:
1376 1455 0 : si no pudo escribir nada
1377 1456 1 : Si escribio el Basic el First Header
1378 1457 """
1379 1458 if self.fp == None:
1380 1459 self.setNextFile()
1381 1460
1382 1461 if self.flagIsNewFile:
1383 1462 return 1
1384 1463
1385 1464 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1386 1465 self.basicHeaderObj.write(self.fp)
1387 1466 return 1
1388 1467
1389 1468 if not( self.setNextFile() ):
1390 1469 return 0
1391 1470
1392 1471 return 1
1393 1472
1394 1473
1395 1474 def writeNextBlock(self):
1396 1475 """
1397 1476 Selecciona el bloque siguiente de datos y los escribe en un file
1398 1477
1399 1478 Return:
1400 1479 0 : Si no hizo pudo escribir el bloque de datos
1401 1480 1 : Si no pudo escribir el bloque de datos
1402 1481 """
1403 1482 if not( self.__setNewBlock() ):
1404 1483 return 0
1405 1484
1406 1485 self.writeBlock()
1407 1486
1408 1487 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1409 1488 self.processingHeaderObj.dataBlocksPerFile)
1410 1489
1411 1490 return 1
1412 1491
1413 1492 def setNextFile(self):
1414 1493 """
1415 1494 Determina el siguiente file que sera escrito
1416 1495
1417 1496 Affected:
1418 1497 self.filename
1419 1498 self.subfolder
1420 1499 self.fp
1421 1500 self.setFile
1422 1501 self.flagIsNewFile
1423 1502
1424 1503 Return:
1425 1504 0 : Si el archivo no puede ser escrito
1426 1505 1 : Si el archivo esta listo para ser escrito
1427 1506 """
1428 1507 ext = self.ext
1429 1508 path = self.path
1430 1509
1431 1510 if self.fp != None:
1432 1511 self.fp.close()
1433 1512
1434 1513 timeTuple = time.localtime( self.dataOut.utctime)
1435 1514 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1436 1515
1437 1516 fullpath = os.path.join( path, subfolder )
1438 1517 setFile = self.setFile
1439 1518
1440 1519 if not( os.path.exists(fullpath) ):
1441 1520 os.mkdir(fullpath)
1442 1521 setFile = -1 #inicializo mi contador de seteo
1443 1522 else:
1444 1523 filesList = os.listdir( fullpath )
1445 1524 if len( filesList ) > 0:
1446 1525 filesList = sorted( filesList, key=str.lower )
1447 1526 filen = filesList[-1]
1448 1527 # el filename debera tener el siguiente formato
1449 1528 # 0 1234 567 89A BCDE (hex)
1450 1529 # x YYYY DDD SSS .ext
1451 1530 if isNumber( filen[8:11] ):
1452 1531 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1453 1532 else:
1454 1533 setFile = -1
1455 1534 else:
1456 1535 setFile = -1 #inicializo mi contador de seteo
1457 1536
1458 1537 setFile += 1
1459 1538
1460 1539 #If this is a new day it resets some values
1461 1540 if self.dataOut.datatime.date() > self.fileDate:
1462 1541 setFile = 0
1463 1542 self.nTotalBlocks = 0
1464 1543
1465 1544 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1466 1545
1467 1546 filename = os.path.join( path, subfolder, filen )
1468 1547
1469 1548 fp = open( filename,'wb' )
1470 1549
1471 1550 self.blockIndex = 0
1472 1551
1473 1552 #guardando atributos
1474 1553 self.filename = filename
1475 1554 self.subfolder = subfolder
1476 1555 self.fp = fp
1477 1556 self.setFile = setFile
1478 1557 self.flagIsNewFile = 1
1479 1558 self.fileDate = self.dataOut.datatime.date()
1480 1559
1481 1560 self.setFirstHeader()
1482 1561
1483 1562 print '[Writing] Opening file: %s'%self.filename
1484 1563
1485 1564 self.__writeFirstHeader()
1486 1565
1487 1566 return 1
1488 1567
1489 1568 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1490 1569 """
1491 1570 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1492 1571
1493 1572 Inputs:
1494 1573 path : directory where data will be saved
1495 1574 profilesPerBlock : number of profiles per block
1496 1575 set : initial file set
1497 1576 datatype : An integer number that defines data type:
1498 1577 0 : int8 (1 byte)
1499 1578 1 : int16 (2 bytes)
1500 1579 2 : int32 (4 bytes)
1501 1580 3 : int64 (8 bytes)
1502 1581 4 : float32 (4 bytes)
1503 1582 5 : double64 (8 bytes)
1504 1583
1505 1584 Return:
1506 1585 0 : Si no realizo un buen seteo
1507 1586 1 : Si realizo un buen seteo
1508 1587 """
1509 1588
1510 1589 if ext == None:
1511 1590 ext = self.ext
1512 1591
1513 1592 self.ext = ext.lower()
1514 1593
1515 1594 self.path = path
1516 1595
1517 1596 if set is None:
1518 1597 self.setFile = -1
1519 1598 else:
1520 1599 self.setFile = set - 1
1521 1600
1522 1601 self.blocksPerFile = blocksPerFile
1523 1602
1524 1603 self.profilesPerBlock = profilesPerBlock
1525 1604
1526 1605 self.dataOut = dataOut
1527 1606 self.fileDate = self.dataOut.datatime.date()
1528 1607 #By default
1529 1608 self.dtype = self.dataOut.dtype
1530 1609
1531 1610 if datatype is not None:
1532 1611 self.dtype = get_numpy_dtype(datatype)
1533 1612
1534 1613 if not(self.setNextFile()):
1535 1614 print "[Writing] There isn't a next file"
1536 1615 return 0
1537 1616
1538 1617 self.setBlockDimension()
1539 1618
1540 1619 return 1
1541 1620
1542 1621 def run(self, dataOut, **kwargs):
1543 1622
1544 1623 if not(self.isConfig):
1545 1624
1546 1625 self.setup(dataOut, **kwargs)
1547 1626 self.isConfig = True
1548 1627
1549 1628 self.putData()
1550 1629
General Comments 0
You need to be logged in to leave comments. Login now