##// END OF EJS Templates
schain-jc
RSS
Data directories are sorted before reading files
Miguel Valdez -
r726:a766e7f85e72
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1639 +1,1646
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 print "The file %s can't be opened" %(filename)
72 72 return 0
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
87 87 """
88 88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89 89
90 90 Inputs:
91 91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92 92
93 93 startDate : fecha inicial del rango seleccionado en formato datetime.date
94 94
95 95 endDate : fecha final del rango seleccionado en formato datetime.date
96 96
97 97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
98 98
99 99 endTime : tiempo final del rango seleccionado en formato datetime.time
100 100
101 101 Return:
102 102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
103 103 fecha especificado, de lo contrario retorna False.
104 104
105 105 Excepciones:
106 106 Si el archivo no existe o no puede ser abierto
107 107 Si la cabecera no puede ser leida.
108 108
109 109 """
110 110
111 111
112 112 try:
113 113 fp = open(filename,'rb')
114 114 except IOError:
115 115 print "The file %s can't be opened" %(filename)
116 116 return None
117 117
118 118 basicHeaderObj = BasicHeader(LOCALTIME)
119 119 sts = basicHeaderObj.read(fp)
120 120 fp.close()
121 121
122 122 thisDatetime = basicHeaderObj.datatime
123 123 thisDate = thisDatetime.date()
124 124 thisTime = thisDatetime.time()
125 125
126 126 if not(sts):
127 127 print "Skipping the file %s because it has not a valid header" %(filename)
128 128 return None
129 129
130 130 #General case
131 131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
132 132 #-----------o----------------------------o-----------
133 133 # startTime endTime
134 134
135 135 if endTime >= startTime:
136 136 if (thisTime < startTime) or (thisTime > endTime):
137 137 return None
138 138
139 139 return thisDatetime
140 140
141 141 #If endTime < startTime then endTime belongs to the next day
142 142
143 143
144 144 #<<<<<<<<<<<o o>>>>>>>>>>>
145 145 #-----------o----------------------------o-----------
146 146 # endTime startTime
147 147
148 148 if (thisDate == startDate) and (thisTime < startTime):
149 149 return None
150 150
151 151 if (thisDate == endDate) and (thisTime > endTime):
152 152 return None
153 153
154 154 if (thisTime < startTime) and (thisTime > endTime):
155 155 return None
156 156
157 157 return thisDatetime
158 158
159 159 def isFolderInDateRange(folder, startDate=None, endDate=None):
160 160 """
161 161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
162 162
163 163 Inputs:
164 164 folder : nombre completo del directorio.
165 165 Su formato deberia ser "/path_root/?YYYYDDD"
166 166
167 167 siendo:
168 168 YYYY : Anio (ejemplo 2015)
169 169 DDD : Dia del anio (ejemplo 305)
170 170
171 171 startDate : fecha inicial del rango seleccionado en formato datetime.date
172 172
173 173 endDate : fecha final del rango seleccionado en formato datetime.date
174 174
175 175 Return:
176 176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
177 177 fecha especificado, de lo contrario retorna False.
178 178 Excepciones:
179 179 Si el directorio no tiene el formato adecuado
180 180 """
181 181
182 182 basename = os.path.basename(folder)
183 183
184 184 if not isRadarFolder(basename):
185 185 print "The folder %s has not the rigth format" %folder
186 186 return 0
187 187
188 188 if startDate and endDate:
189 189 thisDate = getDateFromRadarFolder(basename)
190 190
191 191 if thisDate < startDate:
192 192 return 0
193 193
194 194 if thisDate > endDate:
195 195 return 0
196 196
197 197 return 1
198 198
199 199 def isFileInDateRange(filename, startDate=None, endDate=None):
200 200 """
201 201 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
202 202
203 203 Inputs:
204 204 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
205 205
206 206 Su formato deberia ser "?YYYYDDDsss"
207 207
208 208 siendo:
209 209 YYYY : Anio (ejemplo 2015)
210 210 DDD : Dia del anio (ejemplo 305)
211 211 sss : set
212 212
213 213 startDate : fecha inicial del rango seleccionado en formato datetime.date
214 214
215 215 endDate : fecha final del rango seleccionado en formato datetime.date
216 216
217 217 Return:
218 218 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
219 219 fecha especificado, de lo contrario retorna False.
220 220 Excepciones:
221 221 Si el archivo no tiene el formato adecuado
222 222 """
223 223
224 224 basename = os.path.basename(filename)
225 225
226 226 if not isRadarFile(basename):
227 227 print "The filename %s has not the rigth format" %filename
228 228 return 0
229 229
230 230 if startDate and endDate:
231 231 thisDate = getDateFromRadarFile(basename)
232 232
233 233 if thisDate < startDate:
234 234 return 0
235 235
236 236 if thisDate > endDate:
237 237 return 0
238 238
239 239 return 1
240 240
241 241 def getFileFromSet(path, ext, set):
242 242 validFilelist = []
243 243 fileList = os.listdir(path)
244 244
245 245 # 0 1234 567 89A BCDE
246 246 # H YYYY DDD SSS .ext
247 247
248 248 for thisFile in fileList:
249 249 try:
250 250 year = int(thisFile[1:5])
251 251 doy = int(thisFile[5:8])
252 252 except:
253 253 continue
254 254
255 255 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
256 256 continue
257 257
258 258 validFilelist.append(thisFile)
259 259
260 260 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
261 261
262 262 if len(myfile)!= 0:
263 263 return myfile[0]
264 264 else:
265 265 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
266 266 print 'the filename %s does not exist'%filename
267 267 print '...going to the last file: '
268 268
269 269 if validFilelist:
270 270 validFilelist = sorted( validFilelist, key=str.lower )
271 271 return validFilelist[-1]
272 272
273 273 return None
274 274
275 275 def getlastFileFromPath(path, ext):
276 276 """
277 277 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
278 278 al final de la depuracion devuelve el ultimo file de la lista que quedo.
279 279
280 280 Input:
281 281 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
282 282 ext : extension de los files contenidos en una carpeta
283 283
284 284 Return:
285 285 El ultimo file de una determinada carpeta, no se considera el path.
286 286 """
287 287 validFilelist = []
288 288 fileList = os.listdir(path)
289 289
290 290 # 0 1234 567 89A BCDE
291 291 # H YYYY DDD SSS .ext
292 292
293 293 for thisFile in fileList:
294 294
295 295 year = thisFile[1:5]
296 296 if not isNumber(year):
297 297 continue
298 298
299 299 doy = thisFile[5:8]
300 300 if not isNumber(doy):
301 301 continue
302 302
303 303 year = int(year)
304 304 doy = int(doy)
305 305
306 306 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
307 307 continue
308 308
309 309 validFilelist.append(thisFile)
310 310
311 311 if validFilelist:
312 312 validFilelist = sorted( validFilelist, key=str.lower )
313 313 return validFilelist[-1]
314 314
315 315 return None
316 316
317 317 def checkForRealPath(path, foldercounter, year, doy, set, ext):
318 318 """
319 319 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
320 320 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
321 321 el path exacto de un determinado file.
322 322
323 323 Example :
324 324 nombre correcto del file es .../.../D2009307/P2009307367.ext
325 325
326 326 Entonces la funcion prueba con las siguientes combinaciones
327 327 .../.../y2009307367.ext
328 328 .../.../Y2009307367.ext
329 329 .../.../x2009307/y2009307367.ext
330 330 .../.../x2009307/Y2009307367.ext
331 331 .../.../X2009307/y2009307367.ext
332 332 .../.../X2009307/Y2009307367.ext
333 333 siendo para este caso, la ultima combinacion de letras, identica al file buscado
334 334
335 335 Return:
336 336 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
337 337 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
338 338 para el filename
339 339 """
340 340 fullfilename = None
341 341 find_flag = False
342 342 filename = None
343 343
344 344 prefixDirList = [None,'d','D']
345 345 if ext.lower() == ".r": #voltage
346 346 prefixFileList = ['d','D']
347 347 elif ext.lower() == ".pdata": #spectra
348 348 prefixFileList = ['p','P']
349 349 else:
350 350 return None, filename
351 351
352 352 #barrido por las combinaciones posibles
353 353 for prefixDir in prefixDirList:
354 354 thispath = path
355 355 if prefixDir != None:
356 356 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
357 357 if foldercounter == 0:
358 358 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
359 359 else:
360 360 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
361 361 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
362 362 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
363 363 fullfilename = os.path.join( thispath, filename ) #formo el path completo
364 364
365 365 if os.path.exists( fullfilename ): #verifico que exista
366 366 find_flag = True
367 367 break
368 368 if find_flag:
369 369 break
370 370
371 371 if not(find_flag):
372 372 return None, filename
373 373
374 374 return fullfilename, filename
375 375
376 376 def isRadarFolder(folder):
377 377 try:
378 378 year = int(folder[1:5])
379 379 doy = int(folder[5:8])
380 380 except:
381 381 return 0
382 382
383 383 return 1
384 384
385 385 def isRadarFile(file):
386 386 try:
387 387 year = int(file[1:5])
388 388 doy = int(file[5:8])
389 389 set = int(file[8:11])
390 390 except:
391 391 return 0
392 392
393 393 return 1
394 394
395 395 def getDateFromRadarFile(file):
396 396 try:
397 397 year = int(file[1:5])
398 398 doy = int(file[5:8])
399 399 set = int(file[8:11])
400 400 except:
401 401 return None
402 402
403 403 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
404 404 return thisDate
405 405
406 406 def getDateFromRadarFolder(folder):
407 407 try:
408 408 year = int(folder[1:5])
409 409 doy = int(folder[5:8])
410 410 except:
411 411 return None
412 412
413 413 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
414 414 return thisDate
415 415
416 416 class JRODataIO:
417 417
418 418 c = 3E8
419 419
420 420 isConfig = False
421 421
422 422 basicHeaderObj = None
423 423
424 424 systemHeaderObj = None
425 425
426 426 radarControllerHeaderObj = None
427 427
428 428 processingHeaderObj = None
429 429
430 430 dtype = None
431 431
432 432 pathList = []
433 433
434 434 filenameList = []
435 435
436 436 filename = None
437 437
438 438 ext = None
439 439
440 440 flagIsNewFile = 1
441 441
442 442 flagDiscontinuousBlock = 0
443 443
444 444 flagIsNewBlock = 0
445 445
446 446 fp = None
447 447
448 448 firstHeaderSize = 0
449 449
450 450 basicHeaderSize = 24
451 451
452 452 versionFile = 1103
453 453
454 454 fileSize = None
455 455
456 456 # ippSeconds = None
457 457
458 458 fileSizeByHeader = None
459 459
460 460 fileIndex = None
461 461
462 462 profileIndex = None
463 463
464 464 blockIndex = None
465 465
466 466 nTotalBlocks = None
467 467
468 468 maxTimeStep = 30
469 469
470 470 lastUTTime = None
471 471
472 472 datablock = None
473 473
474 474 dataOut = None
475 475
476 476 blocksize = None
477 477
478 478 getByBlock = False
479 479
480 480 def __init__(self):
481 481
482 482 raise NotImplementedError
483 483
484 484 def run(self):
485 485
486 486 raise NotImplementedError
487 487
488 488 def getDtypeWidth(self):
489 489
490 490 dtype_index = get_dtype_index(self.dtype)
491 491 dtype_width = get_dtype_width(dtype_index)
492 492
493 493 return dtype_width
494 494
495 495 class JRODataReader(JRODataIO):
496 496
497 497
498 498 online = 0
499 499
500 500 realtime = 0
501 501
502 502 nReadBlocks = 0
503 503
504 504 delay = 10 #number of seconds waiting a new file
505 505
506 506 nTries = 3 #quantity tries
507 507
508 508 nFiles = 3 #number of files for searching
509 509
510 510 path = None
511 511
512 512 foldercounter = 0
513 513
514 514 flagNoMoreFiles = 0
515 515
516 516 datetimeList = []
517 517
518 518 __isFirstTimeOnline = 1
519 519
520 520 __printInfo = True
521 521
522 522 profileIndex = None
523 523
524 524 nTxs = 1
525 525
526 526 txIndex = None
527 527
528 528 def __init__(self):
529 529
530 530 """
531 531 This class is used to find data files
532 532
533 533 Example:
534 534 reader = JRODataReader()
535 535 fileList = reader.findDataFiles()
536 536
537 537 """
538 538 pass
539 539
540 540
541 541 def createObjByDefault(self):
542 542 """
543 543
544 544 """
545 545 raise NotImplementedError
546 546
547 547 def getBlockDimension(self):
548 548
549 549 raise NotImplementedError
550 550
551 551 def __searchFilesOffLine(self,
552 552 path,
553 553 startDate=None,
554 554 endDate=None,
555 555 startTime=datetime.time(0,0,0),
556 556 endTime=datetime.time(23,59,59),
557 557 set=None,
558 558 expLabel='',
559 559 ext='.r',
560 560 walk=True):
561 561
562 562 self.filenameList = []
563 563 self.datetimeList = []
564 564
565 565 pathList = []
566 566
567 567 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
568 568
569 569 if dateList == []:
570 570 # print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
571 571 # datetime.datetime.combine(startDate,startTime).ctime(),
572 572 # datetime.datetime.combine(endDate,endTime).ctime())
573 573
574 574 return None, None
575 575
576 576 if len(dateList) > 1:
577 577 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
578 578 else:
579 579 print "[Reading] data was found for the date %s" %(dateList[0])
580 580
581 581 filenameList = []
582 582 datetimeList = []
583 583
584 584 for thisPath in pathList:
585 585 # thisPath = pathList[pathDict[file]]
586 586
587 587 fileList = glob.glob1(thisPath, "*%s" %ext)
588 588 fileList.sort()
589 589
590 590 for file in fileList:
591 591
592 592 filename = os.path.join(thisPath,file)
593 593
594 594 if not isFileInDateRange(filename, startDate, endDate):
595 595 continue
596 596
597 597 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
598 598
599 599 if not(thisDatetime):
600 600 continue
601 601
602 602 filenameList.append(filename)
603 603 datetimeList.append(thisDatetime)
604 604
605 605 if not(filenameList):
606 606 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
607 607 return None, None
608 608
609 609 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
610 610 print
611 611
612 612 for i in range(len(filenameList)):
613 613 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
614 614
615 615 self.filenameList = filenameList
616 616 self.datetimeList = datetimeList
617 617
618 618 return pathList, filenameList
619 619
620 620 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
621 621
622 622 """
623 623 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
624 624 devuelve el archivo encontrado ademas de otros datos.
625 625
626 626 Input:
627 627 path : carpeta donde estan contenidos los files que contiene data
628 628
629 629 expLabel : Nombre del subexperimento (subfolder)
630 630
631 631 ext : extension de los files
632 632
633 633 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
634 634
635 635 Return:
636 636 directory : eL directorio donde esta el file encontrado
637 637 filename : el ultimo file de una determinada carpeta
638 638 year : el anho
639 639 doy : el numero de dia del anho
640 640 set : el set del archivo
641 641
642 642
643 643 """
644 644 if not os.path.isdir(path):
645 645 return None, None, None, None, None, None
646 646
647 647 dirList = []
648 648
649 649 if not walk:
650 650 fullpath = path
651 651 foldercounter = 0
652 652 else:
653 653 #Filtra solo los directorios
654 654 for thisPath in os.listdir(path):
655 655 if not os.path.isdir(os.path.join(path,thisPath)):
656 656 continue
657 657 if not isRadarFolder(thisPath):
658 658 continue
659 659
660 660 dirList.append(thisPath)
661 661
662 662 if not(dirList):
663 663 return None, None, None, None, None, None
664 664
665 665 dirList = sorted( dirList, key=str.lower )
666 666
667 667 doypath = dirList[-1]
668 668 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
669 669 fullpath = os.path.join(path, doypath, expLabel)
670 670
671 671
672 672 print "[Reading] %s folder was found: " %(fullpath )
673 673
674 674 if set == None:
675 675 filename = getlastFileFromPath(fullpath, ext)
676 676 else:
677 677 filename = getFileFromSet(fullpath, ext, set)
678 678
679 679 if not(filename):
680 680 return None, None, None, None, None, None
681 681
682 682 print "[Reading] %s file was found" %(filename)
683 683
684 684 if not(self.__verifyFile(os.path.join(fullpath, filename))):
685 685 return None, None, None, None, None, None
686 686
687 687 year = int( filename[1:5] )
688 688 doy = int( filename[5:8] )
689 689 set = int( filename[8:11] )
690 690
691 691 return fullpath, foldercounter, filename, year, doy, set
692 692
693 693 def __setNextFileOffline(self):
694 694
695 695 idFile = self.fileIndex
696 696
697 697 while (True):
698 698 idFile += 1
699 699 if not(idFile < len(self.filenameList)):
700 700 self.flagNoMoreFiles = 1
701 701 # print "[Reading] No more Files"
702 702 return 0
703 703
704 704 filename = self.filenameList[idFile]
705 705
706 706 if not(self.__verifyFile(filename)):
707 707 continue
708 708
709 709 fileSize = os.path.getsize(filename)
710 710 fp = open(filename,'rb')
711 711 break
712 712
713 713 self.flagIsNewFile = 1
714 714 self.fileIndex = idFile
715 715 self.filename = filename
716 716 self.fileSize = fileSize
717 717 self.fp = fp
718 718
719 719 # print "[Reading] Setting the file: %s"%self.filename
720 720
721 721 return 1
722 722
723 723 def __setNextFileOnline(self):
724 724 """
725 725 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
726 726 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
727 727 siguientes.
728 728
729 729 Affected:
730 730 self.flagIsNewFile
731 731 self.filename
732 732 self.fileSize
733 733 self.fp
734 734 self.set
735 735 self.flagNoMoreFiles
736 736
737 737 Return:
738 738 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
739 739 1 : si el file fue abierto con exito y esta listo a ser leido
740 740
741 741 Excepciones:
742 742 Si un determinado file no puede ser abierto
743 743 """
744 744 nFiles = 0
745 745 fileOk_flag = False
746 746 firstTime_flag = True
747 747
748 748 self.set += 1
749 749
750 750 if self.set > 999:
751 751 self.set = 0
752 752 self.foldercounter += 1
753 753
754 754 #busca el 1er file disponible
755 755 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
756 756 if fullfilename:
757 757 if self.__verifyFile(fullfilename, False):
758 758 fileOk_flag = True
759 759
760 760 #si no encuentra un file entonces espera y vuelve a buscar
761 761 if not(fileOk_flag):
762 762 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
763 763
764 764 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
765 765 tries = self.nTries
766 766 else:
767 767 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
768 768
769 769 for nTries in range( tries ):
770 770 if firstTime_flag:
771 771 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
772 772 sleep( self.delay )
773 773 else:
774 774 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
775 775
776 776 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
777 777 if fullfilename:
778 778 if self.__verifyFile(fullfilename):
779 779 fileOk_flag = True
780 780 break
781 781
782 782 if fileOk_flag:
783 783 break
784 784
785 785 firstTime_flag = False
786 786
787 787 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
788 788 self.set += 1
789 789
790 790 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
791 791 self.set = 0
792 792 self.doy += 1
793 793 self.foldercounter = 0
794 794
795 795 if fileOk_flag:
796 796 self.fileSize = os.path.getsize( fullfilename )
797 797 self.filename = fullfilename
798 798 self.flagIsNewFile = 1
799 799 if self.fp != None: self.fp.close()
800 800 self.fp = open(fullfilename, 'rb')
801 801 self.flagNoMoreFiles = 0
802 802 # print '[Reading] Setting the file: %s' % fullfilename
803 803 else:
804 804 self.fileSize = 0
805 805 self.filename = None
806 806 self.flagIsNewFile = 0
807 807 self.fp = None
808 808 self.flagNoMoreFiles = 1
809 809 # print '[Reading] No more files to read'
810 810
811 811 return fileOk_flag
812 812
813 813 def setNextFile(self):
814 814 if self.fp != None:
815 815 self.fp.close()
816 816
817 817 if self.online:
818 818 newFile = self.__setNextFileOnline()
819 819 else:
820 820 newFile = self.__setNextFileOffline()
821 821
822 822 if not(newFile):
823 823 print '[Reading] No more files to read'
824 824 return 0
825 825
826 826 print '[Reading] Setting the file: %s' % self.filename
827 827
828 828 self.__readFirstHeader()
829 829 self.nReadBlocks = 0
830 830 return 1
831 831
832 832 def __waitNewBlock(self):
833 833 """
834 834 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
835 835
836 836 Si el modo de lectura es OffLine siempre retorn 0
837 837 """
838 838 if not self.online:
839 839 return 0
840 840
841 841 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
842 842 return 0
843 843
844 844 currentPointer = self.fp.tell()
845 845
846 846 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
847 847
848 848 for nTries in range( self.nTries ):
849 849
850 850 self.fp.close()
851 851 self.fp = open( self.filename, 'rb' )
852 852 self.fp.seek( currentPointer )
853 853
854 854 self.fileSize = os.path.getsize( self.filename )
855 855 currentSize = self.fileSize - currentPointer
856 856
857 857 if ( currentSize >= neededSize ):
858 858 self.basicHeaderObj.read(self.fp)
859 859 return 1
860 860
861 861 if self.fileSize == self.fileSizeByHeader:
862 862 # self.flagEoF = True
863 863 return 0
864 864
865 865 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
866 866 sleep( self.delay )
867 867
868 868
869 869 return 0
870 870
871 871 def waitDataBlock(self,pointer_location):
872 872
873 873 currentPointer = pointer_location
874 874
875 875 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
876 876
877 877 for nTries in range( self.nTries ):
878 878 self.fp.close()
879 879 self.fp = open( self.filename, 'rb' )
880 880 self.fp.seek( currentPointer )
881 881
882 882 self.fileSize = os.path.getsize( self.filename )
883 883 currentSize = self.fileSize - currentPointer
884 884
885 885 if ( currentSize >= neededSize ):
886 886 return 1
887 887
888 888 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
889 889 sleep( self.delay )
890 890
891 891 return 0
892 892
893 893 def __jumpToLastBlock(self):
894 894
895 895 if not(self.__isFirstTimeOnline):
896 896 return
897 897
898 898 csize = self.fileSize - self.fp.tell()
899 899 blocksize = self.processingHeaderObj.blockSize
900 900
901 901 #salta el primer bloque de datos
902 902 if csize > self.processingHeaderObj.blockSize:
903 903 self.fp.seek(self.fp.tell() + blocksize)
904 904 else:
905 905 return
906 906
907 907 csize = self.fileSize - self.fp.tell()
908 908 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
909 909 while True:
910 910
911 911 if self.fp.tell()<self.fileSize:
912 912 self.fp.seek(self.fp.tell() + neededsize)
913 913 else:
914 914 self.fp.seek(self.fp.tell() - neededsize)
915 915 break
916 916
917 917 # csize = self.fileSize - self.fp.tell()
918 918 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
919 919 # factor = int(csize/neededsize)
920 920 # if factor > 0:
921 921 # self.fp.seek(self.fp.tell() + factor*neededsize)
922 922
923 923 self.flagIsNewFile = 0
924 924 self.__isFirstTimeOnline = 0
925 925
926 926 def __setNewBlock(self):
927 927
928 928 if self.fp == None:
929 929 return 0
930 930
931 931 # if self.online:
932 932 # self.__jumpToLastBlock()
933 933
934 934 if self.flagIsNewFile:
935 935 self.lastUTTime = self.basicHeaderObj.utc
936 936 return 1
937 937
938 938 if self.realtime:
939 939 self.flagDiscontinuousBlock = 1
940 940 if not(self.setNextFile()):
941 941 return 0
942 942 else:
943 943 return 1
944 944
945 945 currentSize = self.fileSize - self.fp.tell()
946 946 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
947 947
948 948 if (currentSize >= neededSize):
949 949 self.basicHeaderObj.read(self.fp)
950 950 self.lastUTTime = self.basicHeaderObj.utc
951 951 return 1
952 952
953 953 if self.__waitNewBlock():
954 954 self.lastUTTime = self.basicHeaderObj.utc
955 955 return 1
956 956
957 957 if not(self.setNextFile()):
958 958 return 0
959 959
960 960 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
961 961 self.lastUTTime = self.basicHeaderObj.utc
962 962
963 963 self.flagDiscontinuousBlock = 0
964 964
965 965 if deltaTime > self.maxTimeStep:
966 966 self.flagDiscontinuousBlock = 1
967 967
968 968 return 1
969 969
970 970 def readNextBlock(self):
971 971
972 972 if not(self.__setNewBlock()):
973 973 return 0
974 974
975 975 if not(self.readBlock()):
976 976 return 0
977 977
978 978 self.getBasicHeader()
979 979
980 980 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
981 981 self.processingHeaderObj.dataBlocksPerFile,
982 982 self.dataOut.datatime.ctime())
983 983 return 1
984 984
985 985 def __readFirstHeader(self):
986 986
987 987 self.basicHeaderObj.read(self.fp)
988 988 self.systemHeaderObj.read(self.fp)
989 989 self.radarControllerHeaderObj.read(self.fp)
990 990 self.processingHeaderObj.read(self.fp)
991 991
992 992 self.firstHeaderSize = self.basicHeaderObj.size
993 993
994 994 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
995 995 if datatype == 0:
996 996 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
997 997 elif datatype == 1:
998 998 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
999 999 elif datatype == 2:
1000 1000 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1001 1001 elif datatype == 3:
1002 1002 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1003 1003 elif datatype == 4:
1004 1004 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1005 1005 elif datatype == 5:
1006 1006 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1007 1007 else:
1008 1008 raise ValueError, 'Data type was not defined'
1009 1009
1010 1010 self.dtype = datatype_str
1011 1011 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1012 1012 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1013 1013 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1014 1014 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1015 1015 self.getBlockDimension()
1016 1016
1017 1017 def __verifyFile(self, filename, msgFlag=True):
1018 1018
1019 1019 msg = None
1020 1020
1021 1021 try:
1022 1022 fp = open(filename, 'rb')
1023 1023 except IOError:
1024 1024
1025 1025 if msgFlag:
1026 1026 print "[Reading] File %s can't be opened" % (filename)
1027 1027
1028 1028 return False
1029 1029
1030 1030 currentPosition = fp.tell()
1031 1031 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1032 1032
1033 1033 if neededSize == 0:
1034 1034 basicHeaderObj = BasicHeader(LOCALTIME)
1035 1035 systemHeaderObj = SystemHeader()
1036 1036 radarControllerHeaderObj = RadarControllerHeader()
1037 1037 processingHeaderObj = ProcessingHeader()
1038 1038
1039 1039 if not( basicHeaderObj.read(fp) ):
1040 1040 fp.close()
1041 1041 return False
1042 1042
1043 1043 if not( systemHeaderObj.read(fp) ):
1044 1044 fp.close()
1045 1045 return False
1046 1046
1047 1047 if not( radarControllerHeaderObj.read(fp) ):
1048 1048 fp.close()
1049 1049 return False
1050 1050
1051 1051 if not( processingHeaderObj.read(fp) ):
1052 1052 fp.close()
1053 1053 return False
1054 1054
1055 1055 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1056 1056 else:
1057 1057 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1058 1058
1059 1059 fp.close()
1060 1060
1061 1061 fileSize = os.path.getsize(filename)
1062 1062 currentSize = fileSize - currentPosition
1063 1063
1064 1064 if currentSize < neededSize:
1065 1065 if msgFlag and (msg != None):
1066 1066 print msg
1067 1067 return False
1068 1068
1069 1069 return True
1070 1070
1071 1071 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1072 1072
1073 1073 dateList = []
1074 1074 pathList = []
1075 1075
1076 1076 multi_path = path.split(',')
1077 1077
1078 1078 if not walk:
1079 1079
1080 1080 for single_path in multi_path:
1081 1081
1082 1082 if not os.path.isdir(single_path):
1083 1083 continue
1084 1084
1085 1085 fileList = glob.glob1(single_path, "*"+ext)
1086 1086
1087 if not fileList:
1088 continue
1089
1090 fileList.sort()
1091
1087 1092 for thisFile in fileList:
1088 1093
1089 1094 if not os.path.isfile(os.path.join(single_path, thisFile)):
1090 1095 continue
1091 1096
1092 1097 if not isRadarFile(thisFile):
1093 1098 continue
1094 1099
1095 1100 if not isFileInDateRange(thisFile, startDate, endDate):
1096 1101 continue
1097 1102
1098 1103 thisDate = getDateFromRadarFile(thisFile)
1099 1104
1100 1105 if thisDate in dateList:
1101 1106 continue
1102 1107
1103 1108 dateList.append(thisDate)
1104 1109 pathList.append(single_path)
1105 1110
1106 1111 else:
1107 1112 for single_path in multi_path:
1108 1113
1109 1114 if not os.path.isdir(single_path):
1110 1115 continue
1111 1116
1112 1117 dirList = []
1113 1118
1114 1119 for thisPath in os.listdir(single_path):
1115 1120
1116 1121 if not os.path.isdir(os.path.join(single_path,thisPath)):
1117 1122 continue
1118 1123
1119 1124 if not isRadarFolder(thisPath):
1120 1125 continue
1121 1126
1122 1127 if not isFolderInDateRange(thisPath, startDate, endDate):
1123 1128 continue
1124 1129
1125 1130 dirList.append(thisPath)
1126 1131
1127 1132 if not dirList:
1128 1133 continue
1129 1134
1135 dirList.sort()
1136
1130 1137 for thisDir in dirList:
1131 1138
1132 1139 datapath = os.path.join(single_path, thisDir, expLabel)
1133 1140 fileList = glob.glob1(datapath, "*"+ext)
1134 1141
1135 1142 if len(fileList) < 1:
1136 1143 continue
1137 1144
1138 1145 thisDate = getDateFromRadarFolder(thisDir)
1139 1146
1140 1147 pathList.append(datapath)
1141 1148 dateList.append(thisDate)
1142 1149
1143 1150 dateList.sort()
1144 1151
1145 1152 if walk:
1146 1153 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1147 1154 else:
1148 1155 pattern_path = multi_path[0]
1149 1156
1150 1157 if not dateList:
1151 1158 print "[Reading] No *%s files in %s from %s to %s" %(ext, pattern_path, startDate, endDate)
1152 1159
1153 1160 if include_path:
1154 1161 return dateList, pathList
1155 1162
1156 1163 return dateList
1157 1164
1158 1165 def setup(self,
1159 1166 path=None,
1160 1167 startDate=None,
1161 1168 endDate=None,
1162 1169 startTime=datetime.time(0,0,0),
1163 1170 endTime=datetime.time(23,59,59),
1164 1171 set=None,
1165 1172 expLabel = "",
1166 1173 ext = None,
1167 1174 online = False,
1168 1175 delay = 60,
1169 1176 walk = True,
1170 1177 getblock = False,
1171 1178 nTxs = 1,
1172 1179 realtime=False):
1173 1180
1174 1181 if path == None:
1175 1182 raise ValueError, "[Reading] The path is not valid"
1176 1183
1177 1184 if ext == None:
1178 1185 ext = self.ext
1179 1186
1180 1187 if online:
1181 1188 print "[Reading] Searching files in online mode..."
1182 1189
1183 1190 for nTries in range( self.nTries ):
1184 1191 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1185 1192
1186 1193 if fullpath:
1187 1194 break
1188 1195
1189 1196 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1190 1197 sleep( self.delay )
1191 1198
1192 1199 if not(fullpath):
1193 1200 print "[Reading] There 'isn't any valid file in %s" % path
1194 1201 return
1195 1202
1196 1203 self.year = year
1197 1204 self.doy = doy
1198 1205 self.set = set - 1
1199 1206 self.path = path
1200 1207 self.foldercounter = foldercounter
1201 1208 last_set = None
1202 1209
1203 1210 else:
1204 1211 print "[Reading] Searching files in offline mode ..."
1205 1212 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1206 1213 startTime=startTime, endTime=endTime,
1207 1214 set=set, expLabel=expLabel, ext=ext,
1208 1215 walk=walk)
1209 1216
1210 1217 if not(pathList):
1211 1218 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1212 1219 # datetime.datetime.combine(startDate,startTime).ctime(),
1213 1220 # datetime.datetime.combine(endDate,endTime).ctime())
1214 1221
1215 1222 # sys.exit(-1)
1216 1223
1217 1224 self.fileIndex = -1
1218 1225 self.pathList = []
1219 1226 self.filenameList = []
1220 1227 return
1221 1228
1222 1229 self.fileIndex = -1
1223 1230 self.pathList = pathList
1224 1231 self.filenameList = filenameList
1225 1232 file_name = os.path.basename(filenameList[-1])
1226 1233 basename, ext = os.path.splitext(file_name)
1227 1234 last_set = int(basename[-3:])
1228 1235
1229 1236 self.online = online
1230 1237 self.realtime = realtime
1231 1238 self.delay = delay
1232 1239 ext = ext.lower()
1233 1240 self.ext = ext
1234 1241 self.getByBlock = getblock
1235 1242 self.nTxs = int(nTxs)
1236 1243
1237 1244 if not(self.setNextFile()):
1238 1245 if (startDate!=None) and (endDate!=None):
1239 1246 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1240 1247 elif startDate != None:
1241 1248 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1242 1249 else:
1243 1250 print "[Reading] No files"
1244 1251
1245 1252 self.fileIndex = -1
1246 1253 self.pathList = []
1247 1254 self.filenameList = []
1248 1255 return
1249 1256
1250 1257 # self.getBasicHeader()
1251 1258
1252 1259 if last_set != None:
1253 1260 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1254 1261 return
1255 1262
1256 1263 def getBasicHeader(self):
1257 1264
1258 1265 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1259 1266
1260 1267 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1261 1268
1262 1269 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1263 1270
1264 1271 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1265 1272
1266 1273 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1267 1274
1268 1275 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1269 1276
1270 1277 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1271 1278
1272 1279 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1273 1280
1274 1281
1275 1282 def getFirstHeader(self):
1276 1283
1277 1284 raise NotImplementedError
1278 1285
1279 1286 def getData(self):
1280 1287
1281 1288 raise NotImplementedError
1282 1289
1283 1290 def hasNotDataInBuffer(self):
1284 1291
1285 1292 raise NotImplementedError
1286 1293
1287 1294 def readBlock(self):
1288 1295
1289 1296 raise NotImplementedError
1290 1297
1291 1298 def isEndProcess(self):
1292 1299
1293 1300 return self.flagNoMoreFiles
1294 1301
1295 1302 def printReadBlocks(self):
1296 1303
1297 1304 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1298 1305
1299 1306 def printTotalBlocks(self):
1300 1307
1301 1308 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1302 1309
1303 1310 def printNumberOfBlock(self):
1304 1311
1305 1312 if self.flagIsNewBlock:
1306 1313 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1307 1314 self.processingHeaderObj.dataBlocksPerFile,
1308 1315 self.dataOut.datatime.ctime())
1309 1316
1310 1317 def printInfo(self):
1311 1318
1312 1319 if self.__printInfo == False:
1313 1320 return
1314 1321
1315 1322 self.basicHeaderObj.printInfo()
1316 1323 self.systemHeaderObj.printInfo()
1317 1324 self.radarControllerHeaderObj.printInfo()
1318 1325 self.processingHeaderObj.printInfo()
1319 1326
1320 1327 self.__printInfo = False
1321 1328
1322 1329
1323 1330 def run(self, **kwargs):
1324 1331
1325 1332 if not(self.isConfig):
1326 1333
1327 1334 # self.dataOut = dataOut
1328 1335 self.setup(**kwargs)
1329 1336 self.isConfig = True
1330 1337
1331 1338 self.getData()
1332 1339
1333 1340 class JRODataWriter(JRODataIO):
1334 1341
1335 1342 """
1336 1343 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1337 1344 de los datos siempre se realiza por bloques.
1338 1345 """
1339 1346
1340 1347 blockIndex = 0
1341 1348
1342 1349 path = None
1343 1350
1344 1351 setFile = None
1345 1352
1346 1353 profilesPerBlock = None
1347 1354
1348 1355 blocksPerFile = None
1349 1356
1350 1357 nWriteBlocks = 0
1351 1358
1352 1359 fileDate = None
1353 1360
1354 1361 def __init__(self, dataOut=None):
1355 1362 raise NotImplementedError
1356 1363
1357 1364
1358 1365 def hasAllDataInBuffer(self):
1359 1366 raise NotImplementedError
1360 1367
1361 1368
1362 1369 def setBlockDimension(self):
1363 1370 raise NotImplementedError
1364 1371
1365 1372
1366 1373 def writeBlock(self):
1367 1374 raise NotImplementedError
1368 1375
1369 1376
1370 1377 def putData(self):
1371 1378 raise NotImplementedError
1372 1379
1373 1380
1374 1381 def getProcessFlags(self):
1375 1382
1376 1383 processFlags = 0
1377 1384
1378 1385 dtype_index = get_dtype_index(self.dtype)
1379 1386 procflag_dtype = get_procflag_dtype(dtype_index)
1380 1387
1381 1388 processFlags += procflag_dtype
1382 1389
1383 1390 if self.dataOut.flagDecodeData:
1384 1391 processFlags += PROCFLAG.DECODE_DATA
1385 1392
1386 1393 if self.dataOut.flagDeflipData:
1387 1394 processFlags += PROCFLAG.DEFLIP_DATA
1388 1395
1389 1396 if self.dataOut.code is not None:
1390 1397 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1391 1398
1392 1399 if self.dataOut.nCohInt > 1:
1393 1400 processFlags += PROCFLAG.COHERENT_INTEGRATION
1394 1401
1395 1402 if self.dataOut.type == "Spectra":
1396 1403 if self.dataOut.nIncohInt > 1:
1397 1404 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1398 1405
1399 1406 if self.dataOut.data_dc is not None:
1400 1407 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1401 1408
1402 1409 if self.dataOut.flagShiftFFT:
1403 1410 processFlags += PROCFLAG.SHIFT_FFT_DATA
1404 1411
1405 1412 return processFlags
1406 1413
1407 1414 def setBasicHeader(self):
1408 1415
1409 1416 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1410 1417 self.basicHeaderObj.version = self.versionFile
1411 1418 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1412 1419
1413 1420 utc = numpy.floor(self.dataOut.utctime)
1414 1421 milisecond = (self.dataOut.utctime - utc)* 1000.0
1415 1422
1416 1423 self.basicHeaderObj.utc = utc
1417 1424 self.basicHeaderObj.miliSecond = milisecond
1418 1425 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1419 1426 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1420 1427 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1421 1428
1422 1429 def setFirstHeader(self):
1423 1430 """
1424 1431 Obtiene una copia del First Header
1425 1432
1426 1433 Affected:
1427 1434
1428 1435 self.basicHeaderObj
1429 1436 self.systemHeaderObj
1430 1437 self.radarControllerHeaderObj
1431 1438 self.processingHeaderObj self.
1432 1439
1433 1440 Return:
1434 1441 None
1435 1442 """
1436 1443
1437 1444 raise NotImplementedError
1438 1445
1439 1446 def __writeFirstHeader(self):
1440 1447 """
1441 1448 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1442 1449
1443 1450 Affected:
1444 1451 __dataType
1445 1452
1446 1453 Return:
1447 1454 None
1448 1455 """
1449 1456
1450 1457 # CALCULAR PARAMETROS
1451 1458
1452 1459 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1453 1460 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1454 1461
1455 1462 self.basicHeaderObj.write(self.fp)
1456 1463 self.systemHeaderObj.write(self.fp)
1457 1464 self.radarControllerHeaderObj.write(self.fp)
1458 1465 self.processingHeaderObj.write(self.fp)
1459 1466
1460 1467 def __setNewBlock(self):
1461 1468 """
1462 1469 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1463 1470
1464 1471 Return:
1465 1472 0 : si no pudo escribir nada
1466 1473 1 : Si escribio el Basic el First Header
1467 1474 """
1468 1475 if self.fp == None:
1469 1476 self.setNextFile()
1470 1477
1471 1478 if self.flagIsNewFile:
1472 1479 return 1
1473 1480
1474 1481 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1475 1482 self.basicHeaderObj.write(self.fp)
1476 1483 return 1
1477 1484
1478 1485 if not( self.setNextFile() ):
1479 1486 return 0
1480 1487
1481 1488 return 1
1482 1489
1483 1490
1484 1491 def writeNextBlock(self):
1485 1492 """
1486 1493 Selecciona el bloque siguiente de datos y los escribe en un file
1487 1494
1488 1495 Return:
1489 1496 0 : Si no hizo pudo escribir el bloque de datos
1490 1497 1 : Si no pudo escribir el bloque de datos
1491 1498 """
1492 1499 if not( self.__setNewBlock() ):
1493 1500 return 0
1494 1501
1495 1502 self.writeBlock()
1496 1503
1497 1504 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1498 1505 self.processingHeaderObj.dataBlocksPerFile)
1499 1506
1500 1507 return 1
1501 1508
1502 1509 def setNextFile(self):
1503 1510 """
1504 1511 Determina el siguiente file que sera escrito
1505 1512
1506 1513 Affected:
1507 1514 self.filename
1508 1515 self.subfolder
1509 1516 self.fp
1510 1517 self.setFile
1511 1518 self.flagIsNewFile
1512 1519
1513 1520 Return:
1514 1521 0 : Si el archivo no puede ser escrito
1515 1522 1 : Si el archivo esta listo para ser escrito
1516 1523 """
1517 1524 ext = self.ext
1518 1525 path = self.path
1519 1526
1520 1527 if self.fp != None:
1521 1528 self.fp.close()
1522 1529
1523 1530 timeTuple = time.localtime( self.dataOut.utctime)
1524 1531 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1525 1532
1526 1533 fullpath = os.path.join( path, subfolder )
1527 1534 setFile = self.setFile
1528 1535
1529 1536 if not( os.path.exists(fullpath) ):
1530 1537 os.mkdir(fullpath)
1531 1538 setFile = -1 #inicializo mi contador de seteo
1532 1539 else:
1533 1540 filesList = os.listdir( fullpath )
1534 1541 if len( filesList ) > 0:
1535 1542 filesList = sorted( filesList, key=str.lower )
1536 1543 filen = filesList[-1]
1537 1544 # el filename debera tener el siguiente formato
1538 1545 # 0 1234 567 89A BCDE (hex)
1539 1546 # x YYYY DDD SSS .ext
1540 1547 if isNumber( filen[8:11] ):
1541 1548 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1542 1549 else:
1543 1550 setFile = -1
1544 1551 else:
1545 1552 setFile = -1 #inicializo mi contador de seteo
1546 1553
1547 1554 setFile += 1
1548 1555
1549 1556 #If this is a new day it resets some values
1550 1557 if self.dataOut.datatime.date() > self.fileDate:
1551 1558 setFile = 0
1552 1559 self.nTotalBlocks = 0
1553 1560
1554 1561 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1555 1562
1556 1563 filename = os.path.join( path, subfolder, filen )
1557 1564
1558 1565 fp = open( filename,'wb' )
1559 1566
1560 1567 self.blockIndex = 0
1561 1568
1562 1569 #guardando atributos
1563 1570 self.filename = filename
1564 1571 self.subfolder = subfolder
1565 1572 self.fp = fp
1566 1573 self.setFile = setFile
1567 1574 self.flagIsNewFile = 1
1568 1575 self.fileDate = self.dataOut.datatime.date()
1569 1576
1570 1577 self.setFirstHeader()
1571 1578
1572 1579 print '[Writing] Opening file: %s'%self.filename
1573 1580
1574 1581 self.__writeFirstHeader()
1575 1582
1576 1583 return 1
1577 1584
1578 1585 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1579 1586 """
1580 1587 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1581 1588
1582 1589 Inputs:
1583 1590 path : directory where data will be saved
1584 1591 profilesPerBlock : number of profiles per block
1585 1592 set : initial file set
1586 1593 datatype : An integer number that defines data type:
1587 1594 0 : int8 (1 byte)
1588 1595 1 : int16 (2 bytes)
1589 1596 2 : int32 (4 bytes)
1590 1597 3 : int64 (8 bytes)
1591 1598 4 : float32 (4 bytes)
1592 1599 5 : double64 (8 bytes)
1593 1600
1594 1601 Return:
1595 1602 0 : Si no realizo un buen seteo
1596 1603 1 : Si realizo un buen seteo
1597 1604 """
1598 1605
1599 1606 if ext == None:
1600 1607 ext = self.ext
1601 1608
1602 1609 self.ext = ext.lower()
1603 1610
1604 1611 self.path = path
1605 1612
1606 1613 if set is None:
1607 1614 self.setFile = -1
1608 1615 else:
1609 1616 self.setFile = set - 1
1610 1617
1611 1618 self.blocksPerFile = blocksPerFile
1612 1619
1613 1620 self.profilesPerBlock = profilesPerBlock
1614 1621
1615 1622 self.dataOut = dataOut
1616 1623 self.fileDate = self.dataOut.datatime.date()
1617 1624 #By default
1618 1625 self.dtype = self.dataOut.dtype
1619 1626
1620 1627 if datatype is not None:
1621 1628 self.dtype = get_numpy_dtype(datatype)
1622 1629
1623 1630 if not(self.setNextFile()):
1624 1631 print "[Writing] There isn't a next file"
1625 1632 return 0
1626 1633
1627 1634 self.setBlockDimension()
1628 1635
1629 1636 return 1
1630 1637
1631 1638 def run(self, dataOut, **kwargs):
1632 1639
1633 1640 if not(self.isConfig):
1634 1641
1635 1642 self.setup(dataOut, **kwargs)
1636 1643 self.isConfig = True
1637 1644
1638 1645 self.putData()
1639 1646
General Comments 0
You need to be logged in to leave comments. Login now