##// END OF EJS Templates
schain
RSS
Miguel Valdez -
r700:5ddbdeccaf1c
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1636 +1,1639
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 print "The file %s can't be opened" %(filename)
72 72 return 0
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
87 87 """
88 88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89 89
90 90 Inputs:
91 91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92 92
93 93 startDate : fecha inicial del rango seleccionado en formato datetime.date
94 94
95 95 endDate : fecha final del rango seleccionado en formato datetime.date
96 96
97 97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
98 98
99 99 endTime : tiempo final del rango seleccionado en formato datetime.time
100 100
101 101 Return:
102 102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
103 103 fecha especificado, de lo contrario retorna False.
104 104
105 105 Excepciones:
106 106 Si el archivo no existe o no puede ser abierto
107 107 Si la cabecera no puede ser leida.
108 108
109 109 """
110 110
111 111
112 112 try:
113 113 fp = open(filename,'rb')
114 114 except IOError:
115 115 print "The file %s can't be opened" %(filename)
116 116 return None
117 117
118 118 basicHeaderObj = BasicHeader(LOCALTIME)
119 119 sts = basicHeaderObj.read(fp)
120 120 fp.close()
121 121
122 122 thisDatetime = basicHeaderObj.datatime
123 123 thisDate = thisDatetime.date()
124 124 thisTime = thisDatetime.time()
125 125
126 126 if not(sts):
127 127 print "Skipping the file %s because it has not a valid header" %(filename)
128 128 return None
129 129
130 130 #General case
131 131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
132 132 #-----------o----------------------------o-----------
133 133 # startTime endTime
134 134
135 135 if endTime >= startTime:
136 136 if (thisTime < startTime) or (thisTime > endTime):
137 137 return None
138 138
139 139 return thisDatetime
140 140
141 141 #If endTime < startTime then endTime belongs to the next day
142 142
143 143
144 144 #<<<<<<<<<<<o o>>>>>>>>>>>
145 145 #-----------o----------------------------o-----------
146 146 # endTime startTime
147 147
148 148 if (thisDate == startDate) and (thisTime < startTime):
149 149 return None
150 150
151 151 if (thisDate == endDate) and (thisTime > endTime):
152 152 return None
153 153
154 154 if (thisTime < startTime) and (thisTime > endTime):
155 155 return None
156 156
157 157 return thisDatetime
158 158
159 159 def isFolderInDateRange(folder, startDate=None, endDate=None):
160 160 """
161 161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
162 162
163 163 Inputs:
164 164 folder : nombre completo del directorio.
165 165 Su formato deberia ser "/path_root/?YYYYDDD"
166 166
167 167 siendo:
168 168 YYYY : Anio (ejemplo 2015)
169 169 DDD : Dia del anio (ejemplo 305)
170 170
171 171 startDate : fecha inicial del rango seleccionado en formato datetime.date
172 172
173 173 endDate : fecha final del rango seleccionado en formato datetime.date
174 174
175 175 Return:
176 176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
177 177 fecha especificado, de lo contrario retorna False.
178 178 Excepciones:
179 179 Si el directorio no tiene el formato adecuado
180 180 """
181 181
182 182 basename = os.path.basename(folder)
183 183
184 184 if not isRadarFolder(basename):
185 185 print "The folder %s has not the rigth format" %folder
186 186 return 0
187 187
188 188 if startDate and endDate:
189 189 thisDate = getDateFromRadarFolder(basename)
190 190
191 191 if thisDate < startDate:
192 192 return 0
193 193
194 194 if thisDate > endDate:
195 195 return 0
196 196
197 197 return 1
198 198
199 199 def isFileInDateRange(filename, startDate=None, endDate=None):
200 200 """
201 201 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
202 202
203 203 Inputs:
204 204 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
205 205
206 206 Su formato deberia ser "?YYYYDDDsss"
207 207
208 208 siendo:
209 209 YYYY : Anio (ejemplo 2015)
210 210 DDD : Dia del anio (ejemplo 305)
211 211 sss : set
212 212
213 213 startDate : fecha inicial del rango seleccionado en formato datetime.date
214 214
215 215 endDate : fecha final del rango seleccionado en formato datetime.date
216 216
217 217 Return:
218 218 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
219 219 fecha especificado, de lo contrario retorna False.
220 220 Excepciones:
221 221 Si el archivo no tiene el formato adecuado
222 222 """
223 223
224 224 basename = os.path.basename(filename)
225 225
226 226 if not isRadarFile(basename):
227 227 print "The filename %s has not the rigth format" %filename
228 228 return 0
229 229
230 230 if startDate and endDate:
231 231 thisDate = getDateFromRadarFile(basename)
232 232
233 233 if thisDate < startDate:
234 234 return 0
235 235
236 236 if thisDate > endDate:
237 237 return 0
238 238
239 239 return 1
240 240
241 241 def getFileFromSet(path, ext, set):
242 242 validFilelist = []
243 243 fileList = os.listdir(path)
244 244
245 245 # 0 1234 567 89A BCDE
246 246 # H YYYY DDD SSS .ext
247 247
248 248 for thisFile in fileList:
249 249 try:
250 250 year = int(thisFile[1:5])
251 251 doy = int(thisFile[5:8])
252 252 except:
253 253 continue
254 254
255 255 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
256 256 continue
257 257
258 258 validFilelist.append(thisFile)
259 259
260 260 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
261 261
262 262 if len(myfile)!= 0:
263 263 return myfile[0]
264 264 else:
265 265 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
266 266 print 'the filename %s does not exist'%filename
267 267 print '...going to the last file: '
268 268
269 269 if validFilelist:
270 270 validFilelist = sorted( validFilelist, key=str.lower )
271 271 return validFilelist[-1]
272 272
273 273 return None
274 274
275 275 def getlastFileFromPath(path, ext):
276 276 """
277 277 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
278 278 al final de la depuracion devuelve el ultimo file de la lista que quedo.
279 279
280 280 Input:
281 281 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
282 282 ext : extension de los files contenidos en una carpeta
283 283
284 284 Return:
285 285 El ultimo file de una determinada carpeta, no se considera el path.
286 286 """
287 287 validFilelist = []
288 288 fileList = os.listdir(path)
289 289
290 290 # 0 1234 567 89A BCDE
291 291 # H YYYY DDD SSS .ext
292 292
293 293 for thisFile in fileList:
294 294
295 295 year = thisFile[1:5]
296 296 if not isNumber(year):
297 297 continue
298 298
299 299 doy = thisFile[5:8]
300 300 if not isNumber(doy):
301 301 continue
302 302
303 303 year = int(year)
304 304 doy = int(doy)
305 305
306 306 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
307 307 continue
308 308
309 309 validFilelist.append(thisFile)
310 310
311 311 if validFilelist:
312 312 validFilelist = sorted( validFilelist, key=str.lower )
313 313 return validFilelist[-1]
314 314
315 315 return None
316 316
317 317 def checkForRealPath(path, foldercounter, year, doy, set, ext):
318 318 """
319 319 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
320 320 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
321 321 el path exacto de un determinado file.
322 322
323 323 Example :
324 324 nombre correcto del file es .../.../D2009307/P2009307367.ext
325 325
326 326 Entonces la funcion prueba con las siguientes combinaciones
327 327 .../.../y2009307367.ext
328 328 .../.../Y2009307367.ext
329 329 .../.../x2009307/y2009307367.ext
330 330 .../.../x2009307/Y2009307367.ext
331 331 .../.../X2009307/y2009307367.ext
332 332 .../.../X2009307/Y2009307367.ext
333 333 siendo para este caso, la ultima combinacion de letras, identica al file buscado
334 334
335 335 Return:
336 336 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
337 337 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
338 338 para el filename
339 339 """
340 340 fullfilename = None
341 341 find_flag = False
342 342 filename = None
343 343
344 344 prefixDirList = [None,'d','D']
345 345 if ext.lower() == ".r": #voltage
346 346 prefixFileList = ['d','D']
347 347 elif ext.lower() == ".pdata": #spectra
348 348 prefixFileList = ['p','P']
349 349 else:
350 350 return None, filename
351 351
352 352 #barrido por las combinaciones posibles
353 353 for prefixDir in prefixDirList:
354 354 thispath = path
355 355 if prefixDir != None:
356 356 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
357 357 if foldercounter == 0:
358 358 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
359 359 else:
360 360 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
361 361 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
362 362 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
363 363 fullfilename = os.path.join( thispath, filename ) #formo el path completo
364 364
365 365 if os.path.exists( fullfilename ): #verifico que exista
366 366 find_flag = True
367 367 break
368 368 if find_flag:
369 369 break
370 370
371 371 if not(find_flag):
372 372 return None, filename
373 373
374 374 return fullfilename, filename
375 375
376 376 def isRadarFolder(folder):
377 377 try:
378 378 year = int(folder[1:5])
379 379 doy = int(folder[5:8])
380 380 except:
381 381 return 0
382 382
383 383 return 1
384 384
385 385 def isRadarFile(file):
386 386 try:
387 387 year = int(file[1:5])
388 388 doy = int(file[5:8])
389 389 set = int(file[8:11])
390 390 except:
391 391 return 0
392 392
393 393 return 1
394 394
395 395 def getDateFromRadarFile(file):
396 396 try:
397 397 year = int(file[1:5])
398 398 doy = int(file[5:8])
399 399 set = int(file[8:11])
400 400 except:
401 401 return None
402 402
403 403 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
404 404 return thisDate
405 405
406 406 def getDateFromRadarFolder(folder):
407 407 try:
408 408 year = int(folder[1:5])
409 409 doy = int(folder[5:8])
410 410 except:
411 411 return None
412 412
413 413 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
414 414 return thisDate
415 415
416 416 class JRODataIO:
417 417
418 418 c = 3E8
419 419
420 420 isConfig = False
421 421
422 422 basicHeaderObj = None
423 423
424 424 systemHeaderObj = None
425 425
426 426 radarControllerHeaderObj = None
427 427
428 428 processingHeaderObj = None
429 429
430 430 dtype = None
431 431
432 432 pathList = []
433 433
434 434 filenameList = []
435 435
436 436 filename = None
437 437
438 438 ext = None
439 439
440 440 flagIsNewFile = 1
441 441
442 442 flagDiscontinuousBlock = 0
443 443
444 444 flagIsNewBlock = 0
445 445
446 446 fp = None
447 447
448 448 firstHeaderSize = 0
449 449
450 450 basicHeaderSize = 24
451 451
452 452 versionFile = 1103
453 453
454 454 fileSize = None
455 455
456 456 # ippSeconds = None
457 457
458 458 fileSizeByHeader = None
459 459
460 460 fileIndex = None
461 461
462 462 profileIndex = None
463 463
464 464 blockIndex = None
465 465
466 466 nTotalBlocks = None
467 467
468 468 maxTimeStep = 30
469 469
470 470 lastUTTime = None
471 471
472 472 datablock = None
473 473
474 474 dataOut = None
475 475
476 476 blocksize = None
477 477
478 478 getByBlock = False
479 479
480 480 def __init__(self):
481 481
482 482 raise NotImplementedError
483 483
484 484 def run(self):
485 485
486 486 raise NotImplementedError
487 487
488 488 def getDtypeWidth(self):
489 489
490 490 dtype_index = get_dtype_index(self.dtype)
491 491 dtype_width = get_dtype_width(dtype_index)
492 492
493 493 return dtype_width
494 494
495 495 class JRODataReader(JRODataIO):
496 496
497 497
498 498 online = 0
499 499
500 500 realtime = 0
501 501
502 502 nReadBlocks = 0
503 503
504 504 delay = 10 #number of seconds waiting a new file
505 505
506 506 nTries = 3 #quantity tries
507 507
508 508 nFiles = 3 #number of files for searching
509 509
510 510 path = None
511 511
512 512 foldercounter = 0
513 513
514 514 flagNoMoreFiles = 0
515 515
516 516 datetimeList = []
517 517
518 518 __isFirstTimeOnline = 1
519 519
520 520 __printInfo = True
521 521
522 522 profileIndex = None
523 523
524 524 nTxs = 1
525 525
526 526 txIndex = None
527 527
528 528 def __init__(self):
529 529
530 530 """
531 531 This class is used to find data files
532 532
533 533 Example:
534 534 reader = JRODataReader()
535 535 fileList = reader.findDataFiles()
536 536
537 537 """
538 538 pass
539 539
540 540
541 541 def createObjByDefault(self):
542 542 """
543 543
544 544 """
545 545 raise NotImplementedError
546 546
547 547 def getBlockDimension(self):
548 548
549 549 raise NotImplementedError
550 550
551 551 def __searchFilesOffLine(self,
552 552 path,
553 553 startDate=None,
554 554 endDate=None,
555 555 startTime=datetime.time(0,0,0),
556 556 endTime=datetime.time(23,59,59),
557 557 set=None,
558 558 expLabel='',
559 559 ext='.r',
560 560 walk=True):
561 561
562 562 self.filenameList = []
563 563 self.datetimeList = []
564 564
565 565 pathList = []
566 566
567 567 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
568 568
569 569 if dateList == []:
570 570 # print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
571 571 # datetime.datetime.combine(startDate,startTime).ctime(),
572 572 # datetime.datetime.combine(endDate,endTime).ctime())
573 573
574 574 return None, None
575 575
576 576 if len(dateList) > 1:
577 577 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
578 578 else:
579 579 print "[Reading] data was found for the date %s" %(dateList[0])
580 580
581 581 filenameList = []
582 582 datetimeList = []
583 583
584 584 for thisPath in pathList:
585 585 # thisPath = pathList[pathDict[file]]
586 586
587 587 fileList = glob.glob1(thisPath, "*%s" %ext)
588 588 fileList.sort()
589 589
590 590 for file in fileList:
591 591
592 592 filename = os.path.join(thisPath,file)
593 593
594 594 if not isFileInDateRange(filename, startDate, endDate):
595 595 continue
596 596
597 597 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
598 598
599 599 if not(thisDatetime):
600 600 continue
601 601
602 602 filenameList.append(filename)
603 603 datetimeList.append(thisDatetime)
604 604
605 605 if not(filenameList):
606 606 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
607 607 return None, None
608 608
609 609 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
610 610 print
611 611
612 612 for i in range(len(filenameList)):
613 613 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
614 614
615 615 self.filenameList = filenameList
616 616 self.datetimeList = datetimeList
617 617
618 618 return pathList, filenameList
619 619
620 620 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
621 621
622 622 """
623 623 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
624 624 devuelve el archivo encontrado ademas de otros datos.
625 625
626 626 Input:
627 627 path : carpeta donde estan contenidos los files que contiene data
628 628
629 629 expLabel : Nombre del subexperimento (subfolder)
630 630
631 631 ext : extension de los files
632 632
633 633 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
634 634
635 635 Return:
636 636 directory : eL directorio donde esta el file encontrado
637 637 filename : el ultimo file de una determinada carpeta
638 638 year : el anho
639 639 doy : el numero de dia del anho
640 640 set : el set del archivo
641 641
642 642
643 643 """
644 dirList = []
644 if not os.path.isdir(path):
645 return None, None, None, None, None, None
645 646
647 dirList = []
648
646 649 if not walk:
647 650 fullpath = path
648 651 foldercounter = 0
649 652 else:
650 653 #Filtra solo los directorios
651 654 for thisPath in os.listdir(path):
652 655 if not os.path.isdir(os.path.join(path,thisPath)):
653 656 continue
654 657 if not isRadarFolder(thisPath):
655 658 continue
656 659
657 660 dirList.append(thisPath)
658 661
659 662 if not(dirList):
660 663 return None, None, None, None, None, None
661 664
662 665 dirList = sorted( dirList, key=str.lower )
663 666
664 667 doypath = dirList[-1]
665 668 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
666 669 fullpath = os.path.join(path, doypath, expLabel)
667 670
668 671
669 672 print "[Reading] %s folder was found: " %(fullpath )
670 673
671 674 if set == None:
672 675 filename = getlastFileFromPath(fullpath, ext)
673 676 else:
674 677 filename = getFileFromSet(fullpath, ext, set)
675 678
676 679 if not(filename):
677 680 return None, None, None, None, None, None
678 681
679 682 print "[Reading] %s file was found" %(filename)
680 683
681 684 if not(self.__verifyFile(os.path.join(fullpath, filename))):
682 685 return None, None, None, None, None, None
683 686
684 687 year = int( filename[1:5] )
685 688 doy = int( filename[5:8] )
686 689 set = int( filename[8:11] )
687 690
688 691 return fullpath, foldercounter, filename, year, doy, set
689 692
690 693 def __setNextFileOffline(self):
691 694
692 695 idFile = self.fileIndex
693 696
694 697 while (True):
695 698 idFile += 1
696 699 if not(idFile < len(self.filenameList)):
697 700 self.flagNoMoreFiles = 1
698 701 # print "[Reading] No more Files"
699 702 return 0
700 703
701 704 filename = self.filenameList[idFile]
702 705
703 706 if not(self.__verifyFile(filename)):
704 707 continue
705 708
706 709 fileSize = os.path.getsize(filename)
707 710 fp = open(filename,'rb')
708 711 break
709 712
710 713 self.flagIsNewFile = 1
711 714 self.fileIndex = idFile
712 715 self.filename = filename
713 716 self.fileSize = fileSize
714 717 self.fp = fp
715 718
716 719 # print "[Reading] Setting the file: %s"%self.filename
717 720
718 721 return 1
719 722
720 723 def __setNextFileOnline(self):
721 724 """
722 725 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
723 726 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
724 727 siguientes.
725 728
726 729 Affected:
727 730 self.flagIsNewFile
728 731 self.filename
729 732 self.fileSize
730 733 self.fp
731 734 self.set
732 735 self.flagNoMoreFiles
733 736
734 737 Return:
735 738 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
736 739 1 : si el file fue abierto con exito y esta listo a ser leido
737 740
738 741 Excepciones:
739 742 Si un determinado file no puede ser abierto
740 743 """
741 744 nFiles = 0
742 745 fileOk_flag = False
743 746 firstTime_flag = True
744 747
745 748 self.set += 1
746 749
747 750 if self.set > 999:
748 751 self.set = 0
749 752 self.foldercounter += 1
750 753
751 754 #busca el 1er file disponible
752 755 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
753 756 if fullfilename:
754 757 if self.__verifyFile(fullfilename, False):
755 758 fileOk_flag = True
756 759
757 760 #si no encuentra un file entonces espera y vuelve a buscar
758 761 if not(fileOk_flag):
759 762 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
760 763
761 764 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
762 765 tries = self.nTries
763 766 else:
764 767 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
765 768
766 769 for nTries in range( tries ):
767 770 if firstTime_flag:
768 771 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
769 772 sleep( self.delay )
770 773 else:
771 774 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
772 775
773 776 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
774 777 if fullfilename:
775 778 if self.__verifyFile(fullfilename):
776 779 fileOk_flag = True
777 780 break
778 781
779 782 if fileOk_flag:
780 783 break
781 784
782 785 firstTime_flag = False
783 786
784 787 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
785 788 self.set += 1
786 789
787 790 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
788 791 self.set = 0
789 792 self.doy += 1
790 793 self.foldercounter = 0
791 794
792 795 if fileOk_flag:
793 796 self.fileSize = os.path.getsize( fullfilename )
794 797 self.filename = fullfilename
795 798 self.flagIsNewFile = 1
796 799 if self.fp != None: self.fp.close()
797 800 self.fp = open(fullfilename, 'rb')
798 801 self.flagNoMoreFiles = 0
799 802 # print '[Reading] Setting the file: %s' % fullfilename
800 803 else:
801 804 self.fileSize = 0
802 805 self.filename = None
803 806 self.flagIsNewFile = 0
804 807 self.fp = None
805 808 self.flagNoMoreFiles = 1
806 809 # print '[Reading] No more files to read'
807 810
808 811 return fileOk_flag
809 812
810 813 def setNextFile(self):
811 814 if self.fp != None:
812 815 self.fp.close()
813 816
814 817 if self.online:
815 818 newFile = self.__setNextFileOnline()
816 819 else:
817 820 newFile = self.__setNextFileOffline()
818 821
819 822 if not(newFile):
820 823 print '[Reading] No more files to read'
821 824 return 0
822 825
823 826 print '[Reading] Setting the file: %s' % self.filename
824 827
825 828 self.__readFirstHeader()
826 829 self.nReadBlocks = 0
827 830 return 1
828 831
829 832 def __waitNewBlock(self):
830 833 """
831 834 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
832 835
833 836 Si el modo de lectura es OffLine siempre retorn 0
834 837 """
835 838 if not self.online:
836 839 return 0
837 840
838 841 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
839 842 return 0
840 843
841 844 currentPointer = self.fp.tell()
842 845
843 846 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
844 847
845 848 for nTries in range( self.nTries ):
846 849
847 850 self.fp.close()
848 851 self.fp = open( self.filename, 'rb' )
849 852 self.fp.seek( currentPointer )
850 853
851 854 self.fileSize = os.path.getsize( self.filename )
852 855 currentSize = self.fileSize - currentPointer
853 856
854 857 if ( currentSize >= neededSize ):
855 858 self.basicHeaderObj.read(self.fp)
856 859 return 1
857 860
858 861 if self.fileSize == self.fileSizeByHeader:
859 862 # self.flagEoF = True
860 863 return 0
861 864
862 865 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
863 866 sleep( self.delay )
864 867
865 868
866 869 return 0
867 870
868 871 def waitDataBlock(self,pointer_location):
869 872
870 873 currentPointer = pointer_location
871 874
872 875 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
873 876
874 877 for nTries in range( self.nTries ):
875 878 self.fp.close()
876 879 self.fp = open( self.filename, 'rb' )
877 880 self.fp.seek( currentPointer )
878 881
879 882 self.fileSize = os.path.getsize( self.filename )
880 883 currentSize = self.fileSize - currentPointer
881 884
882 885 if ( currentSize >= neededSize ):
883 886 return 1
884 887
885 888 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
886 889 sleep( self.delay )
887 890
888 891 return 0
889 892
890 893 def __jumpToLastBlock(self):
891 894
892 895 if not(self.__isFirstTimeOnline):
893 896 return
894 897
895 898 csize = self.fileSize - self.fp.tell()
896 899 blocksize = self.processingHeaderObj.blockSize
897 900
898 901 #salta el primer bloque de datos
899 902 if csize > self.processingHeaderObj.blockSize:
900 903 self.fp.seek(self.fp.tell() + blocksize)
901 904 else:
902 905 return
903 906
904 907 csize = self.fileSize - self.fp.tell()
905 908 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
906 909 while True:
907 910
908 911 if self.fp.tell()<self.fileSize:
909 912 self.fp.seek(self.fp.tell() + neededsize)
910 913 else:
911 914 self.fp.seek(self.fp.tell() - neededsize)
912 915 break
913 916
914 917 # csize = self.fileSize - self.fp.tell()
915 918 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
916 919 # factor = int(csize/neededsize)
917 920 # if factor > 0:
918 921 # self.fp.seek(self.fp.tell() + factor*neededsize)
919 922
920 923 self.flagIsNewFile = 0
921 924 self.__isFirstTimeOnline = 0
922 925
923 926 def __setNewBlock(self):
924 927
925 928 if self.fp == None:
926 929 return 0
927 930
928 931 # if self.online:
929 932 # self.__jumpToLastBlock()
930 933
931 934 if self.flagIsNewFile:
932 935 self.lastUTTime = self.basicHeaderObj.utc
933 936 return 1
934 937
935 938 if self.realtime:
936 939 self.flagDiscontinuousBlock = 1
937 940 if not(self.setNextFile()):
938 941 return 0
939 942 else:
940 943 return 1
941 944
942 945 currentSize = self.fileSize - self.fp.tell()
943 946 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
944 947
945 948 if (currentSize >= neededSize):
946 949 self.basicHeaderObj.read(self.fp)
947 950 self.lastUTTime = self.basicHeaderObj.utc
948 951 return 1
949 952
950 953 if self.__waitNewBlock():
951 954 self.lastUTTime = self.basicHeaderObj.utc
952 955 return 1
953 956
954 957 if not(self.setNextFile()):
955 958 return 0
956 959
957 960 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
958 961 self.lastUTTime = self.basicHeaderObj.utc
959 962
960 963 self.flagDiscontinuousBlock = 0
961 964
962 965 if deltaTime > self.maxTimeStep:
963 966 self.flagDiscontinuousBlock = 1
964 967
965 968 return 1
966 969
967 970 def readNextBlock(self):
968 971
969 972 if not(self.__setNewBlock()):
970 973 return 0
971 974
972 975 if not(self.readBlock()):
973 976 return 0
974 977
975 978 self.getBasicHeader()
976 979
977 980 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
978 981 self.processingHeaderObj.dataBlocksPerFile,
979 982 self.dataOut.datatime.ctime())
980 983 return 1
981 984
982 985 def __readFirstHeader(self):
983 986
984 987 self.basicHeaderObj.read(self.fp)
985 988 self.systemHeaderObj.read(self.fp)
986 989 self.radarControllerHeaderObj.read(self.fp)
987 990 self.processingHeaderObj.read(self.fp)
988 991
989 992 self.firstHeaderSize = self.basicHeaderObj.size
990 993
991 994 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
992 995 if datatype == 0:
993 996 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
994 997 elif datatype == 1:
995 998 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
996 999 elif datatype == 2:
997 1000 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
998 1001 elif datatype == 3:
999 1002 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1000 1003 elif datatype == 4:
1001 1004 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1002 1005 elif datatype == 5:
1003 1006 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1004 1007 else:
1005 1008 raise ValueError, 'Data type was not defined'
1006 1009
1007 1010 self.dtype = datatype_str
1008 1011 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1009 1012 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1010 1013 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1011 1014 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1012 1015 self.getBlockDimension()
1013 1016
1014 1017 def __verifyFile(self, filename, msgFlag=True):
1015 1018
1016 1019 msg = None
1017 1020
1018 1021 try:
1019 1022 fp = open(filename, 'rb')
1020 1023 except IOError:
1021 1024
1022 1025 if msgFlag:
1023 1026 print "[Reading] File %s can't be opened" % (filename)
1024 1027
1025 1028 return False
1026 1029
1027 1030 currentPosition = fp.tell()
1028 1031 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1029 1032
1030 1033 if neededSize == 0:
1031 1034 basicHeaderObj = BasicHeader(LOCALTIME)
1032 1035 systemHeaderObj = SystemHeader()
1033 1036 radarControllerHeaderObj = RadarControllerHeader()
1034 1037 processingHeaderObj = ProcessingHeader()
1035 1038
1036 1039 if not( basicHeaderObj.read(fp) ):
1037 1040 fp.close()
1038 1041 return False
1039 1042
1040 1043 if not( systemHeaderObj.read(fp) ):
1041 1044 fp.close()
1042 1045 return False
1043 1046
1044 1047 if not( radarControllerHeaderObj.read(fp) ):
1045 1048 fp.close()
1046 1049 return False
1047 1050
1048 1051 if not( processingHeaderObj.read(fp) ):
1049 1052 fp.close()
1050 1053 return False
1051 1054
1052 1055 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1053 1056 else:
1054 1057 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1055 1058
1056 1059 fp.close()
1057 1060
1058 1061 fileSize = os.path.getsize(filename)
1059 1062 currentSize = fileSize - currentPosition
1060 1063
1061 1064 if currentSize < neededSize:
1062 1065 if msgFlag and (msg != None):
1063 1066 print msg
1064 1067 return False
1065 1068
1066 1069 return True
1067 1070
1068 1071 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1069 1072
1070 1073 dateList = []
1071 1074 pathList = []
1072 1075
1073 1076 multi_path = path.split(',')
1074 1077
1075 1078 if not walk:
1076 1079
1077 1080 for single_path in multi_path:
1078 1081
1079 1082 if not os.path.isdir(single_path):
1080 1083 continue
1081 1084
1082 1085 fileList = glob.glob1(single_path, "*"+ext)
1083 1086
1084 1087 for thisFile in fileList:
1085 1088
1086 1089 if not os.path.isfile(os.path.join(single_path, thisFile)):
1087 1090 continue
1088 1091
1089 1092 if not isRadarFile(thisFile):
1090 1093 continue
1091 1094
1092 1095 if not isFileInDateRange(thisFile, startDate, endDate):
1093 1096 continue
1094 1097
1095 1098 thisDate = getDateFromRadarFile(thisFile)
1096 1099
1097 1100 if thisDate in dateList:
1098 1101 continue
1099 1102
1100 1103 dateList.append(thisDate)
1101 1104 pathList.append(single_path)
1102 1105
1103 1106 else:
1104 1107 for single_path in multi_path:
1105 1108
1106 1109 if not os.path.isdir(single_path):
1107 1110 continue
1108 1111
1109 1112 dirList = []
1110 1113
1111 1114 for thisPath in os.listdir(single_path):
1112 1115
1113 1116 if not os.path.isdir(os.path.join(single_path,thisPath)):
1114 1117 continue
1115 1118
1116 1119 if not isRadarFolder(thisPath):
1117 1120 continue
1118 1121
1119 1122 if not isFolderInDateRange(thisPath, startDate, endDate):
1120 1123 continue
1121 1124
1122 1125 dirList.append(thisPath)
1123 1126
1124 1127 if not dirList:
1125 1128 continue
1126 1129
1127 1130 for thisDir in dirList:
1128 1131
1129 1132 datapath = os.path.join(single_path, thisDir, expLabel)
1130 1133 fileList = glob.glob1(datapath, "*"+ext)
1131 1134
1132 1135 if len(fileList) < 1:
1133 1136 continue
1134 1137
1135 1138 thisDate = getDateFromRadarFolder(thisDir)
1136 1139
1137 1140 pathList.append(datapath)
1138 1141 dateList.append(thisDate)
1139 1142
1140 1143 dateList.sort()
1141 1144
1142 1145 if walk:
1143 1146 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1144 1147 else:
1145 1148 pattern_path = multi_path[0]
1146 1149
1147 1150 if not dateList:
1148 1151 print "[Reading] No *%s files in %s from %s to %s" %(ext, pattern_path, startDate, endDate)
1149 1152
1150 1153 if include_path:
1151 1154 return dateList, pathList
1152 1155
1153 1156 return dateList
1154 1157
1155 1158 def setup(self,
1156 1159 path=None,
1157 1160 startDate=None,
1158 1161 endDate=None,
1159 1162 startTime=datetime.time(0,0,0),
1160 1163 endTime=datetime.time(23,59,59),
1161 1164 set=None,
1162 1165 expLabel = "",
1163 1166 ext = None,
1164 1167 online = False,
1165 1168 delay = 60,
1166 1169 walk = True,
1167 1170 getblock = False,
1168 1171 nTxs = 1,
1169 1172 realtime=False):
1170 1173
1171 1174 if path == None:
1172 1175 raise ValueError, "[Reading] The path is not valid"
1173 1176
1174 1177 if ext == None:
1175 1178 ext = self.ext
1176 1179
1177 1180 if online:
1178 1181 print "[Reading] Searching files in online mode..."
1179 1182
1180 1183 for nTries in range( self.nTries ):
1181 1184 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1182 1185
1183 1186 if fullpath:
1184 1187 break
1185 1188
1186 1189 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1187 1190 sleep( self.delay )
1188 1191
1189 1192 if not(fullpath):
1190 1193 print "[Reading] There 'isn't any valid file in %s" % path
1191 1194 return
1192 1195
1193 1196 self.year = year
1194 1197 self.doy = doy
1195 1198 self.set = set - 1
1196 1199 self.path = path
1197 1200 self.foldercounter = foldercounter
1198 1201 last_set = None
1199 1202
1200 1203 else:
1201 1204 print "[Reading] Searching files in offline mode ..."
1202 1205 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1203 1206 startTime=startTime, endTime=endTime,
1204 1207 set=set, expLabel=expLabel, ext=ext,
1205 1208 walk=walk)
1206 1209
1207 1210 if not(pathList):
1208 1211 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1209 1212 # datetime.datetime.combine(startDate,startTime).ctime(),
1210 1213 # datetime.datetime.combine(endDate,endTime).ctime())
1211 1214
1212 1215 # sys.exit(-1)
1213 1216
1214 1217 self.fileIndex = -1
1215 1218 self.pathList = []
1216 1219 self.filenameList = []
1217 1220 return
1218 1221
1219 1222 self.fileIndex = -1
1220 1223 self.pathList = pathList
1221 1224 self.filenameList = filenameList
1222 1225 file_name = os.path.basename(filenameList[-1])
1223 1226 basename, ext = os.path.splitext(file_name)
1224 1227 last_set = int(basename[-3:])
1225 1228
1226 1229 self.online = online
1227 1230 self.realtime = realtime
1228 1231 self.delay = delay
1229 1232 ext = ext.lower()
1230 1233 self.ext = ext
1231 1234 self.getByBlock = getblock
1232 1235 self.nTxs = int(nTxs)
1233 1236
1234 1237 if not(self.setNextFile()):
1235 1238 if (startDate!=None) and (endDate!=None):
1236 1239 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1237 1240 elif startDate != None:
1238 1241 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1239 1242 else:
1240 1243 print "[Reading] No files"
1241 1244
1242 1245 self.fileIndex = -1
1243 1246 self.pathList = []
1244 1247 self.filenameList = []
1245 1248 return
1246 1249
1247 1250 # self.getBasicHeader()
1248 1251
1249 1252 if last_set != None:
1250 1253 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1251 1254 return
1252 1255
1253 1256 def getBasicHeader(self):
1254 1257
1255 1258 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1256 1259
1257 1260 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1258 1261
1259 1262 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1260 1263
1261 1264 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1262 1265
1263 1266 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1264 1267
1265 1268 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1266 1269
1267 1270 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1268 1271
1269 1272 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1270 1273
1271 1274
1272 1275 def getFirstHeader(self):
1273 1276
1274 1277 raise NotImplementedError
1275 1278
1276 1279 def getData(self):
1277 1280
1278 1281 raise NotImplementedError
1279 1282
1280 1283 def hasNotDataInBuffer(self):
1281 1284
1282 1285 raise NotImplementedError
1283 1286
1284 1287 def readBlock(self):
1285 1288
1286 1289 raise NotImplementedError
1287 1290
1288 1291 def isEndProcess(self):
1289 1292
1290 1293 return self.flagNoMoreFiles
1291 1294
1292 1295 def printReadBlocks(self):
1293 1296
1294 1297 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1295 1298
1296 1299 def printTotalBlocks(self):
1297 1300
1298 1301 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1299 1302
1300 1303 def printNumberOfBlock(self):
1301 1304
1302 1305 if self.flagIsNewBlock:
1303 1306 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1304 1307 self.processingHeaderObj.dataBlocksPerFile,
1305 1308 self.dataOut.datatime.ctime())
1306 1309
1307 1310 def printInfo(self):
1308 1311
1309 1312 if self.__printInfo == False:
1310 1313 return
1311 1314
1312 1315 self.basicHeaderObj.printInfo()
1313 1316 self.systemHeaderObj.printInfo()
1314 1317 self.radarControllerHeaderObj.printInfo()
1315 1318 self.processingHeaderObj.printInfo()
1316 1319
1317 1320 self.__printInfo = False
1318 1321
1319 1322
1320 1323 def run(self, **kwargs):
1321 1324
1322 1325 if not(self.isConfig):
1323 1326
1324 1327 # self.dataOut = dataOut
1325 1328 self.setup(**kwargs)
1326 1329 self.isConfig = True
1327 1330
1328 1331 self.getData()
1329 1332
1330 1333 class JRODataWriter(JRODataIO):
1331 1334
1332 1335 """
1333 1336 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1334 1337 de los datos siempre se realiza por bloques.
1335 1338 """
1336 1339
1337 1340 blockIndex = 0
1338 1341
1339 1342 path = None
1340 1343
1341 1344 setFile = None
1342 1345
1343 1346 profilesPerBlock = None
1344 1347
1345 1348 blocksPerFile = None
1346 1349
1347 1350 nWriteBlocks = 0
1348 1351
1349 1352 fileDate = None
1350 1353
1351 1354 def __init__(self, dataOut=None):
1352 1355 raise NotImplementedError
1353 1356
1354 1357
1355 1358 def hasAllDataInBuffer(self):
1356 1359 raise NotImplementedError
1357 1360
1358 1361
1359 1362 def setBlockDimension(self):
1360 1363 raise NotImplementedError
1361 1364
1362 1365
1363 1366 def writeBlock(self):
1364 1367 raise NotImplementedError
1365 1368
1366 1369
1367 1370 def putData(self):
1368 1371 raise NotImplementedError
1369 1372
1370 1373
1371 1374 def getProcessFlags(self):
1372 1375
1373 1376 processFlags = 0
1374 1377
1375 1378 dtype_index = get_dtype_index(self.dtype)
1376 1379 procflag_dtype = get_procflag_dtype(dtype_index)
1377 1380
1378 1381 processFlags += procflag_dtype
1379 1382
1380 1383 if self.dataOut.flagDecodeData:
1381 1384 processFlags += PROCFLAG.DECODE_DATA
1382 1385
1383 1386 if self.dataOut.flagDeflipData:
1384 1387 processFlags += PROCFLAG.DEFLIP_DATA
1385 1388
1386 1389 if self.dataOut.code is not None:
1387 1390 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1388 1391
1389 1392 if self.dataOut.nCohInt > 1:
1390 1393 processFlags += PROCFLAG.COHERENT_INTEGRATION
1391 1394
1392 1395 if self.dataOut.type == "Spectra":
1393 1396 if self.dataOut.nIncohInt > 1:
1394 1397 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1395 1398
1396 1399 if self.dataOut.data_dc is not None:
1397 1400 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1398 1401
1399 1402 if self.dataOut.flagShiftFFT:
1400 1403 processFlags += PROCFLAG.SHIFT_FFT_DATA
1401 1404
1402 1405 return processFlags
1403 1406
1404 1407 def setBasicHeader(self):
1405 1408
1406 1409 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1407 1410 self.basicHeaderObj.version = self.versionFile
1408 1411 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1409 1412
1410 1413 utc = numpy.floor(self.dataOut.utctime)
1411 1414 milisecond = (self.dataOut.utctime - utc)* 1000.0
1412 1415
1413 1416 self.basicHeaderObj.utc = utc
1414 1417 self.basicHeaderObj.miliSecond = milisecond
1415 1418 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1416 1419 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1417 1420 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1418 1421
1419 1422 def setFirstHeader(self):
1420 1423 """
1421 1424 Obtiene una copia del First Header
1422 1425
1423 1426 Affected:
1424 1427
1425 1428 self.basicHeaderObj
1426 1429 self.systemHeaderObj
1427 1430 self.radarControllerHeaderObj
1428 1431 self.processingHeaderObj self.
1429 1432
1430 1433 Return:
1431 1434 None
1432 1435 """
1433 1436
1434 1437 raise NotImplementedError
1435 1438
1436 1439 def __writeFirstHeader(self):
1437 1440 """
1438 1441 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1439 1442
1440 1443 Affected:
1441 1444 __dataType
1442 1445
1443 1446 Return:
1444 1447 None
1445 1448 """
1446 1449
1447 1450 # CALCULAR PARAMETROS
1448 1451
1449 1452 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1450 1453 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1451 1454
1452 1455 self.basicHeaderObj.write(self.fp)
1453 1456 self.systemHeaderObj.write(self.fp)
1454 1457 self.radarControllerHeaderObj.write(self.fp)
1455 1458 self.processingHeaderObj.write(self.fp)
1456 1459
1457 1460 def __setNewBlock(self):
1458 1461 """
1459 1462 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1460 1463
1461 1464 Return:
1462 1465 0 : si no pudo escribir nada
1463 1466 1 : Si escribio el Basic el First Header
1464 1467 """
1465 1468 if self.fp == None:
1466 1469 self.setNextFile()
1467 1470
1468 1471 if self.flagIsNewFile:
1469 1472 return 1
1470 1473
1471 1474 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1472 1475 self.basicHeaderObj.write(self.fp)
1473 1476 return 1
1474 1477
1475 1478 if not( self.setNextFile() ):
1476 1479 return 0
1477 1480
1478 1481 return 1
1479 1482
1480 1483
1481 1484 def writeNextBlock(self):
1482 1485 """
1483 1486 Selecciona el bloque siguiente de datos y los escribe en un file
1484 1487
1485 1488 Return:
1486 1489 0 : Si no hizo pudo escribir el bloque de datos
1487 1490 1 : Si no pudo escribir el bloque de datos
1488 1491 """
1489 1492 if not( self.__setNewBlock() ):
1490 1493 return 0
1491 1494
1492 1495 self.writeBlock()
1493 1496
1494 1497 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1495 1498 self.processingHeaderObj.dataBlocksPerFile)
1496 1499
1497 1500 return 1
1498 1501
1499 1502 def setNextFile(self):
1500 1503 """
1501 1504 Determina el siguiente file que sera escrito
1502 1505
1503 1506 Affected:
1504 1507 self.filename
1505 1508 self.subfolder
1506 1509 self.fp
1507 1510 self.setFile
1508 1511 self.flagIsNewFile
1509 1512
1510 1513 Return:
1511 1514 0 : Si el archivo no puede ser escrito
1512 1515 1 : Si el archivo esta listo para ser escrito
1513 1516 """
1514 1517 ext = self.ext
1515 1518 path = self.path
1516 1519
1517 1520 if self.fp != None:
1518 1521 self.fp.close()
1519 1522
1520 1523 timeTuple = time.localtime( self.dataOut.utctime)
1521 1524 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1522 1525
1523 1526 fullpath = os.path.join( path, subfolder )
1524 1527 setFile = self.setFile
1525 1528
1526 1529 if not( os.path.exists(fullpath) ):
1527 1530 os.mkdir(fullpath)
1528 1531 setFile = -1 #inicializo mi contador de seteo
1529 1532 else:
1530 1533 filesList = os.listdir( fullpath )
1531 1534 if len( filesList ) > 0:
1532 1535 filesList = sorted( filesList, key=str.lower )
1533 1536 filen = filesList[-1]
1534 1537 # el filename debera tener el siguiente formato
1535 1538 # 0 1234 567 89A BCDE (hex)
1536 1539 # x YYYY DDD SSS .ext
1537 1540 if isNumber( filen[8:11] ):
1538 1541 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1539 1542 else:
1540 1543 setFile = -1
1541 1544 else:
1542 1545 setFile = -1 #inicializo mi contador de seteo
1543 1546
1544 1547 setFile += 1
1545 1548
1546 1549 #If this is a new day it resets some values
1547 1550 if self.dataOut.datatime.date() > self.fileDate:
1548 1551 setFile = 0
1549 1552 self.nTotalBlocks = 0
1550 1553
1551 1554 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1552 1555
1553 1556 filename = os.path.join( path, subfolder, filen )
1554 1557
1555 1558 fp = open( filename,'wb' )
1556 1559
1557 1560 self.blockIndex = 0
1558 1561
1559 1562 #guardando atributos
1560 1563 self.filename = filename
1561 1564 self.subfolder = subfolder
1562 1565 self.fp = fp
1563 1566 self.setFile = setFile
1564 1567 self.flagIsNewFile = 1
1565 1568 self.fileDate = self.dataOut.datatime.date()
1566 1569
1567 1570 self.setFirstHeader()
1568 1571
1569 1572 print '[Writing] Opening file: %s'%self.filename
1570 1573
1571 1574 self.__writeFirstHeader()
1572 1575
1573 1576 return 1
1574 1577
1575 1578 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1576 1579 """
1577 1580 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1578 1581
1579 1582 Inputs:
1580 1583 path : directory where data will be saved
1581 1584 profilesPerBlock : number of profiles per block
1582 1585 set : initial file set
1583 1586 datatype : An integer number that defines data type:
1584 1587 0 : int8 (1 byte)
1585 1588 1 : int16 (2 bytes)
1586 1589 2 : int32 (4 bytes)
1587 1590 3 : int64 (8 bytes)
1588 1591 4 : float32 (4 bytes)
1589 1592 5 : double64 (8 bytes)
1590 1593
1591 1594 Return:
1592 1595 0 : Si no realizo un buen seteo
1593 1596 1 : Si realizo un buen seteo
1594 1597 """
1595 1598
1596 1599 if ext == None:
1597 1600 ext = self.ext
1598 1601
1599 1602 self.ext = ext.lower()
1600 1603
1601 1604 self.path = path
1602 1605
1603 1606 if set is None:
1604 1607 self.setFile = -1
1605 1608 else:
1606 1609 self.setFile = set - 1
1607 1610
1608 1611 self.blocksPerFile = blocksPerFile
1609 1612
1610 1613 self.profilesPerBlock = profilesPerBlock
1611 1614
1612 1615 self.dataOut = dataOut
1613 1616 self.fileDate = self.dataOut.datatime.date()
1614 1617 #By default
1615 1618 self.dtype = self.dataOut.dtype
1616 1619
1617 1620 if datatype is not None:
1618 1621 self.dtype = get_numpy_dtype(datatype)
1619 1622
1620 1623 if not(self.setNextFile()):
1621 1624 print "[Writing] There isn't a next file"
1622 1625 return 0
1623 1626
1624 1627 self.setBlockDimension()
1625 1628
1626 1629 return 1
1627 1630
1628 1631 def run(self, dataOut, **kwargs):
1629 1632
1630 1633 if not(self.isConfig):
1631 1634
1632 1635 self.setup(dataOut, **kwargs)
1633 1636 self.isConfig = True
1634 1637
1635 1638 self.putData()
1636 1639
General Comments 0
You need to be logged in to leave comments. Login now