##// END OF EJS Templates
schain
RSS
sys.exit(-1) was deleted from jroIO_base.py
Miguel Valdez -
r676:526ad8aef3b2
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1611 +1,1618
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 traceback.print_exc()
72 72 raise IOError, "The file %s can't be opened" %(filename)
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
87 87 """
88 88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89 89
90 90 Inputs:
91 91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92 92
93 93 startDate : fecha inicial del rango seleccionado en formato datetime.date
94 94
95 95 endDate : fecha final del rango seleccionado en formato datetime.date
96 96
97 97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
98 98
99 99 endTime : tiempo final del rango seleccionado en formato datetime.time
100 100
101 101 Return:
102 102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
103 103 fecha especificado, de lo contrario retorna False.
104 104
105 105 Excepciones:
106 106 Si el archivo no existe o no puede ser abierto
107 107 Si la cabecera no puede ser leida.
108 108
109 109 """
110 110
111 111
112 112 try:
113 113 fp = open(filename,'rb')
114 114 except IOError:
115 115 traceback.print_exc()
116 116 raise IOError, "The file %s can't be opened" %(filename)
117 117
118 118 basicHeaderObj = BasicHeader(LOCALTIME)
119 119 sts = basicHeaderObj.read(fp)
120 120 fp.close()
121 121
122 122 thisDatetime = basicHeaderObj.datatime
123 123 thisDate = thisDatetime.date()
124 124 thisTime = thisDatetime.time()
125 125
126 126 if not(sts):
127 127 print "Skipping the file %s because it has not a valid header" %(filename)
128 128 return None
129 129
130 130 #General case
131 131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
132 132 #-----------o----------------------------o-----------
133 133 # startTime endTime
134 134
135 135 if endTime >= startTime:
136 136 if (thisTime < startTime) or (thisTime > endTime):
137 137 return None
138 138
139 139 return thisDatetime
140 140
141 141 #If endTime < startTime then endTime belongs to the next day
142 142
143 143
144 144 #<<<<<<<<<<<o o>>>>>>>>>>>
145 145 #-----------o----------------------------o-----------
146 146 # endTime startTime
147 147
148 148 if (thisDate == startDate) and (thisTime < startTime):
149 149 return None
150 150
151 151 if (thisDate == endDate) and (thisTime > endTime):
152 152 return None
153 153
154 154 if (thisTime < startTime) and (thisTime > endTime):
155 155 return None
156 156
157 157 return thisDatetime
158 158
159 159 def isFolderInDateRange(folder, startDate=None, endDate=None):
160 160 """
161 161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
162 162
163 163 Inputs:
164 164 folder : nombre completo del directorio.
165 165 Su formato deberia ser "/path_root/?YYYYDDD"
166 166
167 167 siendo:
168 168 YYYY : Anio (ejemplo 2015)
169 169 DDD : Dia del anio (ejemplo 305)
170 170
171 171 startDate : fecha inicial del rango seleccionado en formato datetime.date
172 172
173 173 endDate : fecha final del rango seleccionado en formato datetime.date
174 174
175 175 Return:
176 176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
177 177 fecha especificado, de lo contrario retorna False.
178 178 Excepciones:
179 179 Si el directorio no tiene el formato adecuado
180 180 """
181 181
182 182 basename = os.path.basename(folder)
183 183
184 184 if not isRadarFolder(basename):
185 185 raise IOError, "The folder %s has not the rigth format" %folder
186 186
187 187 if startDate and endDate:
188 188 thisDate = getDateFromRadarFolder(basename)
189 189
190 190 if thisDate < startDate:
191 191 return 0
192 192
193 193 if thisDate > endDate:
194 194 return 0
195 195
196 196 return 1
197 197
198 198 def isFileInDateRange(filename, startDate=None, endDate=None):
199 199 """
200 200 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
201 201
202 202 Inputs:
203 203 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
204 204
205 205 Su formato deberia ser "?YYYYDDDsss"
206 206
207 207 siendo:
208 208 YYYY : Anio (ejemplo 2015)
209 209 DDD : Dia del anio (ejemplo 305)
210 210 sss : set
211 211
212 212 startDate : fecha inicial del rango seleccionado en formato datetime.date
213 213
214 214 endDate : fecha final del rango seleccionado en formato datetime.date
215 215
216 216 Return:
217 217 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
218 218 fecha especificado, de lo contrario retorna False.
219 219 Excepciones:
220 220 Si el archivo no tiene el formato adecuado
221 221 """
222 222
223 223 basename = os.path.basename(filename)
224 224
225 225 if not isRadarFile(basename):
226 226 raise IOError, "The filename %s has not the rigth format" %filename
227 227
228 228 if startDate and endDate:
229 229 thisDate = getDateFromRadarFile(basename)
230 230
231 231 if thisDate < startDate:
232 232 return 0
233 233
234 234 if thisDate > endDate:
235 235 return 0
236 236
237 237 return 1
238 238
239 239 def getFileFromSet(path, ext, set):
240 240 validFilelist = []
241 241 fileList = os.listdir(path)
242 242
243 243 # 0 1234 567 89A BCDE
244 244 # H YYYY DDD SSS .ext
245 245
246 246 for thisFile in fileList:
247 247 try:
248 248 year = int(thisFile[1:5])
249 249 doy = int(thisFile[5:8])
250 250 except:
251 251 continue
252 252
253 253 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
254 254 continue
255 255
256 256 validFilelist.append(thisFile)
257 257
258 258 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
259 259
260 260 if len(myfile)!= 0:
261 261 return myfile[0]
262 262 else:
263 263 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
264 264 print 'the filename %s does not exist'%filename
265 265 print '...going to the last file: '
266 266
267 267 if validFilelist:
268 268 validFilelist = sorted( validFilelist, key=str.lower )
269 269 return validFilelist[-1]
270 270
271 271 return None
272 272
273 273 def getlastFileFromPath(path, ext):
274 274 """
275 275 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
276 276 al final de la depuracion devuelve el ultimo file de la lista que quedo.
277 277
278 278 Input:
279 279 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
280 280 ext : extension de los files contenidos en una carpeta
281 281
282 282 Return:
283 283 El ultimo file de una determinada carpeta, no se considera el path.
284 284 """
285 285 validFilelist = []
286 286 fileList = os.listdir(path)
287 287
288 288 # 0 1234 567 89A BCDE
289 289 # H YYYY DDD SSS .ext
290 290
291 291 for thisFile in fileList:
292 292
293 293 year = thisFile[1:5]
294 294 if not isNumber(year):
295 295 continue
296 296
297 297 doy = thisFile[5:8]
298 298 if not isNumber(doy):
299 299 continue
300 300
301 301 year = int(year)
302 302 doy = int(doy)
303 303
304 304 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
305 305 continue
306 306
307 307 validFilelist.append(thisFile)
308 308
309 309 if validFilelist:
310 310 validFilelist = sorted( validFilelist, key=str.lower )
311 311 return validFilelist[-1]
312 312
313 313 return None
314 314
315 315 def checkForRealPath(path, foldercounter, year, doy, set, ext):
316 316 """
317 317 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
318 318 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
319 319 el path exacto de un determinado file.
320 320
321 321 Example :
322 322 nombre correcto del file es .../.../D2009307/P2009307367.ext
323 323
324 324 Entonces la funcion prueba con las siguientes combinaciones
325 325 .../.../y2009307367.ext
326 326 .../.../Y2009307367.ext
327 327 .../.../x2009307/y2009307367.ext
328 328 .../.../x2009307/Y2009307367.ext
329 329 .../.../X2009307/y2009307367.ext
330 330 .../.../X2009307/Y2009307367.ext
331 331 siendo para este caso, la ultima combinacion de letras, identica al file buscado
332 332
333 333 Return:
334 334 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
335 335 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
336 336 para el filename
337 337 """
338 338 fullfilename = None
339 339 find_flag = False
340 340 filename = None
341 341
342 342 prefixDirList = [None,'d','D']
343 343 if ext.lower() == ".r": #voltage
344 344 prefixFileList = ['d','D']
345 345 elif ext.lower() == ".pdata": #spectra
346 346 prefixFileList = ['p','P']
347 347 else:
348 348 return None, filename
349 349
350 350 #barrido por las combinaciones posibles
351 351 for prefixDir in prefixDirList:
352 352 thispath = path
353 353 if prefixDir != None:
354 354 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
355 355 if foldercounter == 0:
356 356 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
357 357 else:
358 358 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
359 359 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
360 360 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
361 361 fullfilename = os.path.join( thispath, filename ) #formo el path completo
362 362
363 363 if os.path.exists( fullfilename ): #verifico que exista
364 364 find_flag = True
365 365 break
366 366 if find_flag:
367 367 break
368 368
369 369 if not(find_flag):
370 370 return None, filename
371 371
372 372 return fullfilename, filename
373 373
374 374 def isRadarFolder(folder):
375 375 try:
376 376 year = int(folder[1:5])
377 377 doy = int(folder[5:8])
378 378 except:
379 379 return 0
380 380
381 381 return 1
382 382
383 383 def isRadarFile(file):
384 384 try:
385 385 year = int(file[1:5])
386 386 doy = int(file[5:8])
387 387 set = int(file[8:11])
388 388 except:
389 389 return 0
390 390
391 391 return 1
392 392
393 393 def getDateFromRadarFile(file):
394 394 try:
395 395 year = int(file[1:5])
396 396 doy = int(file[5:8])
397 397 set = int(file[8:11])
398 398 except:
399 399 return None
400 400
401 401 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
402 402 return thisDate
403 403
404 404 def getDateFromRadarFolder(folder):
405 405 try:
406 406 year = int(folder[1:5])
407 407 doy = int(folder[5:8])
408 408 except:
409 409 return None
410 410
411 411 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
412 412 return thisDate
413 413
414 414 class JRODataIO:
415 415
416 416 c = 3E8
417 417
418 418 isConfig = False
419 419
420 420 basicHeaderObj = None
421 421
422 422 systemHeaderObj = None
423 423
424 424 radarControllerHeaderObj = None
425 425
426 426 processingHeaderObj = None
427 427
428 428 dtype = None
429 429
430 430 pathList = []
431 431
432 432 filenameList = []
433 433
434 434 filename = None
435 435
436 436 ext = None
437 437
438 438 flagIsNewFile = 1
439 439
440 440 flagDiscontinuousBlock = 0
441 441
442 442 flagIsNewBlock = 0
443 443
444 444 fp = None
445 445
446 446 firstHeaderSize = 0
447 447
448 448 basicHeaderSize = 24
449 449
450 450 versionFile = 1103
451 451
452 452 fileSize = None
453 453
454 454 # ippSeconds = None
455 455
456 456 fileSizeByHeader = None
457 457
458 458 fileIndex = None
459 459
460 460 profileIndex = None
461 461
462 462 blockIndex = None
463 463
464 464 nTotalBlocks = None
465 465
466 466 maxTimeStep = 30
467 467
468 468 lastUTTime = None
469 469
470 470 datablock = None
471 471
472 472 dataOut = None
473 473
474 474 blocksize = None
475 475
476 476 getByBlock = False
477 477
478 478 def __init__(self):
479 479
480 480 raise ValueError, "Not implemented"
481 481
482 482 def run(self):
483 483
484 484 raise ValueError, "Not implemented"
485 485
486 486 def getDtypeWidth(self):
487 487
488 488 dtype_index = get_dtype_index(self.dtype)
489 489 dtype_width = get_dtype_width(dtype_index)
490 490
491 491 return dtype_width
492 492
493 493 class JRODataReader(JRODataIO):
494 494
495 495
496 496 online = 0
497 497
498 498 realtime = 0
499 499
500 500 nReadBlocks = 0
501 501
502 502 delay = 10 #number of seconds waiting a new file
503 503
504 504 nTries = 3 #quantity tries
505 505
506 506 nFiles = 3 #number of files for searching
507 507
508 508 path = None
509 509
510 510 foldercounter = 0
511 511
512 512 flagNoMoreFiles = 0
513 513
514 514 datetimeList = []
515 515
516 516 __isFirstTimeOnline = 1
517 517
518 518 __printInfo = True
519 519
520 520 profileIndex = None
521 521
522 522 nTxs = 1
523 523
524 524 txIndex = None
525 525
526 526 def __init__(self):
527 527
528 528 """
529 529
530 530 """
531 531
532 532 # raise NotImplementedError, "This method has not been implemented"
533 533
534 534
535 535 def createObjByDefault(self):
536 536 """
537 537
538 538 """
539 539 raise NotImplementedError, "This method has not been implemented"
540 540
541 541 def getBlockDimension(self):
542 542
543 543 raise NotImplementedError, "No implemented"
544 544
545 545 def __searchFilesOffLine(self,
546 546 path,
547 547 startDate=None,
548 548 endDate=None,
549 549 startTime=datetime.time(0,0,0),
550 550 endTime=datetime.time(23,59,59),
551 551 set=None,
552 552 expLabel='',
553 553 ext='.r',
554 554 walk=True):
555 555
556 556 self.filenameList = []
557 557 self.datetimeList = []
558 558
559 559 pathList = []
560 560
561 561 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
562 562
563 563 if dateList == []:
564 564 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
565 565 datetime.datetime.combine(startDate,startTime).ctime(),
566 566 datetime.datetime.combine(endDate,endTime).ctime())
567 567
568 568 return None, None
569 569
570 570 if len(dateList) > 1:
571 571 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
572 572 else:
573 573 print "[Reading] data was found for the date %s" %(dateList[0])
574 574
575 575 filenameList = []
576 576 datetimeList = []
577 577
578 578 for thisPath in pathList:
579 579 # thisPath = pathList[pathDict[file]]
580 580
581 581 fileList = glob.glob1(thisPath, "*%s" %ext)
582 582 fileList.sort()
583 583
584 584 for file in fileList:
585 585
586 586 filename = os.path.join(thisPath,file)
587 587
588 588 if not isFileInDateRange(filename, startDate, endDate):
589 589 continue
590 590
591 591 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
592 592
593 593 if not(thisDatetime):
594 594 continue
595 595
596 596 filenameList.append(filename)
597 597 datetimeList.append(thisDatetime)
598 598
599 599 if not(filenameList):
600 600 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
601 601 return None, None
602 602
603 603 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
604 604 print
605 605
606 606 for i in range(len(filenameList)):
607 607 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
608 608
609 609 self.filenameList = filenameList
610 610 self.datetimeList = datetimeList
611 611
612 612 return pathList, filenameList
613 613
614 614 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
615 615
616 616 """
617 617 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
618 618 devuelve el archivo encontrado ademas de otros datos.
619 619
620 620 Input:
621 621 path : carpeta donde estan contenidos los files que contiene data
622 622
623 623 expLabel : Nombre del subexperimento (subfolder)
624 624
625 625 ext : extension de los files
626 626
627 627 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
628 628
629 629 Return:
630 630 directory : eL directorio donde esta el file encontrado
631 631 filename : el ultimo file de una determinada carpeta
632 632 year : el anho
633 633 doy : el numero de dia del anho
634 634 set : el set del archivo
635 635
636 636
637 637 """
638 638 dirList = []
639 639
640 640 if not walk:
641 641 fullpath = path
642 642 foldercounter = 0
643 643 else:
644 644 #Filtra solo los directorios
645 645 for thisPath in os.listdir(path):
646 646 if not os.path.isdir(os.path.join(path,thisPath)):
647 647 continue
648 648 if not isRadarFolder(thisPath):
649 649 continue
650 650
651 651 dirList.append(thisPath)
652 652
653 653 if not(dirList):
654 654 return None, None, None, None, None, None
655 655
656 656 dirList = sorted( dirList, key=str.lower )
657 657
658 658 doypath = dirList[-1]
659 659 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
660 660 fullpath = os.path.join(path, doypath, expLabel)
661 661
662 662
663 663 print "[Reading] %s folder was found: " %(fullpath )
664 664
665 665 if set == None:
666 666 filename = getlastFileFromPath(fullpath, ext)
667 667 else:
668 668 filename = getFileFromSet(fullpath, ext, set)
669 669
670 670 if not(filename):
671 671 return None, None, None, None, None, None
672 672
673 673 print "[Reading] %s file was found" %(filename)
674 674
675 675 if not(self.__verifyFile(os.path.join(fullpath, filename))):
676 676 return None, None, None, None, None, None
677 677
678 678 year = int( filename[1:5] )
679 679 doy = int( filename[5:8] )
680 680 set = int( filename[8:11] )
681 681
682 682 return fullpath, foldercounter, filename, year, doy, set
683 683
684 684 def __setNextFileOffline(self):
685 685
686 686 idFile = self.fileIndex
687 687
688 688 while (True):
689 689 idFile += 1
690 690 if not(idFile < len(self.filenameList)):
691 691 self.flagNoMoreFiles = 1
692 692 # print "[Reading] No more Files"
693 693 return 0
694 694
695 695 filename = self.filenameList[idFile]
696 696
697 697 if not(self.__verifyFile(filename)):
698 698 continue
699 699
700 700 fileSize = os.path.getsize(filename)
701 701 fp = open(filename,'rb')
702 702 break
703 703
704 704 self.flagIsNewFile = 1
705 705 self.fileIndex = idFile
706 706 self.filename = filename
707 707 self.fileSize = fileSize
708 708 self.fp = fp
709 709
710 710 # print "[Reading] Setting the file: %s"%self.filename
711 711
712 712 return 1
713 713
714 714 def __setNextFileOnline(self):
715 715 """
716 716 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
717 717 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
718 718 siguientes.
719 719
720 720 Affected:
721 721 self.flagIsNewFile
722 722 self.filename
723 723 self.fileSize
724 724 self.fp
725 725 self.set
726 726 self.flagNoMoreFiles
727 727
728 728 Return:
729 729 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
730 730 1 : si el file fue abierto con exito y esta listo a ser leido
731 731
732 732 Excepciones:
733 733 Si un determinado file no puede ser abierto
734 734 """
735 735 nFiles = 0
736 736 fileOk_flag = False
737 737 firstTime_flag = True
738 738
739 739 self.set += 1
740 740
741 741 if self.set > 999:
742 742 self.set = 0
743 743 self.foldercounter += 1
744 744
745 745 #busca el 1er file disponible
746 746 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
747 747 if fullfilename:
748 748 if self.__verifyFile(fullfilename, False):
749 749 fileOk_flag = True
750 750
751 751 #si no encuentra un file entonces espera y vuelve a buscar
752 752 if not(fileOk_flag):
753 753 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
754 754
755 755 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
756 756 tries = self.nTries
757 757 else:
758 758 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
759 759
760 760 for nTries in range( tries ):
761 761 if firstTime_flag:
762 762 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
763 763 sleep( self.delay )
764 764 else:
765 765 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
766 766
767 767 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
768 768 if fullfilename:
769 769 if self.__verifyFile(fullfilename):
770 770 fileOk_flag = True
771 771 break
772 772
773 773 if fileOk_flag:
774 774 break
775 775
776 776 firstTime_flag = False
777 777
778 778 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
779 779 self.set += 1
780 780
781 781 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
782 782 self.set = 0
783 783 self.doy += 1
784 784 self.foldercounter = 0
785 785
786 786 if fileOk_flag:
787 787 self.fileSize = os.path.getsize( fullfilename )
788 788 self.filename = fullfilename
789 789 self.flagIsNewFile = 1
790 790 if self.fp != None: self.fp.close()
791 791 self.fp = open(fullfilename, 'rb')
792 792 self.flagNoMoreFiles = 0
793 793 # print '[Reading] Setting the file: %s' % fullfilename
794 794 else:
795 795 self.fileSize = 0
796 796 self.filename = None
797 797 self.flagIsNewFile = 0
798 798 self.fp = None
799 799 self.flagNoMoreFiles = 1
800 800 # print '[Reading] No more files to read'
801 801
802 802 return fileOk_flag
803 803
804 804 def setNextFile(self):
805 805 if self.fp != None:
806 806 self.fp.close()
807 807
808 808 if self.online:
809 809 newFile = self.__setNextFileOnline()
810 810 else:
811 811 newFile = self.__setNextFileOffline()
812 812
813 813 if not(newFile):
814 814 print '[Reading] No more files to read'
815 815 return 0
816 816
817 817 print '[Reading] Setting the file: %s' % self.filename
818 818
819 819 self.__readFirstHeader()
820 820 self.nReadBlocks = 0
821 821 return 1
822 822
823 823 def __waitNewBlock(self):
824 824 """
825 825 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
826 826
827 827 Si el modo de lectura es OffLine siempre retorn 0
828 828 """
829 829 if not self.online:
830 830 return 0
831 831
832 832 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
833 833 return 0
834 834
835 835 currentPointer = self.fp.tell()
836 836
837 837 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
838 838
839 839 for nTries in range( self.nTries ):
840 840
841 841 self.fp.close()
842 842 self.fp = open( self.filename, 'rb' )
843 843 self.fp.seek( currentPointer )
844 844
845 845 self.fileSize = os.path.getsize( self.filename )
846 846 currentSize = self.fileSize - currentPointer
847 847
848 848 if ( currentSize >= neededSize ):
849 849 self.basicHeaderObj.read(self.fp)
850 850 return 1
851 851
852 852 if self.fileSize == self.fileSizeByHeader:
853 853 # self.flagEoF = True
854 854 return 0
855 855
856 856 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
857 857 sleep( self.delay )
858 858
859 859
860 860 return 0
861 861
862 862 def waitDataBlock(self,pointer_location):
863 863
864 864 currentPointer = pointer_location
865 865
866 866 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
867 867
868 868 for nTries in range( self.nTries ):
869 869 self.fp.close()
870 870 self.fp = open( self.filename, 'rb' )
871 871 self.fp.seek( currentPointer )
872 872
873 873 self.fileSize = os.path.getsize( self.filename )
874 874 currentSize = self.fileSize - currentPointer
875 875
876 876 if ( currentSize >= neededSize ):
877 877 return 1
878 878
879 879 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
880 880 sleep( self.delay )
881 881
882 882 return 0
883 883
884 884 def __jumpToLastBlock(self):
885 885
886 886 if not(self.__isFirstTimeOnline):
887 887 return
888 888
889 889 csize = self.fileSize - self.fp.tell()
890 890 blocksize = self.processingHeaderObj.blockSize
891 891
892 892 #salta el primer bloque de datos
893 893 if csize > self.processingHeaderObj.blockSize:
894 894 self.fp.seek(self.fp.tell() + blocksize)
895 895 else:
896 896 return
897 897
898 898 csize = self.fileSize - self.fp.tell()
899 899 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
900 900 while True:
901 901
902 902 if self.fp.tell()<self.fileSize:
903 903 self.fp.seek(self.fp.tell() + neededsize)
904 904 else:
905 905 self.fp.seek(self.fp.tell() - neededsize)
906 906 break
907 907
908 908 # csize = self.fileSize - self.fp.tell()
909 909 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
910 910 # factor = int(csize/neededsize)
911 911 # if factor > 0:
912 912 # self.fp.seek(self.fp.tell() + factor*neededsize)
913 913
914 914 self.flagIsNewFile = 0
915 915 self.__isFirstTimeOnline = 0
916 916
917 917 def __setNewBlock(self):
918 918
919 919 if self.fp == None:
920 920 return 0
921 921
922 922 # if self.online:
923 923 # self.__jumpToLastBlock()
924 924
925 925 if self.flagIsNewFile:
926 926 self.lastUTTime = self.basicHeaderObj.utc
927 927 return 1
928 928
929 929 if self.realtime:
930 930 self.flagDiscontinuousBlock = 1
931 931 if not(self.setNextFile()):
932 932 return 0
933 933 else:
934 934 return 1
935 935
936 936 currentSize = self.fileSize - self.fp.tell()
937 937 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
938 938
939 939 if (currentSize >= neededSize):
940 940 self.basicHeaderObj.read(self.fp)
941 941 self.lastUTTime = self.basicHeaderObj.utc
942 942 return 1
943 943
944 944 if self.__waitNewBlock():
945 945 self.lastUTTime = self.basicHeaderObj.utc
946 946 return 1
947 947
948 948 if not(self.setNextFile()):
949 949 return 0
950 950
951 951 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
952 952 self.lastUTTime = self.basicHeaderObj.utc
953 953
954 954 self.flagDiscontinuousBlock = 0
955 955
956 956 if deltaTime > self.maxTimeStep:
957 957 self.flagDiscontinuousBlock = 1
958 958
959 959 return 1
960 960
961 961 def readNextBlock(self):
962 962
963 963 if not(self.__setNewBlock()):
964 964 return 0
965 965
966 966 if not(self.readBlock()):
967 967 return 0
968 968
969 969 self.getBasicHeader()
970 970
971 971 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
972 972 self.processingHeaderObj.dataBlocksPerFile,
973 973 self.dataOut.datatime.ctime())
974 974 return 1
975 975
976 976 def __readFirstHeader(self):
977 977
978 978 self.basicHeaderObj.read(self.fp)
979 979 self.systemHeaderObj.read(self.fp)
980 980 self.radarControllerHeaderObj.read(self.fp)
981 981 self.processingHeaderObj.read(self.fp)
982 982
983 983 self.firstHeaderSize = self.basicHeaderObj.size
984 984
985 985 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
986 986 if datatype == 0:
987 987 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
988 988 elif datatype == 1:
989 989 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
990 990 elif datatype == 2:
991 991 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
992 992 elif datatype == 3:
993 993 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
994 994 elif datatype == 4:
995 995 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
996 996 elif datatype == 5:
997 997 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
998 998 else:
999 999 raise ValueError, 'Data type was not defined'
1000 1000
1001 1001 self.dtype = datatype_str
1002 1002 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1003 1003 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1004 1004 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1005 1005 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1006 1006 self.getBlockDimension()
1007 1007
1008 1008 def __verifyFile(self, filename, msgFlag=True):
1009 1009 msg = None
1010 1010 try:
1011 1011 fp = open(filename, 'rb')
1012 1012 currentPosition = fp.tell()
1013 1013 except IOError:
1014 1014 traceback.print_exc()
1015 1015 if msgFlag:
1016 1016 print "[Reading] The file %s can't be opened" % (filename)
1017 1017 return False
1018 1018
1019 1019 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1020 1020
1021 1021 if neededSize == 0:
1022 1022 basicHeaderObj = BasicHeader(LOCALTIME)
1023 1023 systemHeaderObj = SystemHeader()
1024 1024 radarControllerHeaderObj = RadarControllerHeader()
1025 1025 processingHeaderObj = ProcessingHeader()
1026 1026
1027 1027 try:
1028 1028 if not( basicHeaderObj.read(fp) ): raise IOError
1029 1029 if not( systemHeaderObj.read(fp) ): raise IOError
1030 1030 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1031 1031 if not( processingHeaderObj.read(fp) ): raise IOError
1032 1032 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1033 1033
1034 1034 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1035 1035
1036 1036 except IOError:
1037 1037 traceback.print_exc()
1038 sys.exit(0)
1038 # sys.exit(0)
1039 1039
1040 1040 if msgFlag:
1041 1041 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1042 1042
1043 1043 fp.close()
1044 1044 return False
1045 1045 else:
1046 1046 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1047 1047
1048 1048 fp.close()
1049 1049 fileSize = os.path.getsize(filename)
1050 1050 currentSize = fileSize - currentPosition
1051 1051 if currentSize < neededSize:
1052 1052 if msgFlag and (msg != None):
1053 1053 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1054 1054 return False
1055 1055
1056 1056 return True
1057 1057
1058 1058 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1059 1059
1060 1060 dateList = []
1061 1061 pathList = []
1062 1062
1063 1063 multi_path = path.split(',')
1064 1064
1065 1065 if not walk:
1066 1066
1067 1067 for single_path in multi_path:
1068 1068
1069 1069 if not os.path.isdir(single_path):
1070 1070 continue
1071 1071
1072 1072 fileList = glob.glob1(single_path, "*"+ext)
1073 1073
1074 1074 for thisFile in fileList:
1075 1075
1076 1076 if not os.path.isfile(os.path.join(single_path, thisFile)):
1077 1077 continue
1078 1078
1079 1079 if not isRadarFile(thisFile):
1080 1080 continue
1081 1081
1082 1082 if not isFileInDateRange(thisFile, startDate, endDate):
1083 1083 continue
1084 1084
1085 1085 thisDate = getDateFromRadarFile(thisFile)
1086 1086
1087 1087 if thisDate in dateList:
1088 1088 continue
1089 1089
1090 1090 dateList.append(thisDate)
1091 1091 pathList.append(single_path)
1092 1092
1093 1093 else:
1094 1094 for single_path in multi_path:
1095 1095
1096 1096 if not os.path.isdir(single_path):
1097 1097 continue
1098 1098
1099 1099 dirList = []
1100 1100
1101 1101 for thisPath in os.listdir(single_path):
1102 1102
1103 1103 if not os.path.isdir(os.path.join(single_path,thisPath)):
1104 1104 continue
1105 1105
1106 1106 if not isRadarFolder(thisPath):
1107 1107 continue
1108 1108
1109 1109 if not isFolderInDateRange(thisPath, startDate, endDate):
1110 1110 continue
1111 1111
1112 1112 dirList.append(thisPath)
1113 1113
1114 1114 if not dirList:
1115 1115 continue
1116 1116
1117 1117 for thisDir in dirList:
1118 1118
1119 1119 datapath = os.path.join(single_path, thisDir, expLabel)
1120 1120 fileList = glob.glob1(datapath, "*"+ext)
1121 1121
1122 1122 if len(fileList) < 1:
1123 1123 continue
1124 1124
1125 1125 thisDate = getDateFromRadarFolder(thisDir)
1126 1126
1127 1127 pathList.append(datapath)
1128 1128 dateList.append(thisDate)
1129 1129
1130 1130 dateList.sort()
1131 1131
1132 1132 if include_path:
1133 1133 return dateList, pathList
1134 1134
1135 1135 return dateList
1136 1136
1137 1137 def setup(self,
1138 1138 path=None,
1139 1139 startDate=None,
1140 1140 endDate=None,
1141 1141 startTime=datetime.time(0,0,0),
1142 1142 endTime=datetime.time(23,59,59),
1143 1143 set=None,
1144 1144 expLabel = "",
1145 1145 ext = None,
1146 1146 online = False,
1147 1147 delay = 60,
1148 1148 walk = True,
1149 1149 getblock = False,
1150 1150 nTxs = 1,
1151 1151 realtime=False):
1152 1152
1153 1153 if path == None:
1154 1154 raise ValueError, "[Reading] The path is not valid"
1155 1155
1156 1156 if ext == None:
1157 1157 ext = self.ext
1158 1158
1159 1159 if online:
1160 1160 print "[Reading] Searching files in online mode..."
1161 1161
1162 1162 for nTries in range( self.nTries ):
1163 1163 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1164 1164
1165 1165 if fullpath:
1166 1166 break
1167 1167
1168 1168 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1169 1169 sleep( self.delay )
1170 1170
1171 1171 if not(fullpath):
1172 1172 print "[Reading] There 'isn't any valid file in %s" % path
1173 return None
1173 return
1174 1174
1175 1175 self.year = year
1176 1176 self.doy = doy
1177 1177 self.set = set - 1
1178 1178 self.path = path
1179 1179 self.foldercounter = foldercounter
1180 1180 last_set = None
1181 1181
1182 1182 else:
1183 1183 print "[Reading] Searching files in offline mode ..."
1184 1184 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1185 1185 startTime=startTime, endTime=endTime,
1186 1186 set=set, expLabel=expLabel, ext=ext,
1187 1187 walk=walk)
1188 1188
1189 1189 if not(pathList):
1190 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1191 # datetime.datetime.combine(startDate,startTime).ctime(),
1192 # datetime.datetime.combine(endDate,endTime).ctime())
1193 #
1194 sys.exit(-1)
1190 print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1191 datetime.datetime.combine(startDate,startTime).ctime(),
1192 datetime.datetime.combine(endDate,endTime).ctime())
1193
1194 # sys.exit(-1)
1195 1195
1196 self.fileIndex = -1
1197 self.pathList = []
1198 self.filenameList = []
1199 return
1196 1200
1197 1201 self.fileIndex = -1
1198 1202 self.pathList = pathList
1199 1203 self.filenameList = filenameList
1200 1204 file_name = os.path.basename(filenameList[-1])
1201 1205 basename, ext = os.path.splitext(file_name)
1202 1206 last_set = int(basename[-3:])
1203 1207
1204 1208 self.online = online
1205 1209 self.realtime = realtime
1206 1210 self.delay = delay
1207 1211 ext = ext.lower()
1208 1212 self.ext = ext
1209 1213 self.getByBlock = getblock
1210 1214 self.nTxs = int(nTxs)
1211 1215
1212 1216 if not(self.setNextFile()):
1213 1217 if (startDate!=None) and (endDate!=None):
1214 1218 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1215 1219 elif startDate != None:
1216 1220 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1217 1221 else:
1218 1222 print "[Reading] No files"
1219
1220 sys.exit(-1)
1223
1224 self.fileIndex = -1
1225 self.pathList = []
1226 self.filenameList = []
1227 return
1221 1228
1222 1229 # self.getBasicHeader()
1223 1230
1224 1231 if last_set != None:
1225 1232 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1226 1233 return
1227 1234
1228 1235 def getBasicHeader(self):
1229 1236
1230 1237 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1231 1238
1232 1239 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1233 1240
1234 1241 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1235 1242
1236 1243 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1237 1244
1238 1245 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1239 1246
1240 1247 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1241 1248
1242 1249 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1243 1250
1244 1251 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1245 1252
1246 1253
1247 1254 def getFirstHeader(self):
1248 1255
1249 1256 raise ValueError, "This method has not been implemented"
1250 1257
1251 1258 def getData(self):
1252 1259
1253 1260 raise ValueError, "This method has not been implemented"
1254 1261
1255 1262 def hasNotDataInBuffer(self):
1256 1263
1257 1264 raise ValueError, "This method has not been implemented"
1258 1265
1259 1266 def readBlock(self):
1260 1267
1261 1268 raise ValueError, "This method has not been implemented"
1262 1269
1263 1270 def isEndProcess(self):
1264 1271
1265 1272 return self.flagNoMoreFiles
1266 1273
1267 1274 def printReadBlocks(self):
1268 1275
1269 1276 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1270 1277
1271 1278 def printTotalBlocks(self):
1272 1279
1273 1280 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1274 1281
1275 1282 def printNumberOfBlock(self):
1276 1283
1277 1284 if self.flagIsNewBlock:
1278 1285 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1279 1286 self.processingHeaderObj.dataBlocksPerFile,
1280 1287 self.dataOut.datatime.ctime())
1281 1288
1282 1289 def printInfo(self):
1283 1290
1284 1291 if self.__printInfo == False:
1285 1292 return
1286 1293
1287 1294 self.basicHeaderObj.printInfo()
1288 1295 self.systemHeaderObj.printInfo()
1289 1296 self.radarControllerHeaderObj.printInfo()
1290 1297 self.processingHeaderObj.printInfo()
1291 1298
1292 1299 self.__printInfo = False
1293 1300
1294 1301
1295 1302 def run(self, **kwargs):
1296 1303
1297 1304 if not(self.isConfig):
1298 1305
1299 1306 # self.dataOut = dataOut
1300 1307 self.setup(**kwargs)
1301 1308 self.isConfig = True
1302 1309
1303 1310 self.getData()
1304 1311
1305 1312 class JRODataWriter(JRODataIO):
1306 1313
1307 1314 """
1308 1315 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1309 1316 de los datos siempre se realiza por bloques.
1310 1317 """
1311 1318
1312 1319 blockIndex = 0
1313 1320
1314 1321 path = None
1315 1322
1316 1323 setFile = None
1317 1324
1318 1325 profilesPerBlock = None
1319 1326
1320 1327 blocksPerFile = None
1321 1328
1322 1329 nWriteBlocks = 0
1323 1330
1324 1331 fileDate = None
1325 1332
1326 1333 def __init__(self, dataOut=None):
1327 1334 raise ValueError, "Not implemented"
1328 1335
1329 1336
1330 1337 def hasAllDataInBuffer(self):
1331 1338 raise ValueError, "Not implemented"
1332 1339
1333 1340
1334 1341 def setBlockDimension(self):
1335 1342 raise ValueError, "Not implemented"
1336 1343
1337 1344
1338 1345 def writeBlock(self):
1339 1346 raise ValueError, "No implemented"
1340 1347
1341 1348
1342 1349 def putData(self):
1343 1350 raise ValueError, "No implemented"
1344 1351
1345 1352
1346 1353 def getProcessFlags(self):
1347 1354
1348 1355 processFlags = 0
1349 1356
1350 1357 dtype_index = get_dtype_index(self.dtype)
1351 1358 procflag_dtype = get_procflag_dtype(dtype_index)
1352 1359
1353 1360 processFlags += procflag_dtype
1354 1361
1355 1362 if self.dataOut.flagDecodeData:
1356 1363 processFlags += PROCFLAG.DECODE_DATA
1357 1364
1358 1365 if self.dataOut.flagDeflipData:
1359 1366 processFlags += PROCFLAG.DEFLIP_DATA
1360 1367
1361 1368 if self.dataOut.code is not None:
1362 1369 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1363 1370
1364 1371 if self.dataOut.nCohInt > 1:
1365 1372 processFlags += PROCFLAG.COHERENT_INTEGRATION
1366 1373
1367 1374 if self.dataOut.type == "Spectra":
1368 1375 if self.dataOut.nIncohInt > 1:
1369 1376 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1370 1377
1371 1378 if self.dataOut.data_dc is not None:
1372 1379 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1373 1380
1374 1381 if self.dataOut.flagShiftFFT:
1375 1382 processFlags += PROCFLAG.SHIFT_FFT_DATA
1376 1383
1377 1384 return processFlags
1378 1385
1379 1386 def setBasicHeader(self):
1380 1387
1381 1388 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1382 1389 self.basicHeaderObj.version = self.versionFile
1383 1390 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1384 1391
1385 1392 utc = numpy.floor(self.dataOut.utctime)
1386 1393 milisecond = (self.dataOut.utctime - utc)* 1000.0
1387 1394
1388 1395 self.basicHeaderObj.utc = utc
1389 1396 self.basicHeaderObj.miliSecond = milisecond
1390 1397 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1391 1398 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1392 1399 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1393 1400
1394 1401 def setFirstHeader(self):
1395 1402 """
1396 1403 Obtiene una copia del First Header
1397 1404
1398 1405 Affected:
1399 1406
1400 1407 self.basicHeaderObj
1401 1408 self.systemHeaderObj
1402 1409 self.radarControllerHeaderObj
1403 1410 self.processingHeaderObj self.
1404 1411
1405 1412 Return:
1406 1413 None
1407 1414 """
1408 1415
1409 1416 raise ValueError, "No implemented"
1410 1417
1411 1418 def __writeFirstHeader(self):
1412 1419 """
1413 1420 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1414 1421
1415 1422 Affected:
1416 1423 __dataType
1417 1424
1418 1425 Return:
1419 1426 None
1420 1427 """
1421 1428
1422 1429 # CALCULAR PARAMETROS
1423 1430
1424 1431 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1425 1432 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1426 1433
1427 1434 self.basicHeaderObj.write(self.fp)
1428 1435 self.systemHeaderObj.write(self.fp)
1429 1436 self.radarControllerHeaderObj.write(self.fp)
1430 1437 self.processingHeaderObj.write(self.fp)
1431 1438
1432 1439 def __setNewBlock(self):
1433 1440 """
1434 1441 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1435 1442
1436 1443 Return:
1437 1444 0 : si no pudo escribir nada
1438 1445 1 : Si escribio el Basic el First Header
1439 1446 """
1440 1447 if self.fp == None:
1441 1448 self.setNextFile()
1442 1449
1443 1450 if self.flagIsNewFile:
1444 1451 return 1
1445 1452
1446 1453 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1447 1454 self.basicHeaderObj.write(self.fp)
1448 1455 return 1
1449 1456
1450 1457 if not( self.setNextFile() ):
1451 1458 return 0
1452 1459
1453 1460 return 1
1454 1461
1455 1462
1456 1463 def writeNextBlock(self):
1457 1464 """
1458 1465 Selecciona el bloque siguiente de datos y los escribe en un file
1459 1466
1460 1467 Return:
1461 1468 0 : Si no hizo pudo escribir el bloque de datos
1462 1469 1 : Si no pudo escribir el bloque de datos
1463 1470 """
1464 1471 if not( self.__setNewBlock() ):
1465 1472 return 0
1466 1473
1467 1474 self.writeBlock()
1468 1475
1469 1476 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1470 1477 self.processingHeaderObj.dataBlocksPerFile)
1471 1478
1472 1479 return 1
1473 1480
1474 1481 def setNextFile(self):
1475 1482 """
1476 1483 Determina el siguiente file que sera escrito
1477 1484
1478 1485 Affected:
1479 1486 self.filename
1480 1487 self.subfolder
1481 1488 self.fp
1482 1489 self.setFile
1483 1490 self.flagIsNewFile
1484 1491
1485 1492 Return:
1486 1493 0 : Si el archivo no puede ser escrito
1487 1494 1 : Si el archivo esta listo para ser escrito
1488 1495 """
1489 1496 ext = self.ext
1490 1497 path = self.path
1491 1498
1492 1499 if self.fp != None:
1493 1500 self.fp.close()
1494 1501
1495 1502 timeTuple = time.localtime( self.dataOut.utctime)
1496 1503 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1497 1504
1498 1505 fullpath = os.path.join( path, subfolder )
1499 1506 setFile = self.setFile
1500 1507
1501 1508 if not( os.path.exists(fullpath) ):
1502 1509 os.mkdir(fullpath)
1503 1510 setFile = -1 #inicializo mi contador de seteo
1504 1511 else:
1505 1512 filesList = os.listdir( fullpath )
1506 1513 if len( filesList ) > 0:
1507 1514 filesList = sorted( filesList, key=str.lower )
1508 1515 filen = filesList[-1]
1509 1516 # el filename debera tener el siguiente formato
1510 1517 # 0 1234 567 89A BCDE (hex)
1511 1518 # x YYYY DDD SSS .ext
1512 1519 if isNumber( filen[8:11] ):
1513 1520 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1514 1521 else:
1515 1522 setFile = -1
1516 1523 else:
1517 1524 setFile = -1 #inicializo mi contador de seteo
1518 1525
1519 1526 setFile += 1
1520 1527
1521 1528 #If this is a new day it resets some values
1522 1529 if self.dataOut.datatime.date() > self.fileDate:
1523 1530 setFile = 0
1524 1531 self.nTotalBlocks = 0
1525 1532
1526 1533 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1527 1534
1528 1535 filename = os.path.join( path, subfolder, filen )
1529 1536
1530 1537 fp = open( filename,'wb' )
1531 1538
1532 1539 self.blockIndex = 0
1533 1540
1534 1541 #guardando atributos
1535 1542 self.filename = filename
1536 1543 self.subfolder = subfolder
1537 1544 self.fp = fp
1538 1545 self.setFile = setFile
1539 1546 self.flagIsNewFile = 1
1540 1547 self.fileDate = self.dataOut.datatime.date()
1541 1548
1542 1549 self.setFirstHeader()
1543 1550
1544 1551 print '[Writing] Opening file: %s'%self.filename
1545 1552
1546 1553 self.__writeFirstHeader()
1547 1554
1548 1555 return 1
1549 1556
1550 1557 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1551 1558 """
1552 1559 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1553 1560
1554 1561 Inputs:
1555 1562 path : directory where data will be saved
1556 1563 profilesPerBlock : number of profiles per block
1557 1564 set : initial file set
1558 1565 datatype : An integer number that defines data type:
1559 1566 0 : int8 (1 byte)
1560 1567 1 : int16 (2 bytes)
1561 1568 2 : int32 (4 bytes)
1562 1569 3 : int64 (8 bytes)
1563 1570 4 : float32 (4 bytes)
1564 1571 5 : double64 (8 bytes)
1565 1572
1566 1573 Return:
1567 1574 0 : Si no realizo un buen seteo
1568 1575 1 : Si realizo un buen seteo
1569 1576 """
1570 1577
1571 1578 if ext == None:
1572 1579 ext = self.ext
1573 1580
1574 1581 self.ext = ext.lower()
1575 1582
1576 1583 self.path = path
1577 1584
1578 1585 if set is None:
1579 1586 self.setFile = -1
1580 1587 else:
1581 1588 self.setFile = set - 1
1582 1589
1583 1590 self.blocksPerFile = blocksPerFile
1584 1591
1585 1592 self.profilesPerBlock = profilesPerBlock
1586 1593
1587 1594 self.dataOut = dataOut
1588 1595 self.fileDate = self.dataOut.datatime.date()
1589 1596 #By default
1590 1597 self.dtype = self.dataOut.dtype
1591 1598
1592 1599 if datatype is not None:
1593 1600 self.dtype = get_numpy_dtype(datatype)
1594 1601
1595 1602 if not(self.setNextFile()):
1596 1603 print "[Writing] There isn't a next file"
1597 1604 return 0
1598 1605
1599 1606 self.setBlockDimension()
1600 1607
1601 1608 return 1
1602 1609
1603 1610 def run(self, dataOut, **kwargs):
1604 1611
1605 1612 if not(self.isConfig):
1606 1613
1607 1614 self.setup(dataOut, **kwargs)
1608 1615 self.isConfig = True
1609 1616
1610 1617 self.putData()
1611 1618
General Comments 0
You need to be logged in to leave comments. Login now