##// END OF EJS Templates
schain-jc
RSS
jroIO_base.py: Compare dates when startTime > endTime
Miguel Valdez -
r652:303180c98d9d
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1585 +1,1596
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 traceback.print_exc()
72 72 raise IOError, "The file %s can't be opened" %(filename)
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 def isFileInTimeRange(filename, startTime, endTime):
86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
87 87 """
88 88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89 89
90 90 Inputs:
91 91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92 92
93 startDate : fecha inicial del rango seleccionado en formato datetime.date
94
95 endDate : fecha final del rango seleccionado en formato datetime.date
96
93 97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
94 98
95 99 endTime : tiempo final del rango seleccionado en formato datetime.time
96 100
97 101 Return:
98 102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
99 103 fecha especificado, de lo contrario retorna False.
100 104
101 105 Excepciones:
102 106 Si el archivo no existe o no puede ser abierto
103 107 Si la cabecera no puede ser leida.
104 108
105 109 """
106 110
107 111
108 112 try:
109 113 fp = open(filename,'rb')
110 114 except IOError:
111 115 traceback.print_exc()
112 116 raise IOError, "The file %s can't be opened" %(filename)
113 117
114 118 basicHeaderObj = BasicHeader(LOCALTIME)
115 119 sts = basicHeaderObj.read(fp)
116 120 fp.close()
117 121
118 122 thisDatetime = basicHeaderObj.datatime
123 thisDate = thisDatetime.date()
119 124 thisTime = thisDatetime.time()
120 125
121 126 if not(sts):
122 127 print "Skipping the file %s because it has not a valid header" %(filename)
123 128 return None
124 129
125 130 #General case
126 131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
127 132 #-----------o----------------------------o-----------
128 133 # startTime endTime
129 134
130 135 if endTime >= startTime:
131 136 if (thisTime < startTime) or (thisTime > endTime):
132 137 return None
133 138
134 139 return thisDatetime
135 140
136 141 #If endTime < startTime then endTime belongs to the next day
137 142
138 143
139 144 #<<<<<<<<<<<o o>>>>>>>>>>>
140 145 #-----------o----------------------------o-----------
141 146 # endTime startTime
142 147
148 if (thisDate == startDate) and (thisTime < startTime):
149 return None
150
151 if (thisDate == endDate) and (thisTime > endTime):
152 return None
153
143 154 if (thisTime < startTime) and (thisTime > endTime):
144 155 return None
145 156
146 157 return thisDatetime
147 158
148 159 def isFolderInDateRange(folder, startDate=None, endDate=None):
149 160 """
150 161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
151 162
152 163 Inputs:
153 164 folder : nombre completo del directorio.
154 165 Su formato deberia ser "/path_root/?YYYYDDD"
155 166
156 167 siendo:
157 168 YYYY : Anio (ejemplo 2015)
158 169 DDD : Dia del anio (ejemplo 305)
159 170
160 171 startDate : fecha inicial del rango seleccionado en formato datetime.date
161 172
162 173 endDate : fecha final del rango seleccionado en formato datetime.date
163 174
164 175 Return:
165 176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
166 177 fecha especificado, de lo contrario retorna False.
167 178 Excepciones:
168 179 Si el directorio no tiene el formato adecuado
169 180 """
170 181
171 182 basename = os.path.basename(folder)
172 183
173 184 if not isRadarFolder(basename):
174 185 raise IOError, "The folder %s has not the rigth format" %folder
175 186
176 187 if startDate and endDate:
177 188 thisDate = getDateFromRadarFolder(basename)
178 189
179 190 if thisDate < startDate:
180 191 return 0
181 192
182 193 if thisDate > endDate:
183 194 return 0
184 195
185 196 return 1
186 197
187 198 def isFileInDateRange(filename, startDate=None, endDate=None):
188 199 """
189 200 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
190 201
191 202 Inputs:
192 203 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
193 204
194 205 Su formato deberia ser "?YYYYDDDsss"
195 206
196 207 siendo:
197 208 YYYY : Anio (ejemplo 2015)
198 209 DDD : Dia del anio (ejemplo 305)
199 210 sss : set
200 211
201 212 startDate : fecha inicial del rango seleccionado en formato datetime.date
202 213
203 214 endDate : fecha final del rango seleccionado en formato datetime.date
204 215
205 216 Return:
206 217 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
207 218 fecha especificado, de lo contrario retorna False.
208 219 Excepciones:
209 220 Si el archivo no tiene el formato adecuado
210 221 """
211 222
212 223 basename = os.path.basename(filename)
213 224
214 225 if not isRadarFile(basename):
215 226 raise IOError, "The filename %s has not the rigth format" %filename
216 227
217 228 if startDate and endDate:
218 229 thisDate = getDateFromRadarFile(basename)
219 230
220 231 if thisDate < startDate:
221 232 return 0
222 233
223 234 if thisDate > endDate:
224 235 return 0
225 236
226 237 return 1
227 238
228 239 def getFileFromSet(path, ext, set):
229 240 validFilelist = []
230 241 fileList = os.listdir(path)
231 242
232 243 # 0 1234 567 89A BCDE
233 244 # H YYYY DDD SSS .ext
234 245
235 246 for thisFile in fileList:
236 247 try:
237 248 year = int(thisFile[1:5])
238 249 doy = int(thisFile[5:8])
239 250 except:
240 251 continue
241 252
242 253 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
243 254 continue
244 255
245 256 validFilelist.append(thisFile)
246 257
247 258 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
248 259
249 260 if len(myfile)!= 0:
250 261 return myfile[0]
251 262 else:
252 263 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
253 264 print 'the filename %s does not exist'%filename
254 265 print '...going to the last file: '
255 266
256 267 if validFilelist:
257 268 validFilelist = sorted( validFilelist, key=str.lower )
258 269 return validFilelist[-1]
259 270
260 271 return None
261 272
262 273 def getlastFileFromPath(path, ext):
263 274 """
264 275 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
265 276 al final de la depuracion devuelve el ultimo file de la lista que quedo.
266 277
267 278 Input:
268 279 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
269 280 ext : extension de los files contenidos en una carpeta
270 281
271 282 Return:
272 283 El ultimo file de una determinada carpeta, no se considera el path.
273 284 """
274 285 validFilelist = []
275 286 fileList = os.listdir(path)
276 287
277 288 # 0 1234 567 89A BCDE
278 289 # H YYYY DDD SSS .ext
279 290
280 291 for thisFile in fileList:
281 292
282 293 year = thisFile[1:5]
283 294 if not isNumber(year):
284 295 continue
285 296
286 297 doy = thisFile[5:8]
287 298 if not isNumber(doy):
288 299 continue
289 300
290 301 year = int(year)
291 302 doy = int(doy)
292 303
293 304 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
294 305 continue
295 306
296 307 validFilelist.append(thisFile)
297 308
298 309 if validFilelist:
299 310 validFilelist = sorted( validFilelist, key=str.lower )
300 311 return validFilelist[-1]
301 312
302 313 return None
303 314
304 315 def checkForRealPath(path, foldercounter, year, doy, set, ext):
305 316 """
306 317 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
307 318 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
308 319 el path exacto de un determinado file.
309 320
310 321 Example :
311 322 nombre correcto del file es .../.../D2009307/P2009307367.ext
312 323
313 324 Entonces la funcion prueba con las siguientes combinaciones
314 325 .../.../y2009307367.ext
315 326 .../.../Y2009307367.ext
316 327 .../.../x2009307/y2009307367.ext
317 328 .../.../x2009307/Y2009307367.ext
318 329 .../.../X2009307/y2009307367.ext
319 330 .../.../X2009307/Y2009307367.ext
320 331 siendo para este caso, la ultima combinacion de letras, identica al file buscado
321 332
322 333 Return:
323 334 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
324 335 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
325 336 para el filename
326 337 """
327 338 fullfilename = None
328 339 find_flag = False
329 340 filename = None
330 341
331 342 prefixDirList = [None,'d','D']
332 343 if ext.lower() == ".r": #voltage
333 344 prefixFileList = ['d','D']
334 345 elif ext.lower() == ".pdata": #spectra
335 346 prefixFileList = ['p','P']
336 347 else:
337 348 return None, filename
338 349
339 350 #barrido por las combinaciones posibles
340 351 for prefixDir in prefixDirList:
341 352 thispath = path
342 353 if prefixDir != None:
343 354 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
344 355 if foldercounter == 0:
345 356 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
346 357 else:
347 358 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
348 359 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
349 360 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
350 361 fullfilename = os.path.join( thispath, filename ) #formo el path completo
351 362
352 363 if os.path.exists( fullfilename ): #verifico que exista
353 364 find_flag = True
354 365 break
355 366 if find_flag:
356 367 break
357 368
358 369 if not(find_flag):
359 370 return None, filename
360 371
361 372 return fullfilename, filename
362 373
363 374 def isRadarFolder(folder):
364 375 try:
365 376 year = int(folder[1:5])
366 377 doy = int(folder[5:8])
367 378 except:
368 379 return 0
369 380
370 381 return 1
371 382
372 383 def isRadarFile(file):
373 384 try:
374 385 year = int(file[1:5])
375 386 doy = int(file[5:8])
376 387 set = int(file[8:11])
377 388 except:
378 389 return 0
379 390
380 391 return 1
381 392
382 393 def getDateFromRadarFile(file):
383 394 try:
384 395 year = int(file[1:5])
385 396 doy = int(file[5:8])
386 397 set = int(file[8:11])
387 398 except:
388 399 return None
389 400
390 401 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
391 402 return thisDate
392 403
393 404 def getDateFromRadarFolder(folder):
394 405 try:
395 406 year = int(folder[1:5])
396 407 doy = int(folder[5:8])
397 408 except:
398 409 return None
399 410
400 411 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
401 412 return thisDate
402 413
403 414 class JRODataIO:
404 415
405 416 c = 3E8
406 417
407 418 isConfig = False
408 419
409 420 basicHeaderObj = None
410 421
411 422 systemHeaderObj = None
412 423
413 424 radarControllerHeaderObj = None
414 425
415 426 processingHeaderObj = None
416 427
417 428 online = 0
418 429
419 430 dtype = None
420 431
421 432 pathList = []
422 433
423 434 filenameList = []
424 435
425 436 filename = None
426 437
427 438 ext = None
428 439
429 440 flagIsNewFile = 1
430 441
431 442 flagDiscontinuousBlock = 0
432 443
433 444 flagIsNewBlock = 0
434 445
435 446 fp = None
436 447
437 448 firstHeaderSize = 0
438 449
439 450 basicHeaderSize = 24
440 451
441 452 versionFile = 1103
442 453
443 454 fileSize = None
444 455
445 456 # ippSeconds = None
446 457
447 458 fileSizeByHeader = None
448 459
449 460 fileIndex = None
450 461
451 462 profileIndex = None
452 463
453 464 blockIndex = None
454 465
455 466 nTotalBlocks = None
456 467
457 468 maxTimeStep = 30
458 469
459 470 lastUTTime = None
460 471
461 472 datablock = None
462 473
463 474 dataOut = None
464 475
465 476 blocksize = None
466 477
467 478 getByBlock = False
468 479
469 480 def __init__(self):
470 481
471 482 raise ValueError, "Not implemented"
472 483
473 484 def run(self):
474 485
475 486 raise ValueError, "Not implemented"
476 487
477 488 def getDtypeWidth(self):
478 489
479 490 dtype_index = get_dtype_index(self.dtype)
480 491 dtype_width = get_dtype_width(dtype_index)
481 492
482 493 return dtype_width
483 494
484 495 class JRODataReader(JRODataIO):
485 496
486 497 nReadBlocks = 0
487 498
488 499 delay = 10 #number of seconds waiting a new file
489 500
490 501 nTries = 3 #quantity tries
491 502
492 503 nFiles = 3 #number of files for searching
493 504
494 505 path = None
495 506
496 507 foldercounter = 0
497 508
498 509 flagNoMoreFiles = 0
499 510
500 511 datetimeList = []
501 512
502 513 __isFirstTimeOnline = 1
503 514
504 515 __printInfo = True
505 516
506 517 profileIndex = None
507 518
508 519 nTxs = 1
509 520
510 521 txIndex = None
511 522
512 523 def __init__(self):
513 524
514 525 """
515 526
516 527 """
517 528
518 529 # raise NotImplementedError, "This method has not been implemented"
519 530
520 531
521 532 def createObjByDefault(self):
522 533 """
523 534
524 535 """
525 536 raise NotImplementedError, "This method has not been implemented"
526 537
527 538 def getBlockDimension(self):
528 539
529 540 raise NotImplementedError, "No implemented"
530 541
531 542 def __searchFilesOffLine(self,
532 543 path,
533 544 startDate=None,
534 545 endDate=None,
535 546 startTime=datetime.time(0,0,0),
536 547 endTime=datetime.time(23,59,59),
537 548 set=None,
538 549 expLabel='',
539 550 ext='.r',
540 551 walk=True):
541 552
542 553 self.filenameList = []
543 554 self.datetimeList = []
544 555
545 556 pathList = []
546 557
547 558 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
548 559
549 560 if dateList == []:
550 561 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
551 562 datetime.datetime.combine(startDate,startTime).ctime(),
552 563 datetime.datetime.combine(endDate,endTime).ctime())
553 564
554 565 return None, None
555 566
556 567 if len(dateList) > 1:
557 print "[Reading] %d dates with data were found for the date range: %s - %s" %(len(dateList), startDate, endDate)
568 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
558 569 else:
559 570 print "[Reading] data was found for the date %s" %(dateList[0])
560 571
561 572 filenameList = []
562 573 datetimeList = []
563 574
564 575 for thisPath in pathList:
565 576 # thisPath = pathList[pathDict[file]]
566 577
567 578 fileList = glob.glob1(thisPath, "*%s" %ext)
568 579 fileList.sort()
569 580
570 581 for file in fileList:
571 582
572 583 filename = os.path.join(thisPath,file)
573 584
574 585 if not isFileInDateRange(filename, startDate, endDate):
575 586 continue
576 587
577 thisDatetime = isFileInTimeRange(filename, startTime, endTime)
588 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
578 589
579 590 if not(thisDatetime):
580 591 continue
581 592
582 593 filenameList.append(filename)
583 594 datetimeList.append(thisDatetime)
584 595
585 596 if not(filenameList):
586 597 print "[Reading] Any file was found int time range %s - %s" %(startTime.ctime(), endTime.ctime())
587 598 return None, None
588 599
589 600 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
590 601 print
591 602
592 603 for i in range(len(filenameList)):
593 604 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
594 605
595 606 self.filenameList = filenameList
596 607 self.datetimeList = datetimeList
597 608
598 609 return pathList, filenameList
599 610
600 611 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
601 612
602 613 """
603 614 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
604 615 devuelve el archivo encontrado ademas de otros datos.
605 616
606 617 Input:
607 618 path : carpeta donde estan contenidos los files que contiene data
608 619
609 620 expLabel : Nombre del subexperimento (subfolder)
610 621
611 622 ext : extension de los files
612 623
613 624 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
614 625
615 626 Return:
616 627 directory : eL directorio donde esta el file encontrado
617 628 filename : el ultimo file de una determinada carpeta
618 629 year : el anho
619 630 doy : el numero de dia del anho
620 631 set : el set del archivo
621 632
622 633
623 634 """
624 635 dirList = []
625 636
626 637 if not walk:
627 638 fullpath = path
628 639 foldercounter = 0
629 640 else:
630 641 #Filtra solo los directorios
631 642 for thisPath in os.listdir(path):
632 643 if not os.path.isdir(os.path.join(path,thisPath)):
633 644 continue
634 645 if not isRadarFolder(thisPath):
635 646 continue
636 647
637 648 dirList.append(thisPath)
638 649
639 650 if not(dirList):
640 651 return None, None, None, None, None, None
641 652
642 653 dirList = sorted( dirList, key=str.lower )
643 654
644 655 doypath = dirList[-1]
645 656 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
646 657 fullpath = os.path.join(path, doypath, expLabel)
647 658
648 659
649 660 print "[Reading] %s folder was found: " %(fullpath )
650 661
651 662 if set == None:
652 663 filename = getlastFileFromPath(fullpath, ext)
653 664 else:
654 665 filename = getFileFromSet(fullpath, ext, set)
655 666
656 667 if not(filename):
657 668 return None, None, None, None, None, None
658 669
659 670 print "[Reading] %s file was found" %(filename)
660 671
661 672 if not(self.__verifyFile(os.path.join(fullpath, filename))):
662 673 return None, None, None, None, None, None
663 674
664 675 year = int( filename[1:5] )
665 676 doy = int( filename[5:8] )
666 677 set = int( filename[8:11] )
667 678
668 679 return fullpath, foldercounter, filename, year, doy, set
669 680
670 681 def __setNextFileOffline(self):
671 682
672 683 idFile = self.fileIndex
673 684
674 685 while (True):
675 686 idFile += 1
676 687 if not(idFile < len(self.filenameList)):
677 688 self.flagNoMoreFiles = 1
678 689 # print "[Reading] No more Files"
679 690 return 0
680 691
681 692 filename = self.filenameList[idFile]
682 693
683 694 if not(self.__verifyFile(filename)):
684 695 continue
685 696
686 697 fileSize = os.path.getsize(filename)
687 698 fp = open(filename,'rb')
688 699 break
689 700
690 701 self.flagIsNewFile = 1
691 702 self.fileIndex = idFile
692 703 self.filename = filename
693 704 self.fileSize = fileSize
694 705 self.fp = fp
695 706
696 707 # print "[Reading] Setting the file: %s"%self.filename
697 708
698 709 return 1
699 710
700 711 def __setNextFileOnline(self):
701 712 """
702 713 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
703 714 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
704 715 siguientes.
705 716
706 717 Affected:
707 718 self.flagIsNewFile
708 719 self.filename
709 720 self.fileSize
710 721 self.fp
711 722 self.set
712 723 self.flagNoMoreFiles
713 724
714 725 Return:
715 726 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
716 727 1 : si el file fue abierto con exito y esta listo a ser leido
717 728
718 729 Excepciones:
719 730 Si un determinado file no puede ser abierto
720 731 """
721 732 nFiles = 0
722 733 fileOk_flag = False
723 734 firstTime_flag = True
724 735
725 736 self.set += 1
726 737
727 738 if self.set > 999:
728 739 self.set = 0
729 740 self.foldercounter += 1
730 741
731 742 #busca el 1er file disponible
732 743 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
733 744 if fullfilename:
734 745 if self.__verifyFile(fullfilename, False):
735 746 fileOk_flag = True
736 747
737 748 #si no encuentra un file entonces espera y vuelve a buscar
738 749 if not(fileOk_flag):
739 750 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
740 751
741 752 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
742 753 tries = self.nTries
743 754 else:
744 755 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
745 756
746 757 for nTries in range( tries ):
747 758 if firstTime_flag:
748 759 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
749 760 sleep( self.delay )
750 761 else:
751 762 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
752 763
753 764 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
754 765 if fullfilename:
755 766 if self.__verifyFile(fullfilename):
756 767 fileOk_flag = True
757 768 break
758 769
759 770 if fileOk_flag:
760 771 break
761 772
762 773 firstTime_flag = False
763 774
764 775 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
765 776 self.set += 1
766 777
767 778 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
768 779 self.set = 0
769 780 self.doy += 1
770 781 self.foldercounter = 0
771 782
772 783 if fileOk_flag:
773 784 self.fileSize = os.path.getsize( fullfilename )
774 785 self.filename = fullfilename
775 786 self.flagIsNewFile = 1
776 787 if self.fp != None: self.fp.close()
777 788 self.fp = open(fullfilename, 'rb')
778 789 self.flagNoMoreFiles = 0
779 790 # print '[Reading] Setting the file: %s' % fullfilename
780 791 else:
781 792 self.fileSize = 0
782 793 self.filename = None
783 794 self.flagIsNewFile = 0
784 795 self.fp = None
785 796 self.flagNoMoreFiles = 1
786 797 # print '[Reading] No more files to read'
787 798
788 799 return fileOk_flag
789 800
790 801 def setNextFile(self):
791 802 if self.fp != None:
792 803 self.fp.close()
793 804
794 805 if self.online:
795 806 newFile = self.__setNextFileOnline()
796 807 else:
797 808 newFile = self.__setNextFileOffline()
798 809
799 810 if not(newFile):
800 811 print '[Reading] No more files to read'
801 812 return 0
802 813
803 814 print '[Reading] Setting the file: %s' % self.filename
804 815
805 816 self.__readFirstHeader()
806 817 self.nReadBlocks = 0
807 818 return 1
808 819
809 820 def __waitNewBlock(self):
810 821 """
811 822 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
812 823
813 824 Si el modo de lectura es OffLine siempre retorn 0
814 825 """
815 826 if not self.online:
816 827 return 0
817 828
818 829 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
819 830 return 0
820 831
821 832 currentPointer = self.fp.tell()
822 833
823 834 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
824 835
825 836 for nTries in range( self.nTries ):
826 837
827 838 self.fp.close()
828 839 self.fp = open( self.filename, 'rb' )
829 840 self.fp.seek( currentPointer )
830 841
831 842 self.fileSize = os.path.getsize( self.filename )
832 843 currentSize = self.fileSize - currentPointer
833 844
834 845 if ( currentSize >= neededSize ):
835 846 self.basicHeaderObj.read(self.fp)
836 847 return 1
837 848
838 849 if self.fileSize == self.fileSizeByHeader:
839 850 # self.flagEoF = True
840 851 return 0
841 852
842 853 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
843 854 sleep( self.delay )
844 855
845 856
846 857 return 0
847 858
848 859 def waitDataBlock(self,pointer_location):
849 860
850 861 currentPointer = pointer_location
851 862
852 863 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
853 864
854 865 for nTries in range( self.nTries ):
855 866 self.fp.close()
856 867 self.fp = open( self.filename, 'rb' )
857 868 self.fp.seek( currentPointer )
858 869
859 870 self.fileSize = os.path.getsize( self.filename )
860 871 currentSize = self.fileSize - currentPointer
861 872
862 873 if ( currentSize >= neededSize ):
863 874 return 1
864 875
865 876 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
866 877 sleep( self.delay )
867 878
868 879 return 0
869 880
870 881 def __jumpToLastBlock(self):
871 882
872 883 if not(self.__isFirstTimeOnline):
873 884 return
874 885
875 886 csize = self.fileSize - self.fp.tell()
876 887 blocksize = self.processingHeaderObj.blockSize
877 888
878 889 #salta el primer bloque de datos
879 890 if csize > self.processingHeaderObj.blockSize:
880 891 self.fp.seek(self.fp.tell() + blocksize)
881 892 else:
882 893 return
883 894
884 895 csize = self.fileSize - self.fp.tell()
885 896 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
886 897 while True:
887 898
888 899 if self.fp.tell()<self.fileSize:
889 900 self.fp.seek(self.fp.tell() + neededsize)
890 901 else:
891 902 self.fp.seek(self.fp.tell() - neededsize)
892 903 break
893 904
894 905 # csize = self.fileSize - self.fp.tell()
895 906 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
896 907 # factor = int(csize/neededsize)
897 908 # if factor > 0:
898 909 # self.fp.seek(self.fp.tell() + factor*neededsize)
899 910
900 911 self.flagIsNewFile = 0
901 912 self.__isFirstTimeOnline = 0
902 913
903 914 def __setNewBlock(self):
904 915
905 916 if self.fp == None:
906 917 return 0
907 918
908 919 if self.online:
909 920 self.__jumpToLastBlock()
910 921
911 922 if self.flagIsNewFile:
912 923 return 1
913 924
914 925 self.lastUTTime = self.basicHeaderObj.utc
915 926 currentSize = self.fileSize - self.fp.tell()
916 927 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
917 928
918 929 if (currentSize >= neededSize):
919 930 self.basicHeaderObj.read(self.fp)
920 931 return 1
921 932
922 933 if self.__waitNewBlock():
923 934 return 1
924 935
925 936 if not(self.setNextFile()):
926 937 return 0
927 938
928 939 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
929 940
930 941 self.flagDiscontinuousBlock = 0
931 942
932 943 if deltaTime > self.maxTimeStep:
933 944 self.flagDiscontinuousBlock = 1
934 945
935 946 return 1
936 947
937 948 def readNextBlock(self):
938 949
939 950 if not(self.__setNewBlock()):
940 951 return 0
941 952
942 953 if not(self.readBlock()):
943 954 return 0
944 955
945 956 self.getBasicHeader()
946 957
947 958 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
948 959 self.processingHeaderObj.dataBlocksPerFile,
949 960 self.dataOut.datatime.ctime())
950 961 return 1
951 962
952 963 def __readFirstHeader(self):
953 964
954 965 self.basicHeaderObj.read(self.fp)
955 966 self.systemHeaderObj.read(self.fp)
956 967 self.radarControllerHeaderObj.read(self.fp)
957 968 self.processingHeaderObj.read(self.fp)
958 969
959 970 self.firstHeaderSize = self.basicHeaderObj.size
960 971
961 972 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
962 973 if datatype == 0:
963 974 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
964 975 elif datatype == 1:
965 976 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
966 977 elif datatype == 2:
967 978 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
968 979 elif datatype == 3:
969 980 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
970 981 elif datatype == 4:
971 982 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
972 983 elif datatype == 5:
973 984 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
974 985 else:
975 986 raise ValueError, 'Data type was not defined'
976 987
977 988 self.dtype = datatype_str
978 989 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
979 990 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
980 991 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
981 992 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
982 993 self.getBlockDimension()
983 994
984 995 def __verifyFile(self, filename, msgFlag=True):
985 996 msg = None
986 997 try:
987 998 fp = open(filename, 'rb')
988 999 currentPosition = fp.tell()
989 1000 except IOError:
990 1001 traceback.print_exc()
991 1002 if msgFlag:
992 1003 print "[Reading] The file %s can't be opened" % (filename)
993 1004 return False
994 1005
995 1006 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
996 1007
997 1008 if neededSize == 0:
998 1009 basicHeaderObj = BasicHeader(LOCALTIME)
999 1010 systemHeaderObj = SystemHeader()
1000 1011 radarControllerHeaderObj = RadarControllerHeader()
1001 1012 processingHeaderObj = ProcessingHeader()
1002 1013
1003 1014 try:
1004 1015 if not( basicHeaderObj.read(fp) ): raise IOError
1005 1016 if not( systemHeaderObj.read(fp) ): raise IOError
1006 1017 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1007 1018 if not( processingHeaderObj.read(fp) ): raise IOError
1008 1019 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1009 1020
1010 1021 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1011 1022
1012 1023 except IOError:
1013 1024 traceback.print_exc()
1014 1025 sys.exit(0)
1015 1026
1016 1027 if msgFlag:
1017 1028 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1018 1029
1019 1030 fp.close()
1020 1031 return False
1021 1032 else:
1022 1033 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1023 1034
1024 1035 fp.close()
1025 1036 fileSize = os.path.getsize(filename)
1026 1037 currentSize = fileSize - currentPosition
1027 1038 if currentSize < neededSize:
1028 1039 if msgFlag and (msg != None):
1029 1040 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1030 1041 return False
1031 1042
1032 1043 return True
1033 1044
1034 1045 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1035 1046
1036 1047 dateList = []
1037 1048 pathList = []
1038 1049
1039 1050 multi_path = path.split(',')
1040 1051
1041 1052 if not walk:
1042 1053
1043 1054 for single_path in multi_path:
1044 1055
1045 1056 if not os.path.isdir(single_path):
1046 1057 continue
1047 1058
1048 1059 fileList = glob.glob1(single_path, "*"+ext)
1049 1060
1050 1061 for thisFile in fileList:
1051 1062
1052 1063 if not os.path.isfile(os.path.join(single_path, thisFile)):
1053 1064 continue
1054 1065
1055 1066 if not isRadarFile(thisFile):
1056 1067 continue
1057 1068
1058 1069 if not isFileInDateRange(thisFile, startDate, endDate):
1059 1070 continue
1060 1071
1061 1072 thisDate = getDateFromRadarFile(thisFile)
1062 1073
1063 1074 if thisDate in dateList:
1064 1075 continue
1065 1076
1066 1077 dateList.append(thisDate)
1067 1078 pathList.append(single_path)
1068 1079
1069 1080 else:
1070 1081 for single_path in multi_path:
1071 1082
1072 1083 if not os.path.isdir(single_path):
1073 1084 continue
1074 1085
1075 1086 dirList = []
1076 1087
1077 1088 for thisPath in os.listdir(single_path):
1078 1089
1079 1090 if not os.path.isdir(os.path.join(single_path,thisPath)):
1080 1091 continue
1081 1092
1082 1093 if not isRadarFolder(thisPath):
1083 1094 continue
1084 1095
1085 1096 if not isFolderInDateRange(thisPath, startDate, endDate):
1086 1097 continue
1087 1098
1088 1099 dirList.append(thisPath)
1089 1100
1090 1101 if not dirList:
1091 1102 continue
1092 1103
1093 1104 for thisDir in dirList:
1094 1105
1095 1106 datapath = os.path.join(single_path, thisDir, expLabel)
1096 1107 fileList = glob.glob1(datapath, "*"+ext)
1097 1108
1098 1109 if len(fileList) < 1:
1099 1110 continue
1100 1111
1101 1112 thisDate = getDateFromRadarFolder(thisDir)
1102 1113
1103 1114 pathList.append(datapath)
1104 1115 dateList.append(thisDate)
1105 1116
1106 1117 dateList.sort()
1107 1118
1108 1119 if include_path:
1109 1120 return dateList, pathList
1110 1121
1111 1122 return dateList
1112 1123
1113 1124 def setup(self,
1114 1125 path=None,
1115 1126 startDate=None,
1116 1127 endDate=None,
1117 1128 startTime=datetime.time(0,0,0),
1118 1129 endTime=datetime.time(23,59,59),
1119 1130 set=None,
1120 1131 expLabel = "",
1121 1132 ext = None,
1122 1133 online = False,
1123 1134 delay = 60,
1124 1135 walk = True,
1125 1136 getblock = False,
1126 1137 nTxs = 1):
1127 1138
1128 1139 if path == None:
1129 1140 raise ValueError, "[Reading] The path is not valid"
1130 1141
1131 1142 if ext == None:
1132 1143 ext = self.ext
1133 1144
1134 1145 if online:
1135 1146 print "[Reading] Searching files in online mode..."
1136 1147
1137 1148 for nTries in range( self.nTries ):
1138 1149 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1139 1150
1140 1151 if fullpath:
1141 1152 break
1142 1153
1143 1154 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1144 1155 sleep( self.delay )
1145 1156
1146 1157 if not(fullpath):
1147 1158 print "[Reading] There 'isn't any valid file in %s" % path
1148 1159 return None
1149 1160
1150 1161 self.year = year
1151 1162 self.doy = doy
1152 1163 self.set = set - 1
1153 1164 self.path = path
1154 1165 self.foldercounter = foldercounter
1155 1166 last_set = None
1156 1167
1157 1168 else:
1158 1169 print "[Reading] Searching files in offline mode ..."
1159 1170 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1160 1171 startTime=startTime, endTime=endTime,
1161 1172 set=set, expLabel=expLabel, ext=ext,
1162 1173 walk=walk)
1163 1174
1164 1175 if not(pathList):
1165 1176 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1166 1177 # datetime.datetime.combine(startDate,startTime).ctime(),
1167 1178 # datetime.datetime.combine(endDate,endTime).ctime())
1168 1179 #
1169 1180 sys.exit(-1)
1170 1181
1171 1182
1172 1183 self.fileIndex = -1
1173 1184 self.pathList = pathList
1174 1185 self.filenameList = filenameList
1175 1186 file_name = os.path.basename(filenameList[-1])
1176 1187 basename, ext = os.path.splitext(file_name)
1177 1188 last_set = int(basename[-3:])
1178 1189
1179 1190 self.online = online
1180 1191 self.delay = delay
1181 1192 ext = ext.lower()
1182 1193 self.ext = ext
1183 1194 self.getByBlock = getblock
1184 1195 self.nTxs = int(nTxs)
1185 1196
1186 1197 if not(self.setNextFile()):
1187 1198 if (startDate!=None) and (endDate!=None):
1188 1199 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1189 1200 elif startDate != None:
1190 1201 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1191 1202 else:
1192 1203 print "[Reading] No files"
1193 1204
1194 1205 sys.exit(-1)
1195 1206
1196 1207 # self.getBasicHeader()
1197 1208
1198 1209 if last_set != None:
1199 1210 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1200 1211 return
1201 1212
1202 1213 def getBasicHeader(self):
1203 1214
1204 1215 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1205 1216
1206 1217 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1207 1218
1208 1219 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1209 1220
1210 1221 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1211 1222
1212 1223 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1213 1224
1214 1225 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1215 1226
1216 1227 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1217 1228
1218 1229 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1219 1230
1220 1231
1221 1232 def getFirstHeader(self):
1222 1233
1223 1234 raise ValueError, "This method has not been implemented"
1224 1235
1225 1236 def getData(self):
1226 1237
1227 1238 raise ValueError, "This method has not been implemented"
1228 1239
1229 1240 def hasNotDataInBuffer(self):
1230 1241
1231 1242 raise ValueError, "This method has not been implemented"
1232 1243
1233 1244 def readBlock(self):
1234 1245
1235 1246 raise ValueError, "This method has not been implemented"
1236 1247
1237 1248 def isEndProcess(self):
1238 1249
1239 1250 return self.flagNoMoreFiles
1240 1251
1241 1252 def printReadBlocks(self):
1242 1253
1243 1254 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1244 1255
1245 1256 def printTotalBlocks(self):
1246 1257
1247 1258 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1248 1259
1249 1260 def printNumberOfBlock(self):
1250 1261
1251 1262 if self.flagIsNewBlock:
1252 1263 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1253 1264 self.processingHeaderObj.dataBlocksPerFile,
1254 1265 self.dataOut.datatime.ctime())
1255 1266
1256 1267 def printInfo(self):
1257 1268
1258 1269 if self.__printInfo == False:
1259 1270 return
1260 1271
1261 1272 self.basicHeaderObj.printInfo()
1262 1273 self.systemHeaderObj.printInfo()
1263 1274 self.radarControllerHeaderObj.printInfo()
1264 1275 self.processingHeaderObj.printInfo()
1265 1276
1266 1277 self.__printInfo = False
1267 1278
1268 1279
1269 1280 def run(self, **kwargs):
1270 1281
1271 1282 if not(self.isConfig):
1272 1283
1273 1284 # self.dataOut = dataOut
1274 1285 self.setup(**kwargs)
1275 1286 self.isConfig = True
1276 1287
1277 1288 self.getData()
1278 1289
1279 1290 class JRODataWriter(JRODataIO):
1280 1291
1281 1292 """
1282 1293 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1283 1294 de los datos siempre se realiza por bloques.
1284 1295 """
1285 1296
1286 1297 blockIndex = 0
1287 1298
1288 1299 path = None
1289 1300
1290 1301 setFile = None
1291 1302
1292 1303 profilesPerBlock = None
1293 1304
1294 1305 blocksPerFile = None
1295 1306
1296 1307 nWriteBlocks = 0
1297 1308
1298 1309 fileDate = None
1299 1310
1300 1311 def __init__(self, dataOut=None):
1301 1312 raise ValueError, "Not implemented"
1302 1313
1303 1314
1304 1315 def hasAllDataInBuffer(self):
1305 1316 raise ValueError, "Not implemented"
1306 1317
1307 1318
1308 1319 def setBlockDimension(self):
1309 1320 raise ValueError, "Not implemented"
1310 1321
1311 1322
1312 1323 def writeBlock(self):
1313 1324 raise ValueError, "No implemented"
1314 1325
1315 1326
1316 1327 def putData(self):
1317 1328 raise ValueError, "No implemented"
1318 1329
1319 1330
1320 1331 def getProcessFlags(self):
1321 1332
1322 1333 processFlags = 0
1323 1334
1324 1335 dtype_index = get_dtype_index(self.dtype)
1325 1336 procflag_dtype = get_procflag_dtype(dtype_index)
1326 1337
1327 1338 processFlags += procflag_dtype
1328 1339
1329 1340 if self.dataOut.flagDecodeData:
1330 1341 processFlags += PROCFLAG.DECODE_DATA
1331 1342
1332 1343 if self.dataOut.flagDeflipData:
1333 1344 processFlags += PROCFLAG.DEFLIP_DATA
1334 1345
1335 1346 if self.dataOut.code is not None:
1336 1347 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1337 1348
1338 1349 if self.dataOut.nCohInt > 1:
1339 1350 processFlags += PROCFLAG.COHERENT_INTEGRATION
1340 1351
1341 1352 if self.dataOut.type == "Spectra":
1342 1353 if self.dataOut.nIncohInt > 1:
1343 1354 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1344 1355
1345 1356 if self.dataOut.data_dc is not None:
1346 1357 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1347 1358
1348 1359 if self.dataOut.flagShiftFFT:
1349 1360 processFlags += PROCFLAG.SHIFT_FFT_DATA
1350 1361
1351 1362 return processFlags
1352 1363
1353 1364 def setBasicHeader(self):
1354 1365
1355 1366 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1356 1367 self.basicHeaderObj.version = self.versionFile
1357 1368 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1358 1369
1359 1370 utc = numpy.floor(self.dataOut.utctime)
1360 1371 milisecond = (self.dataOut.utctime - utc)* 1000.0
1361 1372
1362 1373 self.basicHeaderObj.utc = utc
1363 1374 self.basicHeaderObj.miliSecond = milisecond
1364 1375 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1365 1376 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1366 1377 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1367 1378
1368 1379 def setFirstHeader(self):
1369 1380 """
1370 1381 Obtiene una copia del First Header
1371 1382
1372 1383 Affected:
1373 1384
1374 1385 self.basicHeaderObj
1375 1386 self.systemHeaderObj
1376 1387 self.radarControllerHeaderObj
1377 1388 self.processingHeaderObj self.
1378 1389
1379 1390 Return:
1380 1391 None
1381 1392 """
1382 1393
1383 1394 raise ValueError, "No implemented"
1384 1395
1385 1396 def __writeFirstHeader(self):
1386 1397 """
1387 1398 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1388 1399
1389 1400 Affected:
1390 1401 __dataType
1391 1402
1392 1403 Return:
1393 1404 None
1394 1405 """
1395 1406
1396 1407 # CALCULAR PARAMETROS
1397 1408
1398 1409 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1399 1410 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1400 1411
1401 1412 self.basicHeaderObj.write(self.fp)
1402 1413 self.systemHeaderObj.write(self.fp)
1403 1414 self.radarControllerHeaderObj.write(self.fp)
1404 1415 self.processingHeaderObj.write(self.fp)
1405 1416
1406 1417 def __setNewBlock(self):
1407 1418 """
1408 1419 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1409 1420
1410 1421 Return:
1411 1422 0 : si no pudo escribir nada
1412 1423 1 : Si escribio el Basic el First Header
1413 1424 """
1414 1425 if self.fp == None:
1415 1426 self.setNextFile()
1416 1427
1417 1428 if self.flagIsNewFile:
1418 1429 return 1
1419 1430
1420 1431 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1421 1432 self.basicHeaderObj.write(self.fp)
1422 1433 return 1
1423 1434
1424 1435 if not( self.setNextFile() ):
1425 1436 return 0
1426 1437
1427 1438 return 1
1428 1439
1429 1440
1430 1441 def writeNextBlock(self):
1431 1442 """
1432 1443 Selecciona el bloque siguiente de datos y los escribe en un file
1433 1444
1434 1445 Return:
1435 1446 0 : Si no hizo pudo escribir el bloque de datos
1436 1447 1 : Si no pudo escribir el bloque de datos
1437 1448 """
1438 1449 if not( self.__setNewBlock() ):
1439 1450 return 0
1440 1451
1441 1452 self.writeBlock()
1442 1453
1443 1454 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1444 1455 self.processingHeaderObj.dataBlocksPerFile)
1445 1456
1446 1457 return 1
1447 1458
1448 1459 def setNextFile(self):
1449 1460 """
1450 1461 Determina el siguiente file que sera escrito
1451 1462
1452 1463 Affected:
1453 1464 self.filename
1454 1465 self.subfolder
1455 1466 self.fp
1456 1467 self.setFile
1457 1468 self.flagIsNewFile
1458 1469
1459 1470 Return:
1460 1471 0 : Si el archivo no puede ser escrito
1461 1472 1 : Si el archivo esta listo para ser escrito
1462 1473 """
1463 1474 ext = self.ext
1464 1475 path = self.path
1465 1476
1466 1477 if self.fp != None:
1467 1478 self.fp.close()
1468 1479
1469 1480 timeTuple = time.localtime( self.dataOut.utctime)
1470 1481 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1471 1482
1472 1483 fullpath = os.path.join( path, subfolder )
1473 1484 setFile = self.setFile
1474 1485
1475 1486 if not( os.path.exists(fullpath) ):
1476 1487 os.mkdir(fullpath)
1477 1488 setFile = -1 #inicializo mi contador de seteo
1478 1489 else:
1479 1490 filesList = os.listdir( fullpath )
1480 1491 if len( filesList ) > 0:
1481 1492 filesList = sorted( filesList, key=str.lower )
1482 1493 filen = filesList[-1]
1483 1494 # el filename debera tener el siguiente formato
1484 1495 # 0 1234 567 89A BCDE (hex)
1485 1496 # x YYYY DDD SSS .ext
1486 1497 if isNumber( filen[8:11] ):
1487 1498 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1488 1499 else:
1489 1500 setFile = -1
1490 1501 else:
1491 1502 setFile = -1 #inicializo mi contador de seteo
1492 1503
1493 1504 setFile += 1
1494 1505
1495 1506 #If this is a new day it resets some values
1496 1507 if self.dataOut.datatime.date() > self.fileDate:
1497 1508 setFile = 0
1498 1509 self.nTotalBlocks = 0
1499 1510
1500 1511 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1501 1512
1502 1513 filename = os.path.join( path, subfolder, filen )
1503 1514
1504 1515 fp = open( filename,'wb' )
1505 1516
1506 1517 self.blockIndex = 0
1507 1518
1508 1519 #guardando atributos
1509 1520 self.filename = filename
1510 1521 self.subfolder = subfolder
1511 1522 self.fp = fp
1512 1523 self.setFile = setFile
1513 1524 self.flagIsNewFile = 1
1514 1525 self.fileDate = self.dataOut.datatime.date()
1515 1526
1516 1527 self.setFirstHeader()
1517 1528
1518 1529 print '[Writing] Opening file: %s'%self.filename
1519 1530
1520 1531 self.__writeFirstHeader()
1521 1532
1522 1533 return 1
1523 1534
1524 1535 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1525 1536 """
1526 1537 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1527 1538
1528 1539 Inputs:
1529 1540 path : directory where data will be saved
1530 1541 profilesPerBlock : number of profiles per block
1531 1542 set : initial file set
1532 1543 datatype : An integer number that defines data type:
1533 1544 0 : int8 (1 byte)
1534 1545 1 : int16 (2 bytes)
1535 1546 2 : int32 (4 bytes)
1536 1547 3 : int64 (8 bytes)
1537 1548 4 : float32 (4 bytes)
1538 1549 5 : double64 (8 bytes)
1539 1550
1540 1551 Return:
1541 1552 0 : Si no realizo un buen seteo
1542 1553 1 : Si realizo un buen seteo
1543 1554 """
1544 1555
1545 1556 if ext == None:
1546 1557 ext = self.ext
1547 1558
1548 1559 self.ext = ext.lower()
1549 1560
1550 1561 self.path = path
1551 1562
1552 1563 if set is None:
1553 1564 self.setFile = -1
1554 1565 else:
1555 1566 self.setFile = set - 1
1556 1567
1557 1568 self.blocksPerFile = blocksPerFile
1558 1569
1559 1570 self.profilesPerBlock = profilesPerBlock
1560 1571
1561 1572 self.dataOut = dataOut
1562 1573 self.fileDate = self.dataOut.datatime.date()
1563 1574 #By default
1564 1575 self.dtype = self.dataOut.dtype
1565 1576
1566 1577 if datatype is not None:
1567 1578 self.dtype = get_numpy_dtype(datatype)
1568 1579
1569 1580 if not(self.setNextFile()):
1570 1581 print "[Writing] There isn't a next file"
1571 1582 return 0
1572 1583
1573 1584 self.setBlockDimension()
1574 1585
1575 1586 return 1
1576 1587
1577 1588 def run(self, dataOut, **kwargs):
1578 1589
1579 1590 if not(self.isConfig):
1580 1591
1581 1592 self.setup(dataOut, **kwargs)
1582 1593 self.isConfig = True
1583 1594
1584 1595 self.putData()
1585 1596
General Comments 0
You need to be logged in to leave comments. Login now