##// END OF EJS Templates
schain
RSS
extraargs for searchfileonline
José Chávez -
r1015:3c8b992938f8
parent child
Show More
Side by Side Unified
Context file:
r1015:3c8b992938f8 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1723 +1,1730
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 print "The file %s can't be opened" %(filename)
72 72 return 0
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 86 def isTimeInRange(thisTime, startTime, endTime):
87 87
88 88 if endTime >= startTime:
89 89 if (thisTime < startTime) or (thisTime > endTime):
90 90 return 0
91 91
92 92 return 1
93 93 else:
94 94 if (thisTime < startTime) and (thisTime > endTime):
95 95 return 0
96 96
97 97 return 1
98 98
99 99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 100 """
101 101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102 102
103 103 Inputs:
104 104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105 105
106 106 startDate : fecha inicial del rango seleccionado en formato datetime.date
107 107
108 108 endDate : fecha final del rango seleccionado en formato datetime.date
109 109
110 110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111 111
112 112 endTime : tiempo final del rango seleccionado en formato datetime.time
113 113
114 114 Return:
115 115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 116 fecha especificado, de lo contrario retorna False.
117 117
118 118 Excepciones:
119 119 Si el archivo no existe o no puede ser abierto
120 120 Si la cabecera no puede ser leida.
121 121
122 122 """
123 123
124 124
125 125 try:
126 126 fp = open(filename,'rb')
127 127 except IOError:
128 128 print "The file %s can't be opened" %(filename)
129 129 return None
130 130
131 131 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 132 systemHeaderObj = SystemHeader()
133 133 radarControllerHeaderObj = RadarControllerHeader()
134 134 processingHeaderObj = ProcessingHeader()
135 135
136 136 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137 137
138 138 sts = firstBasicHeaderObj.read(fp)
139 139
140 140 if not(sts):
141 141 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 142 return None
143 143
144 144 if not systemHeaderObj.read(fp):
145 145 return None
146 146
147 147 if not radarControllerHeaderObj.read(fp):
148 148 return None
149 149
150 150 if not processingHeaderObj.read(fp):
151 151 return None
152 152
153 153 filesize = os.path.getsize(filename)
154 154
155 155 offset = processingHeaderObj.blockSize + 24 #header size
156 156
157 157 if filesize <= offset:
158 158 print "[Reading] %s: This file has not enough data" %filename
159 159 return None
160 160
161 161 fp.seek(-offset, 2)
162 162
163 163 sts = lastBasicHeaderObj.read(fp)
164 164
165 165 fp.close()
166 166
167 167 thisDatetime = lastBasicHeaderObj.datatime
168 168 thisTime_last_block = thisDatetime.time()
169 169
170 170 thisDatetime = firstBasicHeaderObj.datatime
171 171 thisDate = thisDatetime.date()
172 172 thisTime_first_block = thisDatetime.time()
173 173
174 174 #General case
175 175 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
176 176 #-----------o----------------------------o-----------
177 177 # startTime endTime
178 178
179 179 if endTime >= startTime:
180 180 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
181 181 return None
182 182
183 183 return thisDatetime
184 184
185 185 #If endTime < startTime then endTime belongs to the next day
186 186
187 187
188 188 #<<<<<<<<<<<o o>>>>>>>>>>>
189 189 #-----------o----------------------------o-----------
190 190 # endTime startTime
191 191
192 192 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 193 return None
194 194
195 195 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 196 return None
197 197
198 198 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 199 return None
200 200
201 201 return thisDatetime
202 202
203 203 def isFolderInDateRange(folder, startDate=None, endDate=None):
204 204 """
205 205 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
206 206
207 207 Inputs:
208 208 folder : nombre completo del directorio.
209 209 Su formato deberia ser "/path_root/?YYYYDDD"
210 210
211 211 siendo:
212 212 YYYY : Anio (ejemplo 2015)
213 213 DDD : Dia del anio (ejemplo 305)
214 214
215 215 startDate : fecha inicial del rango seleccionado en formato datetime.date
216 216
217 217 endDate : fecha final del rango seleccionado en formato datetime.date
218 218
219 219 Return:
220 220 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
221 221 fecha especificado, de lo contrario retorna False.
222 222 Excepciones:
223 223 Si el directorio no tiene el formato adecuado
224 224 """
225 225
226 226 basename = os.path.basename(folder)
227 227
228 228 if not isRadarFolder(basename):
229 229 print "The folder %s has not the rigth format" %folder
230 230 return 0
231 231
232 232 if startDate and endDate:
233 233 thisDate = getDateFromRadarFolder(basename)
234 234
235 235 if thisDate < startDate:
236 236 return 0
237 237
238 238 if thisDate > endDate:
239 239 return 0
240 240
241 241 return 1
242 242
243 243 def isFileInDateRange(filename, startDate=None, endDate=None):
244 244 """
245 245 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
246 246
247 247 Inputs:
248 248 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
249 249
250 250 Su formato deberia ser "?YYYYDDDsss"
251 251
252 252 siendo:
253 253 YYYY : Anio (ejemplo 2015)
254 254 DDD : Dia del anio (ejemplo 305)
255 255 sss : set
256 256
257 257 startDate : fecha inicial del rango seleccionado en formato datetime.date
258 258
259 259 endDate : fecha final del rango seleccionado en formato datetime.date
260 260
261 261 Return:
262 262 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
263 263 fecha especificado, de lo contrario retorna False.
264 264 Excepciones:
265 265 Si el archivo no tiene el formato adecuado
266 266 """
267 267
268 268 basename = os.path.basename(filename)
269 269
270 270 if not isRadarFile(basename):
271 271 print "The filename %s has not the rigth format" %filename
272 272 return 0
273 273
274 274 if startDate and endDate:
275 275 thisDate = getDateFromRadarFile(basename)
276 276
277 277 if thisDate < startDate:
278 278 return 0
279 279
280 280 if thisDate > endDate:
281 281 return 0
282 282
283 283 return 1
284 284
285 285 def getFileFromSet(path, ext, set):
286 286 validFilelist = []
287 287 fileList = os.listdir(path)
288 288
289 289 # 0 1234 567 89A BCDE
290 290 # H YYYY DDD SSS .ext
291 291
292 292 for thisFile in fileList:
293 293 try:
294 294 year = int(thisFile[1:5])
295 295 doy = int(thisFile[5:8])
296 296 except:
297 297 continue
298 298
299 299 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
300 300 continue
301 301
302 302 validFilelist.append(thisFile)
303 303
304 304 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
305 305
306 306 if len(myfile)!= 0:
307 307 return myfile[0]
308 308 else:
309 309 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
310 310 print 'the filename %s does not exist'%filename
311 311 print '...going to the last file: '
312 312
313 313 if validFilelist:
314 314 validFilelist = sorted( validFilelist, key=str.lower )
315 315 return validFilelist[-1]
316 316
317 317 return None
318 318
319 319 def getlastFileFromPath(path, ext):
320 320 """
321 321 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
322 322 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323 323
324 324 Input:
325 325 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
326 326 ext : extension de los files contenidos en una carpeta
327 327
328 328 Return:
329 329 El ultimo file de una determinada carpeta, no se considera el path.
330 330 """
331 331 validFilelist = []
332 332 fileList = os.listdir(path)
333 333
334 334 # 0 1234 567 89A BCDE
335 335 # H YYYY DDD SSS .ext
336 336
337 337 for thisFile in fileList:
338 338
339 339 year = thisFile[1:5]
340 340 if not isNumber(year):
341 341 continue
342 342
343 343 doy = thisFile[5:8]
344 344 if not isNumber(doy):
345 345 continue
346 346
347 347 year = int(year)
348 348 doy = int(doy)
349 349
350 350 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
351 351 continue
352 352
353 353 validFilelist.append(thisFile)
354 354
355 355 if validFilelist:
356 356 validFilelist = sorted( validFilelist, key=str.lower )
357 357 return validFilelist[-1]
358 358
359 359 return None
360 360
361 361 def checkForRealPath(path, foldercounter, year, doy, set, ext):
362 362 """
363 363 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
364 364 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
365 365 el path exacto de un determinado file.
366 366
367 367 Example :
368 368 nombre correcto del file es .../.../D2009307/P2009307367.ext
369 369
370 370 Entonces la funcion prueba con las siguientes combinaciones
371 371 .../.../y2009307367.ext
372 372 .../.../Y2009307367.ext
373 373 .../.../x2009307/y2009307367.ext
374 374 .../.../x2009307/Y2009307367.ext
375 375 .../.../X2009307/y2009307367.ext
376 376 .../.../X2009307/Y2009307367.ext
377 377 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378 378
379 379 Return:
380 380 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 381 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 382 para el filename
383 383 """
384 384 fullfilename = None
385 385 find_flag = False
386 386 filename = None
387 387
388 388 prefixDirList = [None,'d','D']
389 389 if ext.lower() == ".r": #voltage
390 390 prefixFileList = ['d','D']
391 391 elif ext.lower() == ".pdata": #spectra
392 392 prefixFileList = ['p','P']
393 393 else:
394 394 return None, filename
395 395
396 396 #barrido por las combinaciones posibles
397 397 for prefixDir in prefixDirList:
398 398 thispath = path
399 399 if prefixDir != None:
400 400 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
401 401 if foldercounter == 0:
402 402 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
403 403 else:
404 404 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
405 405 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
406 406 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
407 407 fullfilename = os.path.join( thispath, filename ) #formo el path completo
408 408
409 409 if os.path.exists( fullfilename ): #verifico que exista
410 410 find_flag = True
411 411 break
412 412 if find_flag:
413 413 break
414 414
415 415 if not(find_flag):
416 416 return None, filename
417 417
418 418 return fullfilename, filename
419 419
420 420 def isRadarFolder(folder):
421 421 try:
422 422 year = int(folder[1:5])
423 423 doy = int(folder[5:8])
424 424 except:
425 425 return 0
426 426
427 427 return 1
428 428
429 429 def isRadarFile(file):
430 430 try:
431 431 year = int(file[1:5])
432 432 doy = int(file[5:8])
433 433 set = int(file[8:11])
434 434 except:
435 435 return 0
436 436
437 437 return 1
438 438
439 439 def getDateFromRadarFile(file):
440 440 try:
441 441 year = int(file[1:5])
442 442 doy = int(file[5:8])
443 443 set = int(file[8:11])
444 444 except:
445 445 return None
446 446
447 447 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
448 448 return thisDate
449 449
450 450 def getDateFromRadarFolder(folder):
451 451 try:
452 452 year = int(folder[1:5])
453 453 doy = int(folder[5:8])
454 454 except:
455 455 return None
456 456
457 457 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
458 458 return thisDate
459 459
460 460 class JRODataIO:
461 461
462 462 c = 3E8
463 463
464 464 isConfig = False
465 465
466 466 basicHeaderObj = None
467 467
468 468 systemHeaderObj = None
469 469
470 470 radarControllerHeaderObj = None
471 471
472 472 processingHeaderObj = None
473 473
474 474 dtype = None
475 475
476 476 pathList = []
477 477
478 478 filenameList = []
479 479
480 480 filename = None
481 481
482 482 ext = None
483 483
484 484 flagIsNewFile = 1
485 485
486 486 flagDiscontinuousBlock = 0
487 487
488 488 flagIsNewBlock = 0
489 489
490 490 fp = None
491 491
492 492 firstHeaderSize = 0
493 493
494 494 basicHeaderSize = 24
495 495
496 496 versionFile = 1103
497 497
498 498 fileSize = None
499 499
500 500 # ippSeconds = None
501 501
502 502 fileSizeByHeader = None
503 503
504 504 fileIndex = None
505 505
506 506 profileIndex = None
507 507
508 508 blockIndex = None
509 509
510 510 nTotalBlocks = None
511 511
512 512 maxTimeStep = 30
513 513
514 514 lastUTTime = None
515 515
516 516 datablock = None
517 517
518 518 dataOut = None
519 519
520 520 blocksize = None
521 521
522 522 getByBlock = False
523 523
524 524 def __init__(self):
525 525
526 526 raise NotImplementedError
527 527
528 528 def run(self):
529 529
530 530 raise NotImplementedError
531 531
532 532 def getDtypeWidth(self):
533 533
534 534 dtype_index = get_dtype_index(self.dtype)
535 535 dtype_width = get_dtype_width(dtype_index)
536 536
537 537 return dtype_width
538 538
539 539 class JRODataReader(JRODataIO):
540 540
541 541
542 542 online = 0
543 543
544 544 realtime = 0
545 545
546 546 nReadBlocks = 0
547 547
548 548 delay = 10 #number of seconds waiting a new file
549 549
550 550 nTries = 3 #quantity tries
551 551
552 552 nFiles = 3 #number of files for searching
553 553
554 554 path = None
555 555
556 556 foldercounter = 0
557 557
558 558 flagNoMoreFiles = 0
559 559
560 560 datetimeList = []
561 561
562 562 __isFirstTimeOnline = 1
563 563
564 564 __printInfo = True
565 565
566 566 profileIndex = None
567 567
568 568 nTxs = 1
569 569
570 570 txIndex = None
571 571
572 572 #Added--------------------
573 573
574 574 selBlocksize = None
575 575
576 576 selBlocktime = None
577 577
578 578
579 579 def __init__(self):
580 580
581 581 """
582 582 This class is used to find data files
583 583
584 584 Example:
585 585 reader = JRODataReader()
586 586 fileList = reader.findDataFiles()
587 587
588 588 """
589 589 pass
590 590
591 591
592 592 def createObjByDefault(self):
593 593 """
594 594
595 595 """
596 596 raise NotImplementedError
597 597
598 598 def getBlockDimension(self):
599 599
600 600 raise NotImplementedError
601 601
602 602 def __searchFilesOffLine(self,
603 603 path,
604 604 startDate=None,
605 605 endDate=None,
606 606 startTime=datetime.time(0,0,0),
607 607 endTime=datetime.time(23,59,59),
608 608 set=None,
609 609 expLabel='',
610 610 ext='.r',
611 611 walk=True):
612 612
613 613 self.filenameList = []
614 614 self.datetimeList = []
615 615
616 616 pathList = []
617 617
618 618 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
619 619
620 620 if dateList == []:
621 621 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
622 622 return None, None
623 623
624 624 if len(dateList) > 1:
625 625 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
626 626 else:
627 627 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
628 628
629 629 filenameList = []
630 630 datetimeList = []
631 631
632 632 for thisPath in pathList:
633 633 # thisPath = pathList[pathDict[file]]
634 634
635 635 fileList = glob.glob1(thisPath, "*%s" %ext)
636 636 fileList.sort()
637 637
638 638 for file in fileList:
639 639
640 640 filename = os.path.join(thisPath,file)
641 641
642 642 if not isFileInDateRange(filename, startDate, endDate):
643 643 continue
644 644
645 645 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
646 646
647 647 if not(thisDatetime):
648 648 continue
649 649
650 650 filenameList.append(filename)
651 651 datetimeList.append(thisDatetime)
652 652
653 653 if not(filenameList):
654 654 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
655 655 return None, None
656 656
657 657 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
658 658 print
659 659
660 660 for i in range(len(filenameList)):
661 661 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
662 662
663 663 self.filenameList = filenameList
664 664 self.datetimeList = datetimeList
665 665
666 666 return pathList, filenameList
667 667
668 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
668 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None, startDate=None, endDate=None):
669 669
670 670 """
671 671 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
672 672 devuelve el archivo encontrado ademas de otros datos.
673 673
674 674 Input:
675 675 path : carpeta donde estan contenidos los files que contiene data
676 676
677 677 expLabel : Nombre del subexperimento (subfolder)
678 678
679 679 ext : extension de los files
680 680
681 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
681 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
682 682
683 683 Return:
684 684 directory : eL directorio donde esta el file encontrado
685 685 filename : el ultimo file de una determinada carpeta
686 686 year : el anho
687 687 doy : el numero de dia del anho
688 688 set : el set del archivo
689 689
690 690
691 691 """
692 692 if not os.path.isdir(path):
693 693 return None, None, None, None, None, None
694 694
695 695 dirList = []
696 696
697 697 if not walk:
698 698 fullpath = path
699 699 foldercounter = 0
700 700 else:
701 701 #Filtra solo los directorios
702 702 for thisPath in os.listdir(path):
703 703 if not os.path.isdir(os.path.join(path,thisPath)):
704 704 continue
705 705 if not isRadarFolder(thisPath):
706 706 continue
707 707
708 708 dirList.append(thisPath)
709 709
710 710 if not(dirList):
711 711 return None, None, None, None, None, None
712 712
713 713 dirList = sorted( dirList, key=str.lower )
714 714
715 715 doypath = dirList[-1]
716 716 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
717 717 fullpath = os.path.join(path, doypath, expLabel)
718 718
719 719
720 720 print "[Reading] %s folder was found: " %(fullpath )
721 721
722 722 if set == None:
723 723 filename = getlastFileFromPath(fullpath, ext)
724 724 else:
725 725 filename = getFileFromSet(fullpath, ext, set)
726 726
727 727 if not(filename):
728 728 return None, None, None, None, None, None
729 729
730 730 print "[Reading] %s file was found" %(filename)
731 731
732 732 if not(self.__verifyFile(os.path.join(fullpath, filename))):
733 733 return None, None, None, None, None, None
734 734
735 735 year = int( filename[1:5] )
736 736 doy = int( filename[5:8] )
737 737 set = int( filename[8:11] )
738 738
739 739 return fullpath, foldercounter, filename, year, doy, set
740 740
741 741 def __setNextFileOffline(self):
742 742
743 743 idFile = self.fileIndex
744 744
745 745 while (True):
746 746 idFile += 1
747 747 if not(idFile < len(self.filenameList)):
748 748 self.flagNoMoreFiles = 1
749 749 # print "[Reading] No more Files"
750 750 return 0
751 751
752 752 filename = self.filenameList[idFile]
753 753
754 754 if not(self.__verifyFile(filename)):
755 755 continue
756 756
757 757 fileSize = os.path.getsize(filename)
758 758 fp = open(filename,'rb')
759 759 break
760 760
761 761 self.flagIsNewFile = 1
762 762 self.fileIndex = idFile
763 763 self.filename = filename
764 764 self.fileSize = fileSize
765 765 self.fp = fp
766 766
767 767 # print "[Reading] Setting the file: %s"%self.filename
768 768
769 769 return 1
770 770
771 771 def __setNextFileOnline(self):
772 772 """
773 773 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
774 774 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
775 775 siguientes.
776 776
777 777 Affected:
778 778 self.flagIsNewFile
779 779 self.filename
780 780 self.fileSize
781 781 self.fp
782 782 self.set
783 783 self.flagNoMoreFiles
784 784
785 785 Return:
786 786 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
787 787 1 : si el file fue abierto con exito y esta listo a ser leido
788 788
789 789 Excepciones:
790 790 Si un determinado file no puede ser abierto
791 791 """
792 792 nFiles = 0
793 793 fileOk_flag = False
794 794 firstTime_flag = True
795 795
796 796 self.set += 1
797 797
798 798 if self.set > 999:
799 799 self.set = 0
800 800 self.foldercounter += 1
801 801
802 802 #busca el 1er file disponible
803 803 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
804 804 if fullfilename:
805 805 if self.__verifyFile(fullfilename, False):
806 806 fileOk_flag = True
807 807
808 808 #si no encuentra un file entonces espera y vuelve a buscar
809 809 if not(fileOk_flag):
810 810 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
811 811
812 812 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
813 813 tries = self.nTries
814 814 else:
815 815 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
816 816
817 817 for nTries in range( tries ):
818 818 if firstTime_flag:
819 819 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
820 820 sleep( self.delay )
821 821 else:
822 822 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
823 823
824 824 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
825 825 if fullfilename:
826 826 if self.__verifyFile(fullfilename):
827 827 fileOk_flag = True
828 828 break
829 829
830 830 if fileOk_flag:
831 831 break
832 832
833 833 firstTime_flag = False
834 834
835 835 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
836 836 self.set += 1
837 837
838 838 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
839 839 self.set = 0
840 840 self.doy += 1
841 841 self.foldercounter = 0
842 842
843 843 if fileOk_flag:
844 844 self.fileSize = os.path.getsize( fullfilename )
845 845 self.filename = fullfilename
846 846 self.flagIsNewFile = 1
847 847 if self.fp != None: self.fp.close()
848 848 self.fp = open(fullfilename, 'rb')
849 849 self.flagNoMoreFiles = 0
850 850 # print '[Reading] Setting the file: %s' % fullfilename
851 851 else:
852 852 self.fileSize = 0
853 853 self.filename = None
854 854 self.flagIsNewFile = 0
855 855 self.fp = None
856 856 self.flagNoMoreFiles = 1
857 857 # print '[Reading] No more files to read'
858 858
859 859 return fileOk_flag
860 860
861 861 def setNextFile(self):
862 862 if self.fp != None:
863 863 self.fp.close()
864 864
865 865 if self.online:
866 866 newFile = self.__setNextFileOnline()
867 867 else:
868 868 newFile = self.__setNextFileOffline()
869 869
870 870 if not(newFile):
871 871 print '[Reading] No more files to read'
872 872 return 0
873 873
874 874 print '[Reading] Setting the file: %s' % self.filename
875 875
876 876 self.__readFirstHeader()
877 877 self.nReadBlocks = 0
878 878 return 1
879 879
880 880 def __waitNewBlock(self):
881 881 """
882 882 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
883 883
884 884 Si el modo de lectura es OffLine siempre retorn 0
885 885 """
886 886 if not self.online:
887 887 return 0
888 888
889 889 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
890 890 return 0
891 891
892 892 currentPointer = self.fp.tell()
893 893
894 894 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
895 895
896 896 for nTries in range( self.nTries ):
897 897
898 898 self.fp.close()
899 899 self.fp = open( self.filename, 'rb' )
900 900 self.fp.seek( currentPointer )
901 901
902 902 self.fileSize = os.path.getsize( self.filename )
903 903 currentSize = self.fileSize - currentPointer
904 904
905 905 if ( currentSize >= neededSize ):
906 906 self.basicHeaderObj.read(self.fp)
907 907 return 1
908 908
909 909 if self.fileSize == self.fileSizeByHeader:
910 910 # self.flagEoF = True
911 911 return 0
912 912
913 913 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
914 914 sleep( self.delay )
915 915
916 916
917 917 return 0
918 918
919 919 def waitDataBlock(self,pointer_location):
920 920
921 921 currentPointer = pointer_location
922 922
923 923 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
924 924
925 925 for nTries in range( self.nTries ):
926 926 self.fp.close()
927 927 self.fp = open( self.filename, 'rb' )
928 928 self.fp.seek( currentPointer )
929 929
930 930 self.fileSize = os.path.getsize( self.filename )
931 931 currentSize = self.fileSize - currentPointer
932 932
933 933 if ( currentSize >= neededSize ):
934 934 return 1
935 935
936 936 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
937 937 sleep( self.delay )
938 938
939 939 return 0
940 940
941 941 def __jumpToLastBlock(self):
942 942
943 943 if not(self.__isFirstTimeOnline):
944 944 return
945 945
946 946 csize = self.fileSize - self.fp.tell()
947 947 blocksize = self.processingHeaderObj.blockSize
948 948
949 949 #salta el primer bloque de datos
950 950 if csize > self.processingHeaderObj.blockSize:
951 951 self.fp.seek(self.fp.tell() + blocksize)
952 952 else:
953 953 return
954 954
955 955 csize = self.fileSize - self.fp.tell()
956 956 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
957 957 while True:
958 958
959 959 if self.fp.tell()<self.fileSize:
960 960 self.fp.seek(self.fp.tell() + neededsize)
961 961 else:
962 962 self.fp.seek(self.fp.tell() - neededsize)
963 963 break
964 964
965 965 # csize = self.fileSize - self.fp.tell()
966 966 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
967 967 # factor = int(csize/neededsize)
968 968 # if factor > 0:
969 969 # self.fp.seek(self.fp.tell() + factor*neededsize)
970 970
971 971 self.flagIsNewFile = 0
972 972 self.__isFirstTimeOnline = 0
973 973
974 974 def __setNewBlock(self):
975 975
976 976 if self.fp == None:
977 977 return 0
978 978
979 979 # if self.online:
980 980 # self.__jumpToLastBlock()
981 981
982 982 if self.flagIsNewFile:
983 983 self.lastUTTime = self.basicHeaderObj.utc
984 984 return 1
985 985
986 986 if self.realtime:
987 987 self.flagDiscontinuousBlock = 1
988 988 if not(self.setNextFile()):
989 989 return 0
990 990 else:
991 991 return 1
992 992
993 993 currentSize = self.fileSize - self.fp.tell()
994 994 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
995 995
996 996 if (currentSize >= neededSize):
997 997 self.basicHeaderObj.read(self.fp)
998 998 self.lastUTTime = self.basicHeaderObj.utc
999 999 return 1
1000 1000
1001 1001 if self.__waitNewBlock():
1002 1002 self.lastUTTime = self.basicHeaderObj.utc
1003 1003 return 1
1004 1004
1005 1005 if not(self.setNextFile()):
1006 1006 return 0
1007 1007
1008 1008 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1009 1009 self.lastUTTime = self.basicHeaderObj.utc
1010 1010
1011 1011 self.flagDiscontinuousBlock = 0
1012 1012
1013 1013 if deltaTime > self.maxTimeStep:
1014 1014 self.flagDiscontinuousBlock = 1
1015 1015
1016 1016 return 1
1017 1017
1018 1018 def readNextBlock(self):
1019 1019
1020 1020 #Skip block out of startTime and endTime
1021 1021 while True:
1022 1022 if not(self.__setNewBlock()):
1023 1023 return 0
1024 1024
1025 1025 if not(self.readBlock()):
1026 1026 return 0
1027 1027
1028 1028 self.getBasicHeader()
1029 1029
1030 1030 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1031 1031
1032 1032 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1033 1033 self.processingHeaderObj.dataBlocksPerFile,
1034 1034 self.dataOut.datatime.ctime())
1035 1035 continue
1036 1036
1037 1037 break
1038 1038
1039 1039 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1040 1040 self.processingHeaderObj.dataBlocksPerFile,
1041 1041 self.dataOut.datatime.ctime())
1042 1042 return 1
1043 1043
1044 1044 def __readFirstHeader(self):
1045 1045
1046 1046 self.basicHeaderObj.read(self.fp)
1047 1047 self.systemHeaderObj.read(self.fp)
1048 1048 self.radarControllerHeaderObj.read(self.fp)
1049 1049 self.processingHeaderObj.read(self.fp)
1050 1050
1051 1051 self.firstHeaderSize = self.basicHeaderObj.size
1052 1052
1053 1053 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1054 1054 if datatype == 0:
1055 1055 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1056 1056 elif datatype == 1:
1057 1057 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1058 1058 elif datatype == 2:
1059 1059 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1060 1060 elif datatype == 3:
1061 1061 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1062 1062 elif datatype == 4:
1063 1063 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1064 1064 elif datatype == 5:
1065 1065 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1066 1066 else:
1067 1067 raise ValueError, 'Data type was not defined'
1068 1068
1069 1069 self.dtype = datatype_str
1070 1070 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1071 1071 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1072 1072 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1073 1073 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1074 1074 self.getBlockDimension()
1075 1075
1076 1076 def __verifyFile(self, filename, msgFlag=True):
1077 1077
1078 1078 msg = None
1079 1079
1080 1080 try:
1081 1081 fp = open(filename, 'rb')
1082 1082 except IOError:
1083 1083
1084 1084 if msgFlag:
1085 1085 print "[Reading] File %s can't be opened" % (filename)
1086 1086
1087 1087 return False
1088 1088
1089 1089 currentPosition = fp.tell()
1090 1090 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1091 1091
1092 1092 if neededSize == 0:
1093 1093 basicHeaderObj = BasicHeader(LOCALTIME)
1094 1094 systemHeaderObj = SystemHeader()
1095 1095 radarControllerHeaderObj = RadarControllerHeader()
1096 1096 processingHeaderObj = ProcessingHeader()
1097 1097
1098 1098 if not( basicHeaderObj.read(fp) ):
1099 1099 fp.close()
1100 1100 return False
1101 1101
1102 1102 if not( systemHeaderObj.read(fp) ):
1103 1103 fp.close()
1104 1104 return False
1105 1105
1106 1106 if not( radarControllerHeaderObj.read(fp) ):
1107 1107 fp.close()
1108 1108 return False
1109 1109
1110 1110 if not( processingHeaderObj.read(fp) ):
1111 1111 fp.close()
1112 1112 return False
1113 1113
1114 1114 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1115 1115 else:
1116 1116 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1117 1117
1118 1118 fp.close()
1119 1119
1120 1120 fileSize = os.path.getsize(filename)
1121 1121 currentSize = fileSize - currentPosition
1122 1122
1123 1123 if currentSize < neededSize:
1124 1124 if msgFlag and (msg != None):
1125 1125 print msg
1126 1126 return False
1127 1127
1128 1128 return True
1129 1129
1130 1130 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1131 1131
1132 1132 path_empty = True
1133 1133
1134 1134 dateList = []
1135 1135 pathList = []
1136 1136
1137 1137 multi_path = path.split(',')
1138 1138
1139 1139 if not walk:
1140 1140
1141 1141 for single_path in multi_path:
1142 1142
1143 1143 if not os.path.isdir(single_path):
1144 1144 continue
1145 1145
1146 1146 fileList = glob.glob1(single_path, "*"+ext)
1147 1147
1148 1148 if not fileList:
1149 1149 continue
1150 1150
1151 1151 path_empty = False
1152 1152
1153 1153 fileList.sort()
1154 1154
1155 1155 for thisFile in fileList:
1156 1156
1157 1157 if not os.path.isfile(os.path.join(single_path, thisFile)):
1158 1158 continue
1159 1159
1160 1160 if not isRadarFile(thisFile):
1161 1161 continue
1162 1162
1163 1163 if not isFileInDateRange(thisFile, startDate, endDate):
1164 1164 continue
1165 1165
1166 1166 thisDate = getDateFromRadarFile(thisFile)
1167 1167
1168 1168 if thisDate in dateList:
1169 1169 continue
1170 1170
1171 1171 dateList.append(thisDate)
1172 1172 pathList.append(single_path)
1173 1173
1174 1174 else:
1175 1175 for single_path in multi_path:
1176 1176
1177 1177 if not os.path.isdir(single_path):
1178 1178 continue
1179 1179
1180 1180 dirList = []
1181 1181
1182 1182 for thisPath in os.listdir(single_path):
1183 1183
1184 1184 if not os.path.isdir(os.path.join(single_path,thisPath)):
1185 1185 continue
1186 1186
1187 1187 if not isRadarFolder(thisPath):
1188 1188 continue
1189 1189
1190 1190 if not isFolderInDateRange(thisPath, startDate, endDate):
1191 1191 continue
1192 1192
1193 1193 dirList.append(thisPath)
1194 1194
1195 1195 if not dirList:
1196 1196 continue
1197 1197
1198 1198 dirList.sort()
1199 1199
1200 1200 for thisDir in dirList:
1201 1201
1202 1202 datapath = os.path.join(single_path, thisDir, expLabel)
1203 1203 fileList = glob.glob1(datapath, "*"+ext)
1204 1204
1205 1205 if not fileList:
1206 1206 continue
1207 1207
1208 1208 path_empty = False
1209 1209
1210 1210 thisDate = getDateFromRadarFolder(thisDir)
1211 1211
1212 1212 pathList.append(datapath)
1213 1213 dateList.append(thisDate)
1214 1214
1215 1215 dateList.sort()
1216 1216
1217 1217 if walk:
1218 1218 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1219 1219 else:
1220 1220 pattern_path = multi_path[0]
1221 1221
1222 1222 if path_empty:
1223 1223 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1224 1224 else:
1225 1225 if not dateList:
1226 1226 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1227 1227
1228 1228 if include_path:
1229 1229 return dateList, pathList
1230 1230
1231 1231 return dateList
1232 1232
1233 1233 def setup(self,
1234 1234 path=None,
1235 1235 startDate=None,
1236 1236 endDate=None,
1237 1237 startTime=datetime.time(0,0,0),
1238 1238 endTime=datetime.time(23,59,59),
1239 1239 set=None,
1240 1240 expLabel = "",
1241 1241 ext = None,
1242 1242 online = False,
1243 1243 delay = 60,
1244 1244 walk = True,
1245 1245 getblock = False,
1246 1246 nTxs = 1,
1247 1247 realtime=False,
1248 1248 blocksize=None,
1249 1249 blocktime=None):
1250 1250
1251 1251 if path == None:
1252 1252 raise ValueError, "[Reading] The path is not valid"
1253 1253
1254 1254 if ext == None:
1255 1255 ext = self.ext
1256 1256
1257 1257 if online:
1258 1258 print "[Reading] Searching files in online mode..."
1259 1259
1260 1260 for nTries in range( self.nTries ):
1261 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1261 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1262 expLabel=expLabel,
1263 ext=ext,
1264 walk=walk,
1265 startDate=startDate,
1266 endDate=endDate,
1267 startTime=startTime, endTime=endTime,
1268 set=set)
1262 1269
1263 1270 if fullpath:
1264 1271 break
1265 1272
1266 1273 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1267 1274 sleep( self.delay )
1268 1275
1269 1276 if not(fullpath):
1270 1277 print "[Reading] There 'isn't any valid file in %s" % path
1271 1278 return
1272 1279
1273 1280 self.year = year
1274 1281 self.doy = doy
1275 1282 self.set = set - 1
1276 1283 self.path = path
1277 1284 self.foldercounter = foldercounter
1278 1285 last_set = None
1279 1286
1280 1287 else:
1281 1288 print "[Reading] Searching files in offline mode ..."
1282 1289 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1283 1290 startTime=startTime, endTime=endTime,
1284 1291 set=set, expLabel=expLabel, ext=ext,
1285 1292 walk=walk)
1286 1293
1287 1294 if not(pathList):
1288 1295 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1289 1296 # datetime.datetime.combine(startDate,startTime).ctime(),
1290 1297 # datetime.datetime.combine(endDate,endTime).ctime())
1291 1298
1292 1299 # sys.exit(-1)
1293 1300
1294 1301 self.fileIndex = -1
1295 1302 self.pathList = []
1296 1303 self.filenameList = []
1297 1304 return
1298 1305
1299 1306 self.fileIndex = -1
1300 1307 self.pathList = pathList
1301 1308 self.filenameList = filenameList
1302 1309 file_name = os.path.basename(filenameList[-1])
1303 1310 basename, ext = os.path.splitext(file_name)
1304 1311 last_set = int(basename[-3:])
1305 1312
1306 1313 self.online = online
1307 1314 self.realtime = realtime
1308 1315 self.delay = delay
1309 1316 ext = ext.lower()
1310 1317 self.ext = ext
1311 1318 self.getByBlock = getblock
1312 1319 self.nTxs = nTxs
1313 1320 self.startTime = startTime
1314 1321 self.endTime = endTime
1315 1322
1316 1323 #Added-----------------
1317 1324 self.selBlocksize = blocksize
1318 1325 self.selBlocktime = blocktime
1319 1326
1320 1327
1321 1328 if not(self.setNextFile()):
1322 1329 if (startDate!=None) and (endDate!=None):
1323 1330 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1324 1331 elif startDate != None:
1325 1332 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1326 1333 else:
1327 1334 print "[Reading] No files"
1328 1335
1329 1336 self.fileIndex = -1
1330 1337 self.pathList = []
1331 1338 self.filenameList = []
1332 1339 return
1333 1340
1334 1341 # self.getBasicHeader()
1335 1342
1336 1343 if last_set != None:
1337 1344 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1338 1345 return
1339 1346
1340 1347 def getBasicHeader(self):
1341 1348
1342 1349 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1343 1350
1344 1351 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1345 1352
1346 1353 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1347 1354
1348 1355 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1349 1356
1350 1357 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1351 1358
1352 1359 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1353 1360
1354 1361 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1355 1362
1356 1363 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1357 1364
1358 1365
1359 1366 def getFirstHeader(self):
1360 1367
1361 1368 raise NotImplementedError
1362 1369
1363 1370 def getData(self):
1364 1371
1365 1372 raise NotImplementedError
1366 1373
1367 1374 def hasNotDataInBuffer(self):
1368 1375
1369 1376 raise NotImplementedError
1370 1377
1371 1378 def readBlock(self):
1372 1379
1373 1380 raise NotImplementedError
1374 1381
1375 1382 def isEndProcess(self):
1376 1383
1377 1384 return self.flagNoMoreFiles
1378 1385
1379 1386 def printReadBlocks(self):
1380 1387
1381 1388 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1382 1389
1383 1390 def printTotalBlocks(self):
1384 1391
1385 1392 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1386 1393
1387 1394 def printNumberOfBlock(self):
1388 1395
1389 1396 if self.flagIsNewBlock:
1390 1397 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1391 1398 self.processingHeaderObj.dataBlocksPerFile,
1392 1399 self.dataOut.datatime.ctime())
1393 1400
1394 1401 def printInfo(self):
1395 1402
1396 1403 if self.__printInfo == False:
1397 1404 return
1398 1405
1399 1406 self.basicHeaderObj.printInfo()
1400 1407 self.systemHeaderObj.printInfo()
1401 1408 self.radarControllerHeaderObj.printInfo()
1402 1409 self.processingHeaderObj.printInfo()
1403 1410
1404 1411 self.__printInfo = False
1405 1412
1406 1413
1407 1414 def run(self, **kwargs):
1408 1415
1409 1416 if not(self.isConfig):
1410 1417
1411 1418 # self.dataOut = dataOut
1412 1419 self.setup(**kwargs)
1413 1420 self.isConfig = True
1414 1421
1415 1422 self.getData()
1416 1423
1417 1424 class JRODataWriter(JRODataIO):
1418 1425
1419 1426 """
1420 1427 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1421 1428 de los datos siempre se realiza por bloques.
1422 1429 """
1423 1430
1424 1431 blockIndex = 0
1425 1432
1426 1433 path = None
1427 1434
1428 1435 setFile = None
1429 1436
1430 1437 profilesPerBlock = None
1431 1438
1432 1439 blocksPerFile = None
1433 1440
1434 1441 nWriteBlocks = 0
1435 1442
1436 1443 fileDate = None
1437 1444
1438 1445 def __init__(self, dataOut=None):
1439 1446 raise NotImplementedError
1440 1447
1441 1448
1442 1449 def hasAllDataInBuffer(self):
1443 1450 raise NotImplementedError
1444 1451
1445 1452
1446 1453 def setBlockDimension(self):
1447 1454 raise NotImplementedError
1448 1455
1449 1456
1450 1457 def writeBlock(self):
1451 1458 raise NotImplementedError
1452 1459
1453 1460
1454 1461 def putData(self):
1455 1462 raise NotImplementedError
1456 1463
1457 1464
1458 1465 def getProcessFlags(self):
1459 1466
1460 1467 processFlags = 0
1461 1468
1462 1469 dtype_index = get_dtype_index(self.dtype)
1463 1470 procflag_dtype = get_procflag_dtype(dtype_index)
1464 1471
1465 1472 processFlags += procflag_dtype
1466 1473
1467 1474 if self.dataOut.flagDecodeData:
1468 1475 processFlags += PROCFLAG.DECODE_DATA
1469 1476
1470 1477 if self.dataOut.flagDeflipData:
1471 1478 processFlags += PROCFLAG.DEFLIP_DATA
1472 1479
1473 1480 if self.dataOut.code is not None:
1474 1481 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1475 1482
1476 1483 if self.dataOut.nCohInt > 1:
1477 1484 processFlags += PROCFLAG.COHERENT_INTEGRATION
1478 1485
1479 1486 if self.dataOut.type == "Spectra":
1480 1487 if self.dataOut.nIncohInt > 1:
1481 1488 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1482 1489
1483 1490 if self.dataOut.data_dc is not None:
1484 1491 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1485 1492
1486 1493 if self.dataOut.flagShiftFFT:
1487 1494 processFlags += PROCFLAG.SHIFT_FFT_DATA
1488 1495
1489 1496 return processFlags
1490 1497
1491 1498 def setBasicHeader(self):
1492 1499
1493 1500 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1494 1501 self.basicHeaderObj.version = self.versionFile
1495 1502 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1496 1503
1497 1504 utc = numpy.floor(self.dataOut.utctime)
1498 1505 milisecond = (self.dataOut.utctime - utc)* 1000.0
1499 1506
1500 1507 self.basicHeaderObj.utc = utc
1501 1508 self.basicHeaderObj.miliSecond = milisecond
1502 1509 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1503 1510 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1504 1511 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1505 1512
1506 1513 def setFirstHeader(self):
1507 1514 """
1508 1515 Obtiene una copia del First Header
1509 1516
1510 1517 Affected:
1511 1518
1512 1519 self.basicHeaderObj
1513 1520 self.systemHeaderObj
1514 1521 self.radarControllerHeaderObj
1515 1522 self.processingHeaderObj self.
1516 1523
1517 1524 Return:
1518 1525 None
1519 1526 """
1520 1527
1521 1528 raise NotImplementedError
1522 1529
1523 1530 def __writeFirstHeader(self):
1524 1531 """
1525 1532 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1526 1533
1527 1534 Affected:
1528 1535 __dataType
1529 1536
1530 1537 Return:
1531 1538 None
1532 1539 """
1533 1540
1534 1541 # CALCULAR PARAMETROS
1535 1542
1536 1543 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1537 1544 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1538 1545
1539 1546 self.basicHeaderObj.write(self.fp)
1540 1547 self.systemHeaderObj.write(self.fp)
1541 1548 self.radarControllerHeaderObj.write(self.fp)
1542 1549 self.processingHeaderObj.write(self.fp)
1543 1550
1544 1551 def __setNewBlock(self):
1545 1552 """
1546 1553 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1547 1554
1548 1555 Return:
1549 1556 0 : si no pudo escribir nada
1550 1557 1 : Si escribio el Basic el First Header
1551 1558 """
1552 1559 if self.fp == None:
1553 1560 self.setNextFile()
1554 1561
1555 1562 if self.flagIsNewFile:
1556 1563 return 1
1557 1564
1558 1565 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1559 1566 self.basicHeaderObj.write(self.fp)
1560 1567 return 1
1561 1568
1562 1569 if not( self.setNextFile() ):
1563 1570 return 0
1564 1571
1565 1572 return 1
1566 1573
1567 1574
1568 1575 def writeNextBlock(self):
1569 1576 """
1570 1577 Selecciona el bloque siguiente de datos y los escribe en un file
1571 1578
1572 1579 Return:
1573 1580 0 : Si no hizo pudo escribir el bloque de datos
1574 1581 1 : Si no pudo escribir el bloque de datos
1575 1582 """
1576 1583 if not( self.__setNewBlock() ):
1577 1584 return 0
1578 1585
1579 1586 self.writeBlock()
1580 1587
1581 1588 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1582 1589 self.processingHeaderObj.dataBlocksPerFile)
1583 1590
1584 1591 return 1
1585 1592
1586 1593 def setNextFile(self):
1587 1594 """
1588 1595 Determina el siguiente file que sera escrito
1589 1596
1590 1597 Affected:
1591 1598 self.filename
1592 1599 self.subfolder
1593 1600 self.fp
1594 1601 self.setFile
1595 1602 self.flagIsNewFile
1596 1603
1597 1604 Return:
1598 1605 0 : Si el archivo no puede ser escrito
1599 1606 1 : Si el archivo esta listo para ser escrito
1600 1607 """
1601 1608 ext = self.ext
1602 1609 path = self.path
1603 1610
1604 1611 if self.fp != None:
1605 1612 self.fp.close()
1606 1613
1607 1614 timeTuple = time.localtime( self.dataOut.utctime)
1608 1615 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1609 1616
1610 1617 fullpath = os.path.join( path, subfolder )
1611 1618 setFile = self.setFile
1612 1619
1613 1620 if not( os.path.exists(fullpath) ):
1614 1621 os.mkdir(fullpath)
1615 1622 setFile = -1 #inicializo mi contador de seteo
1616 1623 else:
1617 1624 filesList = os.listdir( fullpath )
1618 1625 if len( filesList ) > 0:
1619 1626 filesList = sorted( filesList, key=str.lower )
1620 1627 filen = filesList[-1]
1621 1628 # el filename debera tener el siguiente formato
1622 1629 # 0 1234 567 89A BCDE (hex)
1623 1630 # x YYYY DDD SSS .ext
1624 1631 if isNumber( filen[8:11] ):
1625 1632 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1626 1633 else:
1627 1634 setFile = -1
1628 1635 else:
1629 1636 setFile = -1 #inicializo mi contador de seteo
1630 1637
1631 1638 setFile += 1
1632 1639
1633 1640 #If this is a new day it resets some values
1634 1641 if self.dataOut.datatime.date() > self.fileDate:
1635 1642 setFile = 0
1636 1643 self.nTotalBlocks = 0
1637 1644
1638 1645 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1639 1646
1640 1647 filename = os.path.join( path, subfolder, filen )
1641 1648
1642 1649 fp = open( filename,'wb' )
1643 1650
1644 1651 self.blockIndex = 0
1645 1652
1646 1653 #guardando atributos
1647 1654 self.filename = filename
1648 1655 self.subfolder = subfolder
1649 1656 self.fp = fp
1650 1657 self.setFile = setFile
1651 1658 self.flagIsNewFile = 1
1652 1659 self.fileDate = self.dataOut.datatime.date()
1653 1660
1654 1661 self.setFirstHeader()
1655 1662
1656 1663 print '[Writing] Opening file: %s'%self.filename
1657 1664
1658 1665 self.__writeFirstHeader()
1659 1666
1660 1667 return 1
1661 1668
1662 1669 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1663 1670 """
1664 1671 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1665 1672
1666 1673 Inputs:
1667 1674 path : directory where data will be saved
1668 1675 profilesPerBlock : number of profiles per block
1669 1676 set : initial file set
1670 1677 datatype : An integer number that defines data type:
1671 1678 0 : int8 (1 byte)
1672 1679 1 : int16 (2 bytes)
1673 1680 2 : int32 (4 bytes)
1674 1681 3 : int64 (8 bytes)
1675 1682 4 : float32 (4 bytes)
1676 1683 5 : double64 (8 bytes)
1677 1684
1678 1685 Return:
1679 1686 0 : Si no realizo un buen seteo
1680 1687 1 : Si realizo un buen seteo
1681 1688 """
1682 1689
1683 1690 if ext == None:
1684 1691 ext = self.ext
1685 1692
1686 1693 self.ext = ext.lower()
1687 1694
1688 1695 self.path = path
1689 1696
1690 1697 if set is None:
1691 1698 self.setFile = -1
1692 1699 else:
1693 1700 self.setFile = set - 1
1694 1701
1695 1702 self.blocksPerFile = blocksPerFile
1696 1703
1697 1704 self.profilesPerBlock = profilesPerBlock
1698 1705
1699 1706 self.dataOut = dataOut
1700 1707 self.fileDate = self.dataOut.datatime.date()
1701 1708 #By default
1702 1709 self.dtype = self.dataOut.dtype
1703 1710
1704 1711 if datatype is not None:
1705 1712 self.dtype = get_numpy_dtype(datatype)
1706 1713
1707 1714 if not(self.setNextFile()):
1708 1715 print "[Writing] There isn't a next file"
1709 1716 return 0
1710 1717
1711 1718 self.setBlockDimension()
1712 1719
1713 1720 return 1
1714 1721
1715 1722 def run(self, dataOut, **kwargs):
1716 1723
1717 1724 if not(self.isConfig):
1718 1725
1719 1726 self.setup(dataOut, **kwargs)
1720 1727 self.isConfig = True
1721 1728
1722 1729 self.putData()
1723 1730
General Comments 0
You need to be logged in to leave comments. Login now