##// END OF EJS Templates
schain
RSS
jroIO_base: File size is verified
Miguel Valdez -
r780:52e19e09d5b8
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1698 +1,1704
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 print "The file %s can't be opened" %(filename)
72 72 return 0
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 86 def isTimeInRange(thisTime, startTime, endTime):
87 87
88 88 if endTime >= startTime:
89 89 if (thisTime < startTime) or (thisTime > endTime):
90 90 return 0
91 91
92 92 return 1
93 93 else:
94 94 if (thisTime < startTime) and (thisTime > endTime):
95 95 return 0
96 96
97 97 return 1
98 98
99 99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 100 """
101 101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102 102
103 103 Inputs:
104 104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105 105
106 106 startDate : fecha inicial del rango seleccionado en formato datetime.date
107 107
108 108 endDate : fecha final del rango seleccionado en formato datetime.date
109 109
110 110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111 111
112 112 endTime : tiempo final del rango seleccionado en formato datetime.time
113 113
114 114 Return:
115 115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 116 fecha especificado, de lo contrario retorna False.
117 117
118 118 Excepciones:
119 119 Si el archivo no existe o no puede ser abierto
120 120 Si la cabecera no puede ser leida.
121 121
122 122 """
123 123
124 124
125 125 try:
126 126 fp = open(filename,'rb')
127 127 except IOError:
128 128 print "The file %s can't be opened" %(filename)
129 129 return None
130 130
131 131 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 132 systemHeaderObj = SystemHeader()
133 133 radarControllerHeaderObj = RadarControllerHeader()
134 134 processingHeaderObj = ProcessingHeader()
135 135
136 136 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137 137
138 138 sts = firstBasicHeaderObj.read(fp)
139
140 if not(sts):
141 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 return None
143
139 144 sts = systemHeaderObj.read(fp)
140 145 sts = radarControllerHeaderObj.read(fp)
141 146 sts = processingHeaderObj.read(fp)
142 147
148 filesize = os.path.getsize(filename)
149
143 150 offset = processingHeaderObj.blockSize + 24 #header size
144 151
152 if filesize <= offset:
153 print "[Reading] %s: This file has not enough data" %filename
154 return None
155
145 156 fp.seek(-offset, 2)
146 157
147 158 sts = lastBasicHeaderObj.read(fp)
148 159
149 160 fp.close()
150 161
151 if not(sts):
152 print "Skipping the file %s because it has not a valid header" %(filename)
153 return None
162 thisDatetime = lastBasicHeaderObj.datatime
163 thisTime_last_block = thisDatetime.time()
154 164
155 165 thisDatetime = firstBasicHeaderObj.datatime
156 166 thisDate = thisDatetime.date()
157 167 thisTime_first_block = thisDatetime.time()
158 168
159 thisDatetime = lastBasicHeaderObj.datatime
160 thisTime_last_block = thisDatetime.time()
161
162
163 169 #General case
164 170 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
165 171 #-----------o----------------------------o-----------
166 172 # startTime endTime
167 173
168 174 if endTime >= startTime:
169 175 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
170 176 return None
171 177
172 178 return thisDatetime
173 179
174 180 #If endTime < startTime then endTime belongs to the next day
175 181
176 182
177 183 #<<<<<<<<<<<o o>>>>>>>>>>>
178 184 #-----------o----------------------------o-----------
179 185 # endTime startTime
180 186
181 187 if (thisDate == startDate) and (thisTime_last_block < startTime):
182 188 return None
183 189
184 190 if (thisDate == endDate) and (thisTime_first_block > endTime):
185 191 return None
186 192
187 193 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
188 194 return None
189 195
190 196 return thisDatetime
191 197
192 198 def isFolderInDateRange(folder, startDate=None, endDate=None):
193 199 """
194 200 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
195 201
196 202 Inputs:
197 203 folder : nombre completo del directorio.
198 204 Su formato deberia ser "/path_root/?YYYYDDD"
199 205
200 206 siendo:
201 207 YYYY : Anio (ejemplo 2015)
202 208 DDD : Dia del anio (ejemplo 305)
203 209
204 210 startDate : fecha inicial del rango seleccionado en formato datetime.date
205 211
206 212 endDate : fecha final del rango seleccionado en formato datetime.date
207 213
208 214 Return:
209 215 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
210 216 fecha especificado, de lo contrario retorna False.
211 217 Excepciones:
212 218 Si el directorio no tiene el formato adecuado
213 219 """
214 220
215 221 basename = os.path.basename(folder)
216 222
217 223 if not isRadarFolder(basename):
218 224 print "The folder %s has not the rigth format" %folder
219 225 return 0
220 226
221 227 if startDate and endDate:
222 228 thisDate = getDateFromRadarFolder(basename)
223 229
224 230 if thisDate < startDate:
225 231 return 0
226 232
227 233 if thisDate > endDate:
228 234 return 0
229 235
230 236 return 1
231 237
232 238 def isFileInDateRange(filename, startDate=None, endDate=None):
233 239 """
234 240 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
235 241
236 242 Inputs:
237 243 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
238 244
239 245 Su formato deberia ser "?YYYYDDDsss"
240 246
241 247 siendo:
242 248 YYYY : Anio (ejemplo 2015)
243 249 DDD : Dia del anio (ejemplo 305)
244 250 sss : set
245 251
246 252 startDate : fecha inicial del rango seleccionado en formato datetime.date
247 253
248 254 endDate : fecha final del rango seleccionado en formato datetime.date
249 255
250 256 Return:
251 257 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
252 258 fecha especificado, de lo contrario retorna False.
253 259 Excepciones:
254 260 Si el archivo no tiene el formato adecuado
255 261 """
256 262
257 263 basename = os.path.basename(filename)
258 264
259 265 if not isRadarFile(basename):
260 266 print "The filename %s has not the rigth format" %filename
261 267 return 0
262 268
263 269 if startDate and endDate:
264 270 thisDate = getDateFromRadarFile(basename)
265 271
266 272 if thisDate < startDate:
267 273 return 0
268 274
269 275 if thisDate > endDate:
270 276 return 0
271 277
272 278 return 1
273 279
274 280 def getFileFromSet(path, ext, set):
275 281 validFilelist = []
276 282 fileList = os.listdir(path)
277 283
278 284 # 0 1234 567 89A BCDE
279 285 # H YYYY DDD SSS .ext
280 286
281 287 for thisFile in fileList:
282 288 try:
283 289 year = int(thisFile[1:5])
284 290 doy = int(thisFile[5:8])
285 291 except:
286 292 continue
287 293
288 294 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
289 295 continue
290 296
291 297 validFilelist.append(thisFile)
292 298
293 299 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
294 300
295 301 if len(myfile)!= 0:
296 302 return myfile[0]
297 303 else:
298 304 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
299 305 print 'the filename %s does not exist'%filename
300 306 print '...going to the last file: '
301 307
302 308 if validFilelist:
303 309 validFilelist = sorted( validFilelist, key=str.lower )
304 310 return validFilelist[-1]
305 311
306 312 return None
307 313
308 314 def getlastFileFromPath(path, ext):
309 315 """
310 316 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
311 317 al final de la depuracion devuelve el ultimo file de la lista que quedo.
312 318
313 319 Input:
314 320 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
315 321 ext : extension de los files contenidos en una carpeta
316 322
317 323 Return:
318 324 El ultimo file de una determinada carpeta, no se considera el path.
319 325 """
320 326 validFilelist = []
321 327 fileList = os.listdir(path)
322 328
323 329 # 0 1234 567 89A BCDE
324 330 # H YYYY DDD SSS .ext
325 331
326 332 for thisFile in fileList:
327 333
328 334 year = thisFile[1:5]
329 335 if not isNumber(year):
330 336 continue
331 337
332 338 doy = thisFile[5:8]
333 339 if not isNumber(doy):
334 340 continue
335 341
336 342 year = int(year)
337 343 doy = int(doy)
338 344
339 345 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
340 346 continue
341 347
342 348 validFilelist.append(thisFile)
343 349
344 350 if validFilelist:
345 351 validFilelist = sorted( validFilelist, key=str.lower )
346 352 return validFilelist[-1]
347 353
348 354 return None
349 355
350 356 def checkForRealPath(path, foldercounter, year, doy, set, ext):
351 357 """
352 358 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
353 359 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
354 360 el path exacto de un determinado file.
355 361
356 362 Example :
357 363 nombre correcto del file es .../.../D2009307/P2009307367.ext
358 364
359 365 Entonces la funcion prueba con las siguientes combinaciones
360 366 .../.../y2009307367.ext
361 367 .../.../Y2009307367.ext
362 368 .../.../x2009307/y2009307367.ext
363 369 .../.../x2009307/Y2009307367.ext
364 370 .../.../X2009307/y2009307367.ext
365 371 .../.../X2009307/Y2009307367.ext
366 372 siendo para este caso, la ultima combinacion de letras, identica al file buscado
367 373
368 374 Return:
369 375 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
370 376 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
371 377 para el filename
372 378 """
373 379 fullfilename = None
374 380 find_flag = False
375 381 filename = None
376 382
377 383 prefixDirList = [None,'d','D']
378 384 if ext.lower() == ".r": #voltage
379 385 prefixFileList = ['d','D']
380 386 elif ext.lower() == ".pdata": #spectra
381 387 prefixFileList = ['p','P']
382 388 else:
383 389 return None, filename
384 390
385 391 #barrido por las combinaciones posibles
386 392 for prefixDir in prefixDirList:
387 393 thispath = path
388 394 if prefixDir != None:
389 395 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
390 396 if foldercounter == 0:
391 397 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
392 398 else:
393 399 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
394 400 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
395 401 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
396 402 fullfilename = os.path.join( thispath, filename ) #formo el path completo
397 403
398 404 if os.path.exists( fullfilename ): #verifico que exista
399 405 find_flag = True
400 406 break
401 407 if find_flag:
402 408 break
403 409
404 410 if not(find_flag):
405 411 return None, filename
406 412
407 413 return fullfilename, filename
408 414
409 415 def isRadarFolder(folder):
410 416 try:
411 417 year = int(folder[1:5])
412 418 doy = int(folder[5:8])
413 419 except:
414 420 return 0
415 421
416 422 return 1
417 423
418 424 def isRadarFile(file):
419 425 try:
420 426 year = int(file[1:5])
421 427 doy = int(file[5:8])
422 428 set = int(file[8:11])
423 429 except:
424 430 return 0
425 431
426 432 return 1
427 433
428 434 def getDateFromRadarFile(file):
429 435 try:
430 436 year = int(file[1:5])
431 437 doy = int(file[5:8])
432 438 set = int(file[8:11])
433 439 except:
434 440 return None
435 441
436 442 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
437 443 return thisDate
438 444
439 445 def getDateFromRadarFolder(folder):
440 446 try:
441 447 year = int(folder[1:5])
442 448 doy = int(folder[5:8])
443 449 except:
444 450 return None
445 451
446 452 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
447 453 return thisDate
448 454
449 455 class JRODataIO:
450 456
451 457 c = 3E8
452 458
453 459 isConfig = False
454 460
455 461 basicHeaderObj = None
456 462
457 463 systemHeaderObj = None
458 464
459 465 radarControllerHeaderObj = None
460 466
461 467 processingHeaderObj = None
462 468
463 469 dtype = None
464 470
465 471 pathList = []
466 472
467 473 filenameList = []
468 474
469 475 filename = None
470 476
471 477 ext = None
472 478
473 479 flagIsNewFile = 1
474 480
475 481 flagDiscontinuousBlock = 0
476 482
477 483 flagIsNewBlock = 0
478 484
479 485 fp = None
480 486
481 487 firstHeaderSize = 0
482 488
483 489 basicHeaderSize = 24
484 490
485 491 versionFile = 1103
486 492
487 493 fileSize = None
488 494
489 495 # ippSeconds = None
490 496
491 497 fileSizeByHeader = None
492 498
493 499 fileIndex = None
494 500
495 501 profileIndex = None
496 502
497 503 blockIndex = None
498 504
499 505 nTotalBlocks = None
500 506
501 507 maxTimeStep = 30
502 508
503 509 lastUTTime = None
504 510
505 511 datablock = None
506 512
507 513 dataOut = None
508 514
509 515 blocksize = None
510 516
511 517 getByBlock = False
512 518
513 519 def __init__(self):
514 520
515 521 raise NotImplementedError
516 522
517 523 def run(self):
518 524
519 525 raise NotImplementedError
520 526
521 527 def getDtypeWidth(self):
522 528
523 529 dtype_index = get_dtype_index(self.dtype)
524 530 dtype_width = get_dtype_width(dtype_index)
525 531
526 532 return dtype_width
527 533
528 534 class JRODataReader(JRODataIO):
529 535
530 536
531 537 online = 0
532 538
533 539 realtime = 0
534 540
535 541 nReadBlocks = 0
536 542
537 543 delay = 10 #number of seconds waiting a new file
538 544
539 545 nTries = 3 #quantity tries
540 546
541 547 nFiles = 3 #number of files for searching
542 548
543 549 path = None
544 550
545 551 foldercounter = 0
546 552
547 553 flagNoMoreFiles = 0
548 554
549 555 datetimeList = []
550 556
551 557 __isFirstTimeOnline = 1
552 558
553 559 __printInfo = True
554 560
555 561 profileIndex = None
556 562
557 563 nTxs = 1
558 564
559 565 txIndex = None
560 566
561 567 def __init__(self):
562 568
563 569 """
564 570 This class is used to find data files
565 571
566 572 Example:
567 573 reader = JRODataReader()
568 574 fileList = reader.findDataFiles()
569 575
570 576 """
571 577 pass
572 578
573 579
574 580 def createObjByDefault(self):
575 581 """
576 582
577 583 """
578 584 raise NotImplementedError
579 585
580 586 def getBlockDimension(self):
581 587
582 588 raise NotImplementedError
583 589
584 590 def __searchFilesOffLine(self,
585 591 path,
586 592 startDate=None,
587 593 endDate=None,
588 594 startTime=datetime.time(0,0,0),
589 595 endTime=datetime.time(23,59,59),
590 596 set=None,
591 597 expLabel='',
592 598 ext='.r',
593 599 walk=True):
594 600
595 601 self.filenameList = []
596 602 self.datetimeList = []
597 603
598 604 pathList = []
599 605
600 606 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
601 607
602 608 if dateList == []:
603 609 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
604 610 return None, None
605 611
606 612 if len(dateList) > 1:
607 613 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
608 614 else:
609 615 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
610 616
611 617 filenameList = []
612 618 datetimeList = []
613 619
614 620 for thisPath in pathList:
615 621 # thisPath = pathList[pathDict[file]]
616 622
617 623 fileList = glob.glob1(thisPath, "*%s" %ext)
618 624 fileList.sort()
619 625
620 626 for file in fileList:
621 627
622 628 filename = os.path.join(thisPath,file)
623 629
624 630 if not isFileInDateRange(filename, startDate, endDate):
625 631 continue
626 632
627 633 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
628 634
629 635 if not(thisDatetime):
630 636 continue
631 637
632 638 filenameList.append(filename)
633 639 datetimeList.append(thisDatetime)
634 640
635 641 if not(filenameList):
636 642 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
637 643 return None, None
638 644
639 645 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
640 646 print
641 647
642 648 for i in range(len(filenameList)):
643 649 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
644 650
645 651 self.filenameList = filenameList
646 652 self.datetimeList = datetimeList
647 653
648 654 return pathList, filenameList
649 655
650 656 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
651 657
652 658 """
653 659 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
654 660 devuelve el archivo encontrado ademas de otros datos.
655 661
656 662 Input:
657 663 path : carpeta donde estan contenidos los files que contiene data
658 664
659 665 expLabel : Nombre del subexperimento (subfolder)
660 666
661 667 ext : extension de los files
662 668
663 669 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
664 670
665 671 Return:
666 672 directory : eL directorio donde esta el file encontrado
667 673 filename : el ultimo file de una determinada carpeta
668 674 year : el anho
669 675 doy : el numero de dia del anho
670 676 set : el set del archivo
671 677
672 678
673 679 """
674 680 if not os.path.isdir(path):
675 681 return None, None, None, None, None, None
676 682
677 683 dirList = []
678 684
679 685 if not walk:
680 686 fullpath = path
681 687 foldercounter = 0
682 688 else:
683 689 #Filtra solo los directorios
684 690 for thisPath in os.listdir(path):
685 691 if not os.path.isdir(os.path.join(path,thisPath)):
686 692 continue
687 693 if not isRadarFolder(thisPath):
688 694 continue
689 695
690 696 dirList.append(thisPath)
691 697
692 698 if not(dirList):
693 699 return None, None, None, None, None, None
694 700
695 701 dirList = sorted( dirList, key=str.lower )
696 702
697 703 doypath = dirList[-1]
698 704 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
699 705 fullpath = os.path.join(path, doypath, expLabel)
700 706
701 707
702 708 print "[Reading] %s folder was found: " %(fullpath )
703 709
704 710 if set == None:
705 711 filename = getlastFileFromPath(fullpath, ext)
706 712 else:
707 713 filename = getFileFromSet(fullpath, ext, set)
708 714
709 715 if not(filename):
710 716 return None, None, None, None, None, None
711 717
712 718 print "[Reading] %s file was found" %(filename)
713 719
714 720 if not(self.__verifyFile(os.path.join(fullpath, filename))):
715 721 return None, None, None, None, None, None
716 722
717 723 year = int( filename[1:5] )
718 724 doy = int( filename[5:8] )
719 725 set = int( filename[8:11] )
720 726
721 727 return fullpath, foldercounter, filename, year, doy, set
722 728
723 729 def __setNextFileOffline(self):
724 730
725 731 idFile = self.fileIndex
726 732
727 733 while (True):
728 734 idFile += 1
729 735 if not(idFile < len(self.filenameList)):
730 736 self.flagNoMoreFiles = 1
731 737 # print "[Reading] No more Files"
732 738 return 0
733 739
734 740 filename = self.filenameList[idFile]
735 741
736 742 if not(self.__verifyFile(filename)):
737 743 continue
738 744
739 745 fileSize = os.path.getsize(filename)
740 746 fp = open(filename,'rb')
741 747 break
742 748
743 749 self.flagIsNewFile = 1
744 750 self.fileIndex = idFile
745 751 self.filename = filename
746 752 self.fileSize = fileSize
747 753 self.fp = fp
748 754
749 755 # print "[Reading] Setting the file: %s"%self.filename
750 756
751 757 return 1
752 758
753 759 def __setNextFileOnline(self):
754 760 """
755 761 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
756 762 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
757 763 siguientes.
758 764
759 765 Affected:
760 766 self.flagIsNewFile
761 767 self.filename
762 768 self.fileSize
763 769 self.fp
764 770 self.set
765 771 self.flagNoMoreFiles
766 772
767 773 Return:
768 774 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
769 775 1 : si el file fue abierto con exito y esta listo a ser leido
770 776
771 777 Excepciones:
772 778 Si un determinado file no puede ser abierto
773 779 """
774 780 nFiles = 0
775 781 fileOk_flag = False
776 782 firstTime_flag = True
777 783
778 784 self.set += 1
779 785
780 786 if self.set > 999:
781 787 self.set = 0
782 788 self.foldercounter += 1
783 789
784 790 #busca el 1er file disponible
785 791 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
786 792 if fullfilename:
787 793 if self.__verifyFile(fullfilename, False):
788 794 fileOk_flag = True
789 795
790 796 #si no encuentra un file entonces espera y vuelve a buscar
791 797 if not(fileOk_flag):
792 798 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
793 799
794 800 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
795 801 tries = self.nTries
796 802 else:
797 803 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
798 804
799 805 for nTries in range( tries ):
800 806 if firstTime_flag:
801 807 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
802 808 sleep( self.delay )
803 809 else:
804 810 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
805 811
806 812 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
807 813 if fullfilename:
808 814 if self.__verifyFile(fullfilename):
809 815 fileOk_flag = True
810 816 break
811 817
812 818 if fileOk_flag:
813 819 break
814 820
815 821 firstTime_flag = False
816 822
817 823 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
818 824 self.set += 1
819 825
820 826 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
821 827 self.set = 0
822 828 self.doy += 1
823 829 self.foldercounter = 0
824 830
825 831 if fileOk_flag:
826 832 self.fileSize = os.path.getsize( fullfilename )
827 833 self.filename = fullfilename
828 834 self.flagIsNewFile = 1
829 835 if self.fp != None: self.fp.close()
830 836 self.fp = open(fullfilename, 'rb')
831 837 self.flagNoMoreFiles = 0
832 838 # print '[Reading] Setting the file: %s' % fullfilename
833 839 else:
834 840 self.fileSize = 0
835 841 self.filename = None
836 842 self.flagIsNewFile = 0
837 843 self.fp = None
838 844 self.flagNoMoreFiles = 1
839 845 # print '[Reading] No more files to read'
840 846
841 847 return fileOk_flag
842 848
843 849 def setNextFile(self):
844 850 if self.fp != None:
845 851 self.fp.close()
846 852
847 853 if self.online:
848 854 newFile = self.__setNextFileOnline()
849 855 else:
850 856 newFile = self.__setNextFileOffline()
851 857
852 858 if not(newFile):
853 859 print '[Reading] No more files to read'
854 860 return 0
855 861
856 862 print '[Reading] Setting the file: %s' % self.filename
857 863
858 864 self.__readFirstHeader()
859 865 self.nReadBlocks = 0
860 866 return 1
861 867
862 868 def __waitNewBlock(self):
863 869 """
864 870 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
865 871
866 872 Si el modo de lectura es OffLine siempre retorn 0
867 873 """
868 874 if not self.online:
869 875 return 0
870 876
871 877 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
872 878 return 0
873 879
874 880 currentPointer = self.fp.tell()
875 881
876 882 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
877 883
878 884 for nTries in range( self.nTries ):
879 885
880 886 self.fp.close()
881 887 self.fp = open( self.filename, 'rb' )
882 888 self.fp.seek( currentPointer )
883 889
884 890 self.fileSize = os.path.getsize( self.filename )
885 891 currentSize = self.fileSize - currentPointer
886 892
887 893 if ( currentSize >= neededSize ):
888 894 self.basicHeaderObj.read(self.fp)
889 895 return 1
890 896
891 897 if self.fileSize == self.fileSizeByHeader:
892 898 # self.flagEoF = True
893 899 return 0
894 900
895 901 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
896 902 sleep( self.delay )
897 903
898 904
899 905 return 0
900 906
901 907 def waitDataBlock(self,pointer_location):
902 908
903 909 currentPointer = pointer_location
904 910
905 911 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
906 912
907 913 for nTries in range( self.nTries ):
908 914 self.fp.close()
909 915 self.fp = open( self.filename, 'rb' )
910 916 self.fp.seek( currentPointer )
911 917
912 918 self.fileSize = os.path.getsize( self.filename )
913 919 currentSize = self.fileSize - currentPointer
914 920
915 921 if ( currentSize >= neededSize ):
916 922 return 1
917 923
918 924 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
919 925 sleep( self.delay )
920 926
921 927 return 0
922 928
923 929 def __jumpToLastBlock(self):
924 930
925 931 if not(self.__isFirstTimeOnline):
926 932 return
927 933
928 934 csize = self.fileSize - self.fp.tell()
929 935 blocksize = self.processingHeaderObj.blockSize
930 936
931 937 #salta el primer bloque de datos
932 938 if csize > self.processingHeaderObj.blockSize:
933 939 self.fp.seek(self.fp.tell() + blocksize)
934 940 else:
935 941 return
936 942
937 943 csize = self.fileSize - self.fp.tell()
938 944 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
939 945 while True:
940 946
941 947 if self.fp.tell()<self.fileSize:
942 948 self.fp.seek(self.fp.tell() + neededsize)
943 949 else:
944 950 self.fp.seek(self.fp.tell() - neededsize)
945 951 break
946 952
947 953 # csize = self.fileSize - self.fp.tell()
948 954 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
949 955 # factor = int(csize/neededsize)
950 956 # if factor > 0:
951 957 # self.fp.seek(self.fp.tell() + factor*neededsize)
952 958
953 959 self.flagIsNewFile = 0
954 960 self.__isFirstTimeOnline = 0
955 961
956 962 def __setNewBlock(self):
957 963
958 964 if self.fp == None:
959 965 return 0
960 966
961 967 # if self.online:
962 968 # self.__jumpToLastBlock()
963 969
964 970 if self.flagIsNewFile:
965 971 self.lastUTTime = self.basicHeaderObj.utc
966 972 return 1
967 973
968 974 if self.realtime:
969 975 self.flagDiscontinuousBlock = 1
970 976 if not(self.setNextFile()):
971 977 return 0
972 978 else:
973 979 return 1
974 980
975 981 currentSize = self.fileSize - self.fp.tell()
976 982 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
977 983
978 984 if (currentSize >= neededSize):
979 985 self.basicHeaderObj.read(self.fp)
980 986 self.lastUTTime = self.basicHeaderObj.utc
981 987 return 1
982 988
983 989 if self.__waitNewBlock():
984 990 self.lastUTTime = self.basicHeaderObj.utc
985 991 return 1
986 992
987 993 if not(self.setNextFile()):
988 994 return 0
989 995
990 996 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
991 997 self.lastUTTime = self.basicHeaderObj.utc
992 998
993 999 self.flagDiscontinuousBlock = 0
994 1000
995 1001 if deltaTime > self.maxTimeStep:
996 1002 self.flagDiscontinuousBlock = 1
997 1003
998 1004 return 1
999 1005
1000 1006 def readNextBlock(self):
1001 1007
1002 1008 #Skip block out of startTime and endTime
1003 1009 while True:
1004 1010 if not(self.__setNewBlock()):
1005 1011 return 0
1006 1012
1007 1013 if not(self.readBlock()):
1008 1014 return 0
1009 1015
1010 1016 self.getBasicHeader()
1011 1017
1012 1018 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1013 1019
1014 1020 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1015 1021 self.processingHeaderObj.dataBlocksPerFile,
1016 1022 self.dataOut.datatime.ctime())
1017 1023 continue
1018 1024
1019 1025 break
1020 1026
1021 1027 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1022 1028 self.processingHeaderObj.dataBlocksPerFile,
1023 1029 self.dataOut.datatime.ctime())
1024 1030 return 1
1025 1031
1026 1032 def __readFirstHeader(self):
1027 1033
1028 1034 self.basicHeaderObj.read(self.fp)
1029 1035 self.systemHeaderObj.read(self.fp)
1030 1036 self.radarControllerHeaderObj.read(self.fp)
1031 1037 self.processingHeaderObj.read(self.fp)
1032 1038
1033 1039 self.firstHeaderSize = self.basicHeaderObj.size
1034 1040
1035 1041 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1036 1042 if datatype == 0:
1037 1043 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1038 1044 elif datatype == 1:
1039 1045 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1040 1046 elif datatype == 2:
1041 1047 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1042 1048 elif datatype == 3:
1043 1049 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1044 1050 elif datatype == 4:
1045 1051 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1046 1052 elif datatype == 5:
1047 1053 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1048 1054 else:
1049 1055 raise ValueError, 'Data type was not defined'
1050 1056
1051 1057 self.dtype = datatype_str
1052 1058 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1053 1059 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1054 1060 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1055 1061 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1056 1062 self.getBlockDimension()
1057 1063
1058 1064 def __verifyFile(self, filename, msgFlag=True):
1059 1065
1060 1066 msg = None
1061 1067
1062 1068 try:
1063 1069 fp = open(filename, 'rb')
1064 1070 except IOError:
1065 1071
1066 1072 if msgFlag:
1067 1073 print "[Reading] File %s can't be opened" % (filename)
1068 1074
1069 1075 return False
1070 1076
1071 1077 currentPosition = fp.tell()
1072 1078 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1073 1079
1074 1080 if neededSize == 0:
1075 1081 basicHeaderObj = BasicHeader(LOCALTIME)
1076 1082 systemHeaderObj = SystemHeader()
1077 1083 radarControllerHeaderObj = RadarControllerHeader()
1078 1084 processingHeaderObj = ProcessingHeader()
1079 1085
1080 1086 if not( basicHeaderObj.read(fp) ):
1081 1087 fp.close()
1082 1088 return False
1083 1089
1084 1090 if not( systemHeaderObj.read(fp) ):
1085 1091 fp.close()
1086 1092 return False
1087 1093
1088 1094 if not( radarControllerHeaderObj.read(fp) ):
1089 1095 fp.close()
1090 1096 return False
1091 1097
1092 1098 if not( processingHeaderObj.read(fp) ):
1093 1099 fp.close()
1094 1100 return False
1095 1101
1096 1102 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1097 1103 else:
1098 1104 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1099 1105
1100 1106 fp.close()
1101 1107
1102 1108 fileSize = os.path.getsize(filename)
1103 1109 currentSize = fileSize - currentPosition
1104 1110
1105 1111 if currentSize < neededSize:
1106 1112 if msgFlag and (msg != None):
1107 1113 print msg
1108 1114 return False
1109 1115
1110 1116 return True
1111 1117
1112 1118 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1113 1119
1114 1120 path_empty = True
1115 1121
1116 1122 dateList = []
1117 1123 pathList = []
1118 1124
1119 1125 multi_path = path.split(',')
1120 1126
1121 1127 if not walk:
1122 1128
1123 1129 for single_path in multi_path:
1124 1130
1125 1131 if not os.path.isdir(single_path):
1126 1132 continue
1127 1133
1128 1134 fileList = glob.glob1(single_path, "*"+ext)
1129 1135
1130 1136 if not fileList:
1131 1137 continue
1132 1138
1133 1139 path_empty = False
1134 1140
1135 1141 fileList.sort()
1136 1142
1137 1143 for thisFile in fileList:
1138 1144
1139 1145 if not os.path.isfile(os.path.join(single_path, thisFile)):
1140 1146 continue
1141 1147
1142 1148 if not isRadarFile(thisFile):
1143 1149 continue
1144 1150
1145 1151 if not isFileInDateRange(thisFile, startDate, endDate):
1146 1152 continue
1147 1153
1148 1154 thisDate = getDateFromRadarFile(thisFile)
1149 1155
1150 1156 if thisDate in dateList:
1151 1157 continue
1152 1158
1153 1159 dateList.append(thisDate)
1154 1160 pathList.append(single_path)
1155 1161
1156 1162 else:
1157 1163 for single_path in multi_path:
1158 1164
1159 1165 if not os.path.isdir(single_path):
1160 1166 continue
1161 1167
1162 1168 dirList = []
1163 1169
1164 1170 for thisPath in os.listdir(single_path):
1165 1171
1166 1172 if not os.path.isdir(os.path.join(single_path,thisPath)):
1167 1173 continue
1168 1174
1169 1175 if not isRadarFolder(thisPath):
1170 1176 continue
1171 1177
1172 1178 if not isFolderInDateRange(thisPath, startDate, endDate):
1173 1179 continue
1174 1180
1175 1181 dirList.append(thisPath)
1176 1182
1177 1183 if not dirList:
1178 1184 continue
1179 1185
1180 1186 dirList.sort()
1181 1187
1182 1188 for thisDir in dirList:
1183 1189
1184 1190 datapath = os.path.join(single_path, thisDir, expLabel)
1185 1191 fileList = glob.glob1(datapath, "*"+ext)
1186 1192
1187 1193 if not fileList:
1188 1194 continue
1189 1195
1190 1196 path_empty = False
1191 1197
1192 1198 thisDate = getDateFromRadarFolder(thisDir)
1193 1199
1194 1200 pathList.append(datapath)
1195 1201 dateList.append(thisDate)
1196 1202
1197 1203 dateList.sort()
1198 1204
1199 1205 if walk:
1200 1206 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1201 1207 else:
1202 1208 pattern_path = multi_path[0]
1203 1209
1204 1210 if path_empty:
1205 1211 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1206 1212 else:
1207 1213 if not dateList:
1208 1214 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1209 1215
1210 1216 if include_path:
1211 1217 return dateList, pathList
1212 1218
1213 1219 return dateList
1214 1220
1215 1221 def setup(self,
1216 1222 path=None,
1217 1223 startDate=None,
1218 1224 endDate=None,
1219 1225 startTime=datetime.time(0,0,0),
1220 1226 endTime=datetime.time(23,59,59),
1221 1227 set=None,
1222 1228 expLabel = "",
1223 1229 ext = None,
1224 1230 online = False,
1225 1231 delay = 60,
1226 1232 walk = True,
1227 1233 getblock = False,
1228 1234 nTxs = 1,
1229 1235 realtime=False):
1230 1236
1231 1237 if path == None:
1232 1238 raise ValueError, "[Reading] The path is not valid"
1233 1239
1234 1240 if ext == None:
1235 1241 ext = self.ext
1236 1242
1237 1243 if online:
1238 1244 print "[Reading] Searching files in online mode..."
1239 1245
1240 1246 for nTries in range( self.nTries ):
1241 1247 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1242 1248
1243 1249 if fullpath:
1244 1250 break
1245 1251
1246 1252 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1247 1253 sleep( self.delay )
1248 1254
1249 1255 if not(fullpath):
1250 1256 print "[Reading] There 'isn't any valid file in %s" % path
1251 1257 return
1252 1258
1253 1259 self.year = year
1254 1260 self.doy = doy
1255 1261 self.set = set - 1
1256 1262 self.path = path
1257 1263 self.foldercounter = foldercounter
1258 1264 last_set = None
1259 1265
1260 1266 else:
1261 1267 print "[Reading] Searching files in offline mode ..."
1262 1268 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1263 1269 startTime=startTime, endTime=endTime,
1264 1270 set=set, expLabel=expLabel, ext=ext,
1265 1271 walk=walk)
1266 1272
1267 1273 if not(pathList):
1268 1274 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1269 1275 # datetime.datetime.combine(startDate,startTime).ctime(),
1270 1276 # datetime.datetime.combine(endDate,endTime).ctime())
1271 1277
1272 1278 # sys.exit(-1)
1273 1279
1274 1280 self.fileIndex = -1
1275 1281 self.pathList = []
1276 1282 self.filenameList = []
1277 1283 return
1278 1284
1279 1285 self.fileIndex = -1
1280 1286 self.pathList = pathList
1281 1287 self.filenameList = filenameList
1282 1288 file_name = os.path.basename(filenameList[-1])
1283 1289 basename, ext = os.path.splitext(file_name)
1284 1290 last_set = int(basename[-3:])
1285 1291
1286 1292 self.online = online
1287 1293 self.realtime = realtime
1288 1294 self.delay = delay
1289 1295 ext = ext.lower()
1290 1296 self.ext = ext
1291 1297 self.getByBlock = getblock
1292 1298 self.nTxs = nTxs
1293 1299 self.startTime = startTime
1294 1300 self.endTime = endTime
1295 1301
1296 1302 if not(self.setNextFile()):
1297 1303 if (startDate!=None) and (endDate!=None):
1298 1304 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1299 1305 elif startDate != None:
1300 1306 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1301 1307 else:
1302 1308 print "[Reading] No files"
1303 1309
1304 1310 self.fileIndex = -1
1305 1311 self.pathList = []
1306 1312 self.filenameList = []
1307 1313 return
1308 1314
1309 1315 # self.getBasicHeader()
1310 1316
1311 1317 if last_set != None:
1312 1318 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1313 1319 return
1314 1320
1315 1321 def getBasicHeader(self):
1316 1322
1317 1323 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1318 1324
1319 1325 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1320 1326
1321 1327 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1322 1328
1323 1329 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1324 1330
1325 1331 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1326 1332
1327 1333 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1328 1334
1329 1335 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1330 1336
1331 1337 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1332 1338
1333 1339
1334 1340 def getFirstHeader(self):
1335 1341
1336 1342 raise NotImplementedError
1337 1343
1338 1344 def getData(self):
1339 1345
1340 1346 raise NotImplementedError
1341 1347
1342 1348 def hasNotDataInBuffer(self):
1343 1349
1344 1350 raise NotImplementedError
1345 1351
1346 1352 def readBlock(self):
1347 1353
1348 1354 raise NotImplementedError
1349 1355
1350 1356 def isEndProcess(self):
1351 1357
1352 1358 return self.flagNoMoreFiles
1353 1359
1354 1360 def printReadBlocks(self):
1355 1361
1356 1362 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1357 1363
1358 1364 def printTotalBlocks(self):
1359 1365
1360 1366 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1361 1367
1362 1368 def printNumberOfBlock(self):
1363 1369
1364 1370 if self.flagIsNewBlock:
1365 1371 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1366 1372 self.processingHeaderObj.dataBlocksPerFile,
1367 1373 self.dataOut.datatime.ctime())
1368 1374
1369 1375 def printInfo(self):
1370 1376
1371 1377 if self.__printInfo == False:
1372 1378 return
1373 1379
1374 1380 self.basicHeaderObj.printInfo()
1375 1381 self.systemHeaderObj.printInfo()
1376 1382 self.radarControllerHeaderObj.printInfo()
1377 1383 self.processingHeaderObj.printInfo()
1378 1384
1379 1385 self.__printInfo = False
1380 1386
1381 1387
1382 1388 def run(self, **kwargs):
1383 1389
1384 1390 if not(self.isConfig):
1385 1391
1386 1392 # self.dataOut = dataOut
1387 1393 self.setup(**kwargs)
1388 1394 self.isConfig = True
1389 1395
1390 1396 self.getData()
1391 1397
1392 1398 class JRODataWriter(JRODataIO):
1393 1399
1394 1400 """
1395 1401 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1396 1402 de los datos siempre se realiza por bloques.
1397 1403 """
1398 1404
1399 1405 blockIndex = 0
1400 1406
1401 1407 path = None
1402 1408
1403 1409 setFile = None
1404 1410
1405 1411 profilesPerBlock = None
1406 1412
1407 1413 blocksPerFile = None
1408 1414
1409 1415 nWriteBlocks = 0
1410 1416
1411 1417 fileDate = None
1412 1418
1413 1419 def __init__(self, dataOut=None):
1414 1420 raise NotImplementedError
1415 1421
1416 1422
1417 1423 def hasAllDataInBuffer(self):
1418 1424 raise NotImplementedError
1419 1425
1420 1426
1421 1427 def setBlockDimension(self):
1422 1428 raise NotImplementedError
1423 1429
1424 1430
1425 1431 def writeBlock(self):
1426 1432 raise NotImplementedError
1427 1433
1428 1434
1429 1435 def putData(self):
1430 1436 raise NotImplementedError
1431 1437
1432 1438
1433 1439 def getProcessFlags(self):
1434 1440
1435 1441 processFlags = 0
1436 1442
1437 1443 dtype_index = get_dtype_index(self.dtype)
1438 1444 procflag_dtype = get_procflag_dtype(dtype_index)
1439 1445
1440 1446 processFlags += procflag_dtype
1441 1447
1442 1448 if self.dataOut.flagDecodeData:
1443 1449 processFlags += PROCFLAG.DECODE_DATA
1444 1450
1445 1451 if self.dataOut.flagDeflipData:
1446 1452 processFlags += PROCFLAG.DEFLIP_DATA
1447 1453
1448 1454 if self.dataOut.code is not None:
1449 1455 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1450 1456
1451 1457 if self.dataOut.nCohInt > 1:
1452 1458 processFlags += PROCFLAG.COHERENT_INTEGRATION
1453 1459
1454 1460 if self.dataOut.type == "Spectra":
1455 1461 if self.dataOut.nIncohInt > 1:
1456 1462 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1457 1463
1458 1464 if self.dataOut.data_dc is not None:
1459 1465 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1460 1466
1461 1467 if self.dataOut.flagShiftFFT:
1462 1468 processFlags += PROCFLAG.SHIFT_FFT_DATA
1463 1469
1464 1470 return processFlags
1465 1471
1466 1472 def setBasicHeader(self):
1467 1473
1468 1474 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1469 1475 self.basicHeaderObj.version = self.versionFile
1470 1476 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1471 1477
1472 1478 utc = numpy.floor(self.dataOut.utctime)
1473 1479 milisecond = (self.dataOut.utctime - utc)* 1000.0
1474 1480
1475 1481 self.basicHeaderObj.utc = utc
1476 1482 self.basicHeaderObj.miliSecond = milisecond
1477 1483 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1478 1484 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1479 1485 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1480 1486
1481 1487 def setFirstHeader(self):
1482 1488 """
1483 1489 Obtiene una copia del First Header
1484 1490
1485 1491 Affected:
1486 1492
1487 1493 self.basicHeaderObj
1488 1494 self.systemHeaderObj
1489 1495 self.radarControllerHeaderObj
1490 1496 self.processingHeaderObj self.
1491 1497
1492 1498 Return:
1493 1499 None
1494 1500 """
1495 1501
1496 1502 raise NotImplementedError
1497 1503
1498 1504 def __writeFirstHeader(self):
1499 1505 """
1500 1506 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1501 1507
1502 1508 Affected:
1503 1509 __dataType
1504 1510
1505 1511 Return:
1506 1512 None
1507 1513 """
1508 1514
1509 1515 # CALCULAR PARAMETROS
1510 1516
1511 1517 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1512 1518 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1513 1519
1514 1520 self.basicHeaderObj.write(self.fp)
1515 1521 self.systemHeaderObj.write(self.fp)
1516 1522 self.radarControllerHeaderObj.write(self.fp)
1517 1523 self.processingHeaderObj.write(self.fp)
1518 1524
1519 1525 def __setNewBlock(self):
1520 1526 """
1521 1527 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1522 1528
1523 1529 Return:
1524 1530 0 : si no pudo escribir nada
1525 1531 1 : Si escribio el Basic el First Header
1526 1532 """
1527 1533 if self.fp == None:
1528 1534 self.setNextFile()
1529 1535
1530 1536 if self.flagIsNewFile:
1531 1537 return 1
1532 1538
1533 1539 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1534 1540 self.basicHeaderObj.write(self.fp)
1535 1541 return 1
1536 1542
1537 1543 if not( self.setNextFile() ):
1538 1544 return 0
1539 1545
1540 1546 return 1
1541 1547
1542 1548
1543 1549 def writeNextBlock(self):
1544 1550 """
1545 1551 Selecciona el bloque siguiente de datos y los escribe en un file
1546 1552
1547 1553 Return:
1548 1554 0 : Si no hizo pudo escribir el bloque de datos
1549 1555 1 : Si no pudo escribir el bloque de datos
1550 1556 """
1551 1557 if not( self.__setNewBlock() ):
1552 1558 return 0
1553 1559
1554 1560 self.writeBlock()
1555 1561
1556 1562 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1557 1563 self.processingHeaderObj.dataBlocksPerFile)
1558 1564
1559 1565 return 1
1560 1566
1561 1567 def setNextFile(self):
1562 1568 """
1563 1569 Determina el siguiente file que sera escrito
1564 1570
1565 1571 Affected:
1566 1572 self.filename
1567 1573 self.subfolder
1568 1574 self.fp
1569 1575 self.setFile
1570 1576 self.flagIsNewFile
1571 1577
1572 1578 Return:
1573 1579 0 : Si el archivo no puede ser escrito
1574 1580 1 : Si el archivo esta listo para ser escrito
1575 1581 """
1576 1582 ext = self.ext
1577 1583 path = self.path
1578 1584
1579 1585 if self.fp != None:
1580 1586 self.fp.close()
1581 1587
1582 1588 timeTuple = time.localtime( self.dataOut.utctime)
1583 1589 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1584 1590
1585 1591 fullpath = os.path.join( path, subfolder )
1586 1592 setFile = self.setFile
1587 1593
1588 1594 if not( os.path.exists(fullpath) ):
1589 1595 os.mkdir(fullpath)
1590 1596 setFile = -1 #inicializo mi contador de seteo
1591 1597 else:
1592 1598 filesList = os.listdir( fullpath )
1593 1599 if len( filesList ) > 0:
1594 1600 filesList = sorted( filesList, key=str.lower )
1595 1601 filen = filesList[-1]
1596 1602 # el filename debera tener el siguiente formato
1597 1603 # 0 1234 567 89A BCDE (hex)
1598 1604 # x YYYY DDD SSS .ext
1599 1605 if isNumber( filen[8:11] ):
1600 1606 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1601 1607 else:
1602 1608 setFile = -1
1603 1609 else:
1604 1610 setFile = -1 #inicializo mi contador de seteo
1605 1611
1606 1612 setFile += 1
1607 1613
1608 1614 #If this is a new day it resets some values
1609 1615 if self.dataOut.datatime.date() > self.fileDate:
1610 1616 setFile = 0
1611 1617 self.nTotalBlocks = 0
1612 1618
1613 1619 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1614 1620
1615 1621 filename = os.path.join( path, subfolder, filen )
1616 1622
1617 1623 fp = open( filename,'wb' )
1618 1624
1619 1625 self.blockIndex = 0
1620 1626
1621 1627 #guardando atributos
1622 1628 self.filename = filename
1623 1629 self.subfolder = subfolder
1624 1630 self.fp = fp
1625 1631 self.setFile = setFile
1626 1632 self.flagIsNewFile = 1
1627 1633 self.fileDate = self.dataOut.datatime.date()
1628 1634
1629 1635 self.setFirstHeader()
1630 1636
1631 1637 print '[Writing] Opening file: %s'%self.filename
1632 1638
1633 1639 self.__writeFirstHeader()
1634 1640
1635 1641 return 1
1636 1642
1637 1643 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1638 1644 """
1639 1645 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1640 1646
1641 1647 Inputs:
1642 1648 path : directory where data will be saved
1643 1649 profilesPerBlock : number of profiles per block
1644 1650 set : initial file set
1645 1651 datatype : An integer number that defines data type:
1646 1652 0 : int8 (1 byte)
1647 1653 1 : int16 (2 bytes)
1648 1654 2 : int32 (4 bytes)
1649 1655 3 : int64 (8 bytes)
1650 1656 4 : float32 (4 bytes)
1651 1657 5 : double64 (8 bytes)
1652 1658
1653 1659 Return:
1654 1660 0 : Si no realizo un buen seteo
1655 1661 1 : Si realizo un buen seteo
1656 1662 """
1657 1663
1658 1664 if ext == None:
1659 1665 ext = self.ext
1660 1666
1661 1667 self.ext = ext.lower()
1662 1668
1663 1669 self.path = path
1664 1670
1665 1671 if set is None:
1666 1672 self.setFile = -1
1667 1673 else:
1668 1674 self.setFile = set - 1
1669 1675
1670 1676 self.blocksPerFile = blocksPerFile
1671 1677
1672 1678 self.profilesPerBlock = profilesPerBlock
1673 1679
1674 1680 self.dataOut = dataOut
1675 1681 self.fileDate = self.dataOut.datatime.date()
1676 1682 #By default
1677 1683 self.dtype = self.dataOut.dtype
1678 1684
1679 1685 if datatype is not None:
1680 1686 self.dtype = get_numpy_dtype(datatype)
1681 1687
1682 1688 if not(self.setNextFile()):
1683 1689 print "[Writing] There isn't a next file"
1684 1690 return 0
1685 1691
1686 1692 self.setBlockDimension()
1687 1693
1688 1694 return 1
1689 1695
1690 1696 def run(self, dataOut, **kwargs):
1691 1697
1692 1698 if not(self.isConfig):
1693 1699
1694 1700 self.setup(dataOut, **kwargs)
1695 1701 self.isConfig = True
1696 1702
1697 1703 self.putData()
1698 1704
General Comments 0
You need to be logged in to leave comments. Login now