##// END OF EJS Templates
schain-jc
RSS
realtime argument added to JROReader
Miguel Valdez -
r659:33724e2ee44a
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1596 +1,1611
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 traceback.print_exc()
72 72 raise IOError, "The file %s can't be opened" %(filename)
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
87 87 """
88 88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89 89
90 90 Inputs:
91 91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92 92
93 93 startDate : fecha inicial del rango seleccionado en formato datetime.date
94 94
95 95 endDate : fecha final del rango seleccionado en formato datetime.date
96 96
97 97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
98 98
99 99 endTime : tiempo final del rango seleccionado en formato datetime.time
100 100
101 101 Return:
102 102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
103 103 fecha especificado, de lo contrario retorna False.
104 104
105 105 Excepciones:
106 106 Si el archivo no existe o no puede ser abierto
107 107 Si la cabecera no puede ser leida.
108 108
109 109 """
110 110
111 111
112 112 try:
113 113 fp = open(filename,'rb')
114 114 except IOError:
115 115 traceback.print_exc()
116 116 raise IOError, "The file %s can't be opened" %(filename)
117 117
118 118 basicHeaderObj = BasicHeader(LOCALTIME)
119 119 sts = basicHeaderObj.read(fp)
120 120 fp.close()
121 121
122 122 thisDatetime = basicHeaderObj.datatime
123 123 thisDate = thisDatetime.date()
124 124 thisTime = thisDatetime.time()
125 125
126 126 if not(sts):
127 127 print "Skipping the file %s because it has not a valid header" %(filename)
128 128 return None
129 129
130 130 #General case
131 131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
132 132 #-----------o----------------------------o-----------
133 133 # startTime endTime
134 134
135 135 if endTime >= startTime:
136 136 if (thisTime < startTime) or (thisTime > endTime):
137 137 return None
138 138
139 139 return thisDatetime
140 140
141 141 #If endTime < startTime then endTime belongs to the next day
142 142
143 143
144 144 #<<<<<<<<<<<o o>>>>>>>>>>>
145 145 #-----------o----------------------------o-----------
146 146 # endTime startTime
147 147
148 148 if (thisDate == startDate) and (thisTime < startTime):
149 149 return None
150 150
151 151 if (thisDate == endDate) and (thisTime > endTime):
152 152 return None
153 153
154 154 if (thisTime < startTime) and (thisTime > endTime):
155 155 return None
156 156
157 157 return thisDatetime
158 158
159 159 def isFolderInDateRange(folder, startDate=None, endDate=None):
160 160 """
161 161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
162 162
163 163 Inputs:
164 164 folder : nombre completo del directorio.
165 165 Su formato deberia ser "/path_root/?YYYYDDD"
166 166
167 167 siendo:
168 168 YYYY : Anio (ejemplo 2015)
169 169 DDD : Dia del anio (ejemplo 305)
170 170
171 171 startDate : fecha inicial del rango seleccionado en formato datetime.date
172 172
173 173 endDate : fecha final del rango seleccionado en formato datetime.date
174 174
175 175 Return:
176 176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
177 177 fecha especificado, de lo contrario retorna False.
178 178 Excepciones:
179 179 Si el directorio no tiene el formato adecuado
180 180 """
181 181
182 182 basename = os.path.basename(folder)
183 183
184 184 if not isRadarFolder(basename):
185 185 raise IOError, "The folder %s has not the rigth format" %folder
186 186
187 187 if startDate and endDate:
188 188 thisDate = getDateFromRadarFolder(basename)
189 189
190 190 if thisDate < startDate:
191 191 return 0
192 192
193 193 if thisDate > endDate:
194 194 return 0
195 195
196 196 return 1
197 197
198 198 def isFileInDateRange(filename, startDate=None, endDate=None):
199 199 """
200 200 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
201 201
202 202 Inputs:
203 203 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
204 204
205 205 Su formato deberia ser "?YYYYDDDsss"
206 206
207 207 siendo:
208 208 YYYY : Anio (ejemplo 2015)
209 209 DDD : Dia del anio (ejemplo 305)
210 210 sss : set
211 211
212 212 startDate : fecha inicial del rango seleccionado en formato datetime.date
213 213
214 214 endDate : fecha final del rango seleccionado en formato datetime.date
215 215
216 216 Return:
217 217 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
218 218 fecha especificado, de lo contrario retorna False.
219 219 Excepciones:
220 220 Si el archivo no tiene el formato adecuado
221 221 """
222 222
223 223 basename = os.path.basename(filename)
224 224
225 225 if not isRadarFile(basename):
226 226 raise IOError, "The filename %s has not the rigth format" %filename
227 227
228 228 if startDate and endDate:
229 229 thisDate = getDateFromRadarFile(basename)
230 230
231 231 if thisDate < startDate:
232 232 return 0
233 233
234 234 if thisDate > endDate:
235 235 return 0
236 236
237 237 return 1
238 238
239 239 def getFileFromSet(path, ext, set):
240 240 validFilelist = []
241 241 fileList = os.listdir(path)
242 242
243 243 # 0 1234 567 89A BCDE
244 244 # H YYYY DDD SSS .ext
245 245
246 246 for thisFile in fileList:
247 247 try:
248 248 year = int(thisFile[1:5])
249 249 doy = int(thisFile[5:8])
250 250 except:
251 251 continue
252 252
253 253 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
254 254 continue
255 255
256 256 validFilelist.append(thisFile)
257 257
258 258 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
259 259
260 260 if len(myfile)!= 0:
261 261 return myfile[0]
262 262 else:
263 263 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
264 264 print 'the filename %s does not exist'%filename
265 265 print '...going to the last file: '
266 266
267 267 if validFilelist:
268 268 validFilelist = sorted( validFilelist, key=str.lower )
269 269 return validFilelist[-1]
270 270
271 271 return None
272 272
273 273 def getlastFileFromPath(path, ext):
274 274 """
275 275 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
276 276 al final de la depuracion devuelve el ultimo file de la lista que quedo.
277 277
278 278 Input:
279 279 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
280 280 ext : extension de los files contenidos en una carpeta
281 281
282 282 Return:
283 283 El ultimo file de una determinada carpeta, no se considera el path.
284 284 """
285 285 validFilelist = []
286 286 fileList = os.listdir(path)
287 287
288 288 # 0 1234 567 89A BCDE
289 289 # H YYYY DDD SSS .ext
290 290
291 291 for thisFile in fileList:
292 292
293 293 year = thisFile[1:5]
294 294 if not isNumber(year):
295 295 continue
296 296
297 297 doy = thisFile[5:8]
298 298 if not isNumber(doy):
299 299 continue
300 300
301 301 year = int(year)
302 302 doy = int(doy)
303 303
304 304 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
305 305 continue
306 306
307 307 validFilelist.append(thisFile)
308 308
309 309 if validFilelist:
310 310 validFilelist = sorted( validFilelist, key=str.lower )
311 311 return validFilelist[-1]
312 312
313 313 return None
314 314
315 315 def checkForRealPath(path, foldercounter, year, doy, set, ext):
316 316 """
317 317 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
318 318 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
319 319 el path exacto de un determinado file.
320 320
321 321 Example :
322 322 nombre correcto del file es .../.../D2009307/P2009307367.ext
323 323
324 324 Entonces la funcion prueba con las siguientes combinaciones
325 325 .../.../y2009307367.ext
326 326 .../.../Y2009307367.ext
327 327 .../.../x2009307/y2009307367.ext
328 328 .../.../x2009307/Y2009307367.ext
329 329 .../.../X2009307/y2009307367.ext
330 330 .../.../X2009307/Y2009307367.ext
331 331 siendo para este caso, la ultima combinacion de letras, identica al file buscado
332 332
333 333 Return:
334 334 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
335 335 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
336 336 para el filename
337 337 """
338 338 fullfilename = None
339 339 find_flag = False
340 340 filename = None
341 341
342 342 prefixDirList = [None,'d','D']
343 343 if ext.lower() == ".r": #voltage
344 344 prefixFileList = ['d','D']
345 345 elif ext.lower() == ".pdata": #spectra
346 346 prefixFileList = ['p','P']
347 347 else:
348 348 return None, filename
349 349
350 350 #barrido por las combinaciones posibles
351 351 for prefixDir in prefixDirList:
352 352 thispath = path
353 353 if prefixDir != None:
354 354 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
355 355 if foldercounter == 0:
356 356 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
357 357 else:
358 358 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
359 359 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
360 360 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
361 361 fullfilename = os.path.join( thispath, filename ) #formo el path completo
362 362
363 363 if os.path.exists( fullfilename ): #verifico que exista
364 364 find_flag = True
365 365 break
366 366 if find_flag:
367 367 break
368 368
369 369 if not(find_flag):
370 370 return None, filename
371 371
372 372 return fullfilename, filename
373 373
374 374 def isRadarFolder(folder):
375 375 try:
376 376 year = int(folder[1:5])
377 377 doy = int(folder[5:8])
378 378 except:
379 379 return 0
380 380
381 381 return 1
382 382
383 383 def isRadarFile(file):
384 384 try:
385 385 year = int(file[1:5])
386 386 doy = int(file[5:8])
387 387 set = int(file[8:11])
388 388 except:
389 389 return 0
390 390
391 391 return 1
392 392
393 393 def getDateFromRadarFile(file):
394 394 try:
395 395 year = int(file[1:5])
396 396 doy = int(file[5:8])
397 397 set = int(file[8:11])
398 398 except:
399 399 return None
400 400
401 401 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
402 402 return thisDate
403 403
404 404 def getDateFromRadarFolder(folder):
405 405 try:
406 406 year = int(folder[1:5])
407 407 doy = int(folder[5:8])
408 408 except:
409 409 return None
410 410
411 411 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
412 412 return thisDate
413 413
414 414 class JRODataIO:
415 415
416 416 c = 3E8
417 417
418 418 isConfig = False
419 419
420 420 basicHeaderObj = None
421 421
422 422 systemHeaderObj = None
423 423
424 424 radarControllerHeaderObj = None
425 425
426 426 processingHeaderObj = None
427 427
428 online = 0
429
430 428 dtype = None
431 429
432 430 pathList = []
433 431
434 432 filenameList = []
435 433
436 434 filename = None
437 435
438 436 ext = None
439 437
440 438 flagIsNewFile = 1
441 439
442 440 flagDiscontinuousBlock = 0
443 441
444 442 flagIsNewBlock = 0
445 443
446 444 fp = None
447 445
448 446 firstHeaderSize = 0
449 447
450 448 basicHeaderSize = 24
451 449
452 450 versionFile = 1103
453 451
454 452 fileSize = None
455 453
456 454 # ippSeconds = None
457 455
458 456 fileSizeByHeader = None
459 457
460 458 fileIndex = None
461 459
462 460 profileIndex = None
463 461
464 462 blockIndex = None
465 463
466 464 nTotalBlocks = None
467 465
468 466 maxTimeStep = 30
469 467
470 468 lastUTTime = None
471 469
472 470 datablock = None
473 471
474 472 dataOut = None
475 473
476 474 blocksize = None
477 475
478 476 getByBlock = False
479 477
480 478 def __init__(self):
481 479
482 480 raise ValueError, "Not implemented"
483 481
484 482 def run(self):
485 483
486 484 raise ValueError, "Not implemented"
487 485
488 486 def getDtypeWidth(self):
489 487
490 488 dtype_index = get_dtype_index(self.dtype)
491 489 dtype_width = get_dtype_width(dtype_index)
492 490
493 491 return dtype_width
494 492
495 493 class JRODataReader(JRODataIO):
496 494
495
496 online = 0
497
498 realtime = 0
499
497 500 nReadBlocks = 0
498 501
499 502 delay = 10 #number of seconds waiting a new file
500 503
501 504 nTries = 3 #quantity tries
502 505
503 506 nFiles = 3 #number of files for searching
504 507
505 508 path = None
506 509
507 510 foldercounter = 0
508 511
509 512 flagNoMoreFiles = 0
510 513
511 514 datetimeList = []
512 515
513 516 __isFirstTimeOnline = 1
514 517
515 518 __printInfo = True
516 519
517 520 profileIndex = None
518 521
519 522 nTxs = 1
520 523
521 524 txIndex = None
522 525
523 526 def __init__(self):
524 527
525 528 """
526 529
527 530 """
528 531
529 532 # raise NotImplementedError, "This method has not been implemented"
530 533
531 534
532 535 def createObjByDefault(self):
533 536 """
534 537
535 538 """
536 539 raise NotImplementedError, "This method has not been implemented"
537 540
538 541 def getBlockDimension(self):
539 542
540 543 raise NotImplementedError, "No implemented"
541 544
542 545 def __searchFilesOffLine(self,
543 546 path,
544 547 startDate=None,
545 548 endDate=None,
546 549 startTime=datetime.time(0,0,0),
547 550 endTime=datetime.time(23,59,59),
548 551 set=None,
549 552 expLabel='',
550 553 ext='.r',
551 554 walk=True):
552 555
553 556 self.filenameList = []
554 557 self.datetimeList = []
555 558
556 559 pathList = []
557 560
558 561 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
559 562
560 563 if dateList == []:
561 564 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
562 565 datetime.datetime.combine(startDate,startTime).ctime(),
563 566 datetime.datetime.combine(endDate,endTime).ctime())
564 567
565 568 return None, None
566 569
567 570 if len(dateList) > 1:
568 571 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
569 572 else:
570 573 print "[Reading] data was found for the date %s" %(dateList[0])
571 574
572 575 filenameList = []
573 576 datetimeList = []
574 577
575 578 for thisPath in pathList:
576 579 # thisPath = pathList[pathDict[file]]
577 580
578 581 fileList = glob.glob1(thisPath, "*%s" %ext)
579 582 fileList.sort()
580 583
581 584 for file in fileList:
582 585
583 586 filename = os.path.join(thisPath,file)
584 587
585 588 if not isFileInDateRange(filename, startDate, endDate):
586 589 continue
587 590
588 591 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
589 592
590 593 if not(thisDatetime):
591 594 continue
592 595
593 596 filenameList.append(filename)
594 597 datetimeList.append(thisDatetime)
595 598
596 599 if not(filenameList):
597 600 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
598 601 return None, None
599 602
600 603 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
601 604 print
602 605
603 606 for i in range(len(filenameList)):
604 607 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
605 608
606 609 self.filenameList = filenameList
607 610 self.datetimeList = datetimeList
608 611
609 612 return pathList, filenameList
610 613
611 614 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
612 615
613 616 """
614 617 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
615 618 devuelve el archivo encontrado ademas de otros datos.
616 619
617 620 Input:
618 621 path : carpeta donde estan contenidos los files que contiene data
619 622
620 623 expLabel : Nombre del subexperimento (subfolder)
621 624
622 625 ext : extension de los files
623 626
624 627 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
625 628
626 629 Return:
627 630 directory : eL directorio donde esta el file encontrado
628 631 filename : el ultimo file de una determinada carpeta
629 632 year : el anho
630 633 doy : el numero de dia del anho
631 634 set : el set del archivo
632 635
633 636
634 637 """
635 638 dirList = []
636 639
637 640 if not walk:
638 641 fullpath = path
639 642 foldercounter = 0
640 643 else:
641 644 #Filtra solo los directorios
642 645 for thisPath in os.listdir(path):
643 646 if not os.path.isdir(os.path.join(path,thisPath)):
644 647 continue
645 648 if not isRadarFolder(thisPath):
646 649 continue
647 650
648 651 dirList.append(thisPath)
649 652
650 653 if not(dirList):
651 654 return None, None, None, None, None, None
652 655
653 656 dirList = sorted( dirList, key=str.lower )
654 657
655 658 doypath = dirList[-1]
656 659 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
657 660 fullpath = os.path.join(path, doypath, expLabel)
658 661
659 662
660 663 print "[Reading] %s folder was found: " %(fullpath )
661 664
662 665 if set == None:
663 666 filename = getlastFileFromPath(fullpath, ext)
664 667 else:
665 668 filename = getFileFromSet(fullpath, ext, set)
666 669
667 670 if not(filename):
668 671 return None, None, None, None, None, None
669 672
670 673 print "[Reading] %s file was found" %(filename)
671 674
672 675 if not(self.__verifyFile(os.path.join(fullpath, filename))):
673 676 return None, None, None, None, None, None
674 677
675 678 year = int( filename[1:5] )
676 679 doy = int( filename[5:8] )
677 680 set = int( filename[8:11] )
678 681
679 682 return fullpath, foldercounter, filename, year, doy, set
680 683
681 684 def __setNextFileOffline(self):
682 685
683 686 idFile = self.fileIndex
684 687
685 688 while (True):
686 689 idFile += 1
687 690 if not(idFile < len(self.filenameList)):
688 691 self.flagNoMoreFiles = 1
689 692 # print "[Reading] No more Files"
690 693 return 0
691 694
692 695 filename = self.filenameList[idFile]
693 696
694 697 if not(self.__verifyFile(filename)):
695 698 continue
696 699
697 700 fileSize = os.path.getsize(filename)
698 701 fp = open(filename,'rb')
699 702 break
700 703
701 704 self.flagIsNewFile = 1
702 705 self.fileIndex = idFile
703 706 self.filename = filename
704 707 self.fileSize = fileSize
705 708 self.fp = fp
706 709
707 710 # print "[Reading] Setting the file: %s"%self.filename
708 711
709 712 return 1
710 713
711 714 def __setNextFileOnline(self):
712 715 """
713 716 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
714 717 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
715 718 siguientes.
716 719
717 720 Affected:
718 721 self.flagIsNewFile
719 722 self.filename
720 723 self.fileSize
721 724 self.fp
722 725 self.set
723 726 self.flagNoMoreFiles
724 727
725 728 Return:
726 729 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
727 730 1 : si el file fue abierto con exito y esta listo a ser leido
728 731
729 732 Excepciones:
730 733 Si un determinado file no puede ser abierto
731 734 """
732 735 nFiles = 0
733 736 fileOk_flag = False
734 737 firstTime_flag = True
735 738
736 739 self.set += 1
737 740
738 741 if self.set > 999:
739 742 self.set = 0
740 743 self.foldercounter += 1
741 744
742 745 #busca el 1er file disponible
743 746 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
744 747 if fullfilename:
745 748 if self.__verifyFile(fullfilename, False):
746 749 fileOk_flag = True
747 750
748 751 #si no encuentra un file entonces espera y vuelve a buscar
749 752 if not(fileOk_flag):
750 753 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
751 754
752 755 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
753 756 tries = self.nTries
754 757 else:
755 758 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
756 759
757 760 for nTries in range( tries ):
758 761 if firstTime_flag:
759 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
762 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
760 763 sleep( self.delay )
761 764 else:
762 765 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
763 766
764 767 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
765 768 if fullfilename:
766 769 if self.__verifyFile(fullfilename):
767 770 fileOk_flag = True
768 771 break
769 772
770 773 if fileOk_flag:
771 774 break
772 775
773 776 firstTime_flag = False
774 777
775 778 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
776 779 self.set += 1
777 780
778 781 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
779 782 self.set = 0
780 783 self.doy += 1
781 784 self.foldercounter = 0
782 785
783 786 if fileOk_flag:
784 787 self.fileSize = os.path.getsize( fullfilename )
785 788 self.filename = fullfilename
786 789 self.flagIsNewFile = 1
787 790 if self.fp != None: self.fp.close()
788 791 self.fp = open(fullfilename, 'rb')
789 792 self.flagNoMoreFiles = 0
790 793 # print '[Reading] Setting the file: %s' % fullfilename
791 794 else:
792 795 self.fileSize = 0
793 796 self.filename = None
794 797 self.flagIsNewFile = 0
795 798 self.fp = None
796 799 self.flagNoMoreFiles = 1
797 800 # print '[Reading] No more files to read'
798 801
799 802 return fileOk_flag
800 803
801 804 def setNextFile(self):
802 805 if self.fp != None:
803 806 self.fp.close()
804 807
805 808 if self.online:
806 809 newFile = self.__setNextFileOnline()
807 810 else:
808 811 newFile = self.__setNextFileOffline()
809 812
810 813 if not(newFile):
811 814 print '[Reading] No more files to read'
812 815 return 0
813 816
814 817 print '[Reading] Setting the file: %s' % self.filename
815 818
816 819 self.__readFirstHeader()
817 820 self.nReadBlocks = 0
818 821 return 1
819 822
820 823 def __waitNewBlock(self):
821 824 """
822 825 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
823 826
824 827 Si el modo de lectura es OffLine siempre retorn 0
825 828 """
826 829 if not self.online:
827 830 return 0
828 831
829 832 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
830 833 return 0
831 834
832 835 currentPointer = self.fp.tell()
833 836
834 837 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
835 838
836 839 for nTries in range( self.nTries ):
837 840
838 841 self.fp.close()
839 842 self.fp = open( self.filename, 'rb' )
840 843 self.fp.seek( currentPointer )
841 844
842 845 self.fileSize = os.path.getsize( self.filename )
843 846 currentSize = self.fileSize - currentPointer
844 847
845 848 if ( currentSize >= neededSize ):
846 849 self.basicHeaderObj.read(self.fp)
847 850 return 1
848 851
849 852 if self.fileSize == self.fileSizeByHeader:
850 853 # self.flagEoF = True
851 854 return 0
852 855
853 856 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
854 857 sleep( self.delay )
855 858
856 859
857 860 return 0
858 861
859 862 def waitDataBlock(self,pointer_location):
860 863
861 864 currentPointer = pointer_location
862 865
863 866 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
864 867
865 868 for nTries in range( self.nTries ):
866 869 self.fp.close()
867 870 self.fp = open( self.filename, 'rb' )
868 871 self.fp.seek( currentPointer )
869 872
870 873 self.fileSize = os.path.getsize( self.filename )
871 874 currentSize = self.fileSize - currentPointer
872 875
873 876 if ( currentSize >= neededSize ):
874 877 return 1
875 878
876 879 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
877 880 sleep( self.delay )
878 881
879 882 return 0
880 883
881 884 def __jumpToLastBlock(self):
882 885
883 886 if not(self.__isFirstTimeOnline):
884 887 return
885 888
886 889 csize = self.fileSize - self.fp.tell()
887 890 blocksize = self.processingHeaderObj.blockSize
888 891
889 892 #salta el primer bloque de datos
890 893 if csize > self.processingHeaderObj.blockSize:
891 894 self.fp.seek(self.fp.tell() + blocksize)
892 895 else:
893 896 return
894 897
895 898 csize = self.fileSize - self.fp.tell()
896 899 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
897 900 while True:
898 901
899 902 if self.fp.tell()<self.fileSize:
900 903 self.fp.seek(self.fp.tell() + neededsize)
901 904 else:
902 905 self.fp.seek(self.fp.tell() - neededsize)
903 906 break
904 907
905 908 # csize = self.fileSize - self.fp.tell()
906 909 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
907 910 # factor = int(csize/neededsize)
908 911 # if factor > 0:
909 912 # self.fp.seek(self.fp.tell() + factor*neededsize)
910 913
911 914 self.flagIsNewFile = 0
912 915 self.__isFirstTimeOnline = 0
913 916
914 917 def __setNewBlock(self):
915 918
916 919 if self.fp == None:
917 920 return 0
918
919 if self.online:
920 self.__jumpToLastBlock()
921
922 # if self.online:
923 # self.__jumpToLastBlock()
921 924
922 925 if self.flagIsNewFile:
926 self.lastUTTime = self.basicHeaderObj.utc
923 927 return 1
924 928
925 self.lastUTTime = self.basicHeaderObj.utc
929 if self.realtime:
930 self.flagDiscontinuousBlock = 1
931 if not(self.setNextFile()):
932 return 0
933 else:
934 return 1
935
926 936 currentSize = self.fileSize - self.fp.tell()
927 937 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
928 938
929 939 if (currentSize >= neededSize):
930 940 self.basicHeaderObj.read(self.fp)
941 self.lastUTTime = self.basicHeaderObj.utc
931 942 return 1
932 943
933 944 if self.__waitNewBlock():
945 self.lastUTTime = self.basicHeaderObj.utc
934 946 return 1
935 947
936 948 if not(self.setNextFile()):
937 949 return 0
938 950
939 951 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
952 self.lastUTTime = self.basicHeaderObj.utc
940 953
941 954 self.flagDiscontinuousBlock = 0
942 955
943 956 if deltaTime > self.maxTimeStep:
944 957 self.flagDiscontinuousBlock = 1
945 958
946 959 return 1
947 960
948 961 def readNextBlock(self):
949 962
950 963 if not(self.__setNewBlock()):
951 964 return 0
952 965
953 966 if not(self.readBlock()):
954 967 return 0
955 968
956 969 self.getBasicHeader()
957 970
958 971 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
959 972 self.processingHeaderObj.dataBlocksPerFile,
960 973 self.dataOut.datatime.ctime())
961 974 return 1
962 975
963 976 def __readFirstHeader(self):
964 977
965 978 self.basicHeaderObj.read(self.fp)
966 979 self.systemHeaderObj.read(self.fp)
967 980 self.radarControllerHeaderObj.read(self.fp)
968 981 self.processingHeaderObj.read(self.fp)
969 982
970 983 self.firstHeaderSize = self.basicHeaderObj.size
971 984
972 985 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
973 986 if datatype == 0:
974 987 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
975 988 elif datatype == 1:
976 989 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
977 990 elif datatype == 2:
978 991 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
979 992 elif datatype == 3:
980 993 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
981 994 elif datatype == 4:
982 995 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
983 996 elif datatype == 5:
984 997 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
985 998 else:
986 999 raise ValueError, 'Data type was not defined'
987 1000
988 1001 self.dtype = datatype_str
989 1002 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
990 1003 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
991 1004 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
992 1005 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
993 1006 self.getBlockDimension()
994 1007
995 1008 def __verifyFile(self, filename, msgFlag=True):
996 1009 msg = None
997 1010 try:
998 1011 fp = open(filename, 'rb')
999 1012 currentPosition = fp.tell()
1000 1013 except IOError:
1001 1014 traceback.print_exc()
1002 1015 if msgFlag:
1003 1016 print "[Reading] The file %s can't be opened" % (filename)
1004 1017 return False
1005 1018
1006 1019 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1007 1020
1008 1021 if neededSize == 0:
1009 1022 basicHeaderObj = BasicHeader(LOCALTIME)
1010 1023 systemHeaderObj = SystemHeader()
1011 1024 radarControllerHeaderObj = RadarControllerHeader()
1012 1025 processingHeaderObj = ProcessingHeader()
1013 1026
1014 1027 try:
1015 1028 if not( basicHeaderObj.read(fp) ): raise IOError
1016 1029 if not( systemHeaderObj.read(fp) ): raise IOError
1017 1030 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1018 1031 if not( processingHeaderObj.read(fp) ): raise IOError
1019 1032 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1020 1033
1021 1034 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1022 1035
1023 1036 except IOError:
1024 1037 traceback.print_exc()
1025 1038 sys.exit(0)
1026 1039
1027 1040 if msgFlag:
1028 1041 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1029 1042
1030 1043 fp.close()
1031 1044 return False
1032 1045 else:
1033 1046 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1034 1047
1035 1048 fp.close()
1036 1049 fileSize = os.path.getsize(filename)
1037 1050 currentSize = fileSize - currentPosition
1038 1051 if currentSize < neededSize:
1039 1052 if msgFlag and (msg != None):
1040 1053 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1041 1054 return False
1042 1055
1043 1056 return True
1044 1057
1045 1058 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1046 1059
1047 1060 dateList = []
1048 1061 pathList = []
1049 1062
1050 1063 multi_path = path.split(',')
1051 1064
1052 1065 if not walk:
1053 1066
1054 1067 for single_path in multi_path:
1055 1068
1056 1069 if not os.path.isdir(single_path):
1057 1070 continue
1058 1071
1059 1072 fileList = glob.glob1(single_path, "*"+ext)
1060 1073
1061 1074 for thisFile in fileList:
1062 1075
1063 1076 if not os.path.isfile(os.path.join(single_path, thisFile)):
1064 1077 continue
1065 1078
1066 1079 if not isRadarFile(thisFile):
1067 1080 continue
1068 1081
1069 1082 if not isFileInDateRange(thisFile, startDate, endDate):
1070 1083 continue
1071 1084
1072 1085 thisDate = getDateFromRadarFile(thisFile)
1073 1086
1074 1087 if thisDate in dateList:
1075 1088 continue
1076 1089
1077 1090 dateList.append(thisDate)
1078 1091 pathList.append(single_path)
1079 1092
1080 1093 else:
1081 1094 for single_path in multi_path:
1082 1095
1083 1096 if not os.path.isdir(single_path):
1084 1097 continue
1085 1098
1086 1099 dirList = []
1087 1100
1088 1101 for thisPath in os.listdir(single_path):
1089 1102
1090 1103 if not os.path.isdir(os.path.join(single_path,thisPath)):
1091 1104 continue
1092 1105
1093 1106 if not isRadarFolder(thisPath):
1094 1107 continue
1095 1108
1096 1109 if not isFolderInDateRange(thisPath, startDate, endDate):
1097 1110 continue
1098 1111
1099 1112 dirList.append(thisPath)
1100 1113
1101 1114 if not dirList:
1102 1115 continue
1103 1116
1104 1117 for thisDir in dirList:
1105 1118
1106 1119 datapath = os.path.join(single_path, thisDir, expLabel)
1107 1120 fileList = glob.glob1(datapath, "*"+ext)
1108 1121
1109 1122 if len(fileList) < 1:
1110 1123 continue
1111 1124
1112 1125 thisDate = getDateFromRadarFolder(thisDir)
1113 1126
1114 1127 pathList.append(datapath)
1115 1128 dateList.append(thisDate)
1116 1129
1117 1130 dateList.sort()
1118 1131
1119 1132 if include_path:
1120 1133 return dateList, pathList
1121 1134
1122 1135 return dateList
1123 1136
1124 1137 def setup(self,
1125 1138 path=None,
1126 1139 startDate=None,
1127 1140 endDate=None,
1128 1141 startTime=datetime.time(0,0,0),
1129 1142 endTime=datetime.time(23,59,59),
1130 1143 set=None,
1131 1144 expLabel = "",
1132 1145 ext = None,
1133 1146 online = False,
1134 1147 delay = 60,
1135 1148 walk = True,
1136 1149 getblock = False,
1137 nTxs = 1):
1150 nTxs = 1,
1151 realtime=False):
1138 1152
1139 1153 if path == None:
1140 1154 raise ValueError, "[Reading] The path is not valid"
1141 1155
1142 1156 if ext == None:
1143 1157 ext = self.ext
1144 1158
1145 1159 if online:
1146 1160 print "[Reading] Searching files in online mode..."
1147 1161
1148 1162 for nTries in range( self.nTries ):
1149 1163 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1150 1164
1151 1165 if fullpath:
1152 1166 break
1153 1167
1154 1168 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1155 1169 sleep( self.delay )
1156 1170
1157 1171 if not(fullpath):
1158 1172 print "[Reading] There 'isn't any valid file in %s" % path
1159 1173 return None
1160 1174
1161 1175 self.year = year
1162 1176 self.doy = doy
1163 1177 self.set = set - 1
1164 1178 self.path = path
1165 1179 self.foldercounter = foldercounter
1166 1180 last_set = None
1167 1181
1168 1182 else:
1169 1183 print "[Reading] Searching files in offline mode ..."
1170 1184 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1171 1185 startTime=startTime, endTime=endTime,
1172 1186 set=set, expLabel=expLabel, ext=ext,
1173 1187 walk=walk)
1174 1188
1175 1189 if not(pathList):
1176 1190 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1177 1191 # datetime.datetime.combine(startDate,startTime).ctime(),
1178 1192 # datetime.datetime.combine(endDate,endTime).ctime())
1179 1193 #
1180 1194 sys.exit(-1)
1181 1195
1182 1196
1183 1197 self.fileIndex = -1
1184 1198 self.pathList = pathList
1185 1199 self.filenameList = filenameList
1186 1200 file_name = os.path.basename(filenameList[-1])
1187 1201 basename, ext = os.path.splitext(file_name)
1188 1202 last_set = int(basename[-3:])
1189 1203
1190 1204 self.online = online
1205 self.realtime = realtime
1191 1206 self.delay = delay
1192 1207 ext = ext.lower()
1193 1208 self.ext = ext
1194 1209 self.getByBlock = getblock
1195 1210 self.nTxs = int(nTxs)
1196 1211
1197 1212 if not(self.setNextFile()):
1198 1213 if (startDate!=None) and (endDate!=None):
1199 1214 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1200 1215 elif startDate != None:
1201 1216 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1202 1217 else:
1203 1218 print "[Reading] No files"
1204 1219
1205 1220 sys.exit(-1)
1206 1221
1207 1222 # self.getBasicHeader()
1208 1223
1209 1224 if last_set != None:
1210 1225 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1211 1226 return
1212 1227
1213 1228 def getBasicHeader(self):
1214 1229
1215 1230 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1216 1231
1217 1232 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1218 1233
1219 1234 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1220 1235
1221 1236 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1222 1237
1223 1238 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1224 1239
1225 1240 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1226 1241
1227 1242 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1228 1243
1229 1244 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1230 1245
1231 1246
1232 1247 def getFirstHeader(self):
1233 1248
1234 1249 raise ValueError, "This method has not been implemented"
1235 1250
1236 1251 def getData(self):
1237 1252
1238 1253 raise ValueError, "This method has not been implemented"
1239 1254
1240 1255 def hasNotDataInBuffer(self):
1241 1256
1242 1257 raise ValueError, "This method has not been implemented"
1243 1258
1244 1259 def readBlock(self):
1245 1260
1246 1261 raise ValueError, "This method has not been implemented"
1247 1262
1248 1263 def isEndProcess(self):
1249 1264
1250 1265 return self.flagNoMoreFiles
1251 1266
1252 1267 def printReadBlocks(self):
1253 1268
1254 1269 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1255 1270
1256 1271 def printTotalBlocks(self):
1257 1272
1258 1273 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1259 1274
1260 1275 def printNumberOfBlock(self):
1261 1276
1262 1277 if self.flagIsNewBlock:
1263 1278 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1264 1279 self.processingHeaderObj.dataBlocksPerFile,
1265 1280 self.dataOut.datatime.ctime())
1266 1281
1267 1282 def printInfo(self):
1268 1283
1269 1284 if self.__printInfo == False:
1270 1285 return
1271 1286
1272 1287 self.basicHeaderObj.printInfo()
1273 1288 self.systemHeaderObj.printInfo()
1274 1289 self.radarControllerHeaderObj.printInfo()
1275 1290 self.processingHeaderObj.printInfo()
1276 1291
1277 1292 self.__printInfo = False
1278 1293
1279 1294
1280 1295 def run(self, **kwargs):
1281 1296
1282 1297 if not(self.isConfig):
1283 1298
1284 1299 # self.dataOut = dataOut
1285 1300 self.setup(**kwargs)
1286 1301 self.isConfig = True
1287 1302
1288 1303 self.getData()
1289 1304
1290 1305 class JRODataWriter(JRODataIO):
1291 1306
1292 1307 """
1293 1308 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1294 1309 de los datos siempre se realiza por bloques.
1295 1310 """
1296 1311
1297 1312 blockIndex = 0
1298 1313
1299 1314 path = None
1300 1315
1301 1316 setFile = None
1302 1317
1303 1318 profilesPerBlock = None
1304 1319
1305 1320 blocksPerFile = None
1306 1321
1307 1322 nWriteBlocks = 0
1308 1323
1309 1324 fileDate = None
1310 1325
1311 1326 def __init__(self, dataOut=None):
1312 1327 raise ValueError, "Not implemented"
1313 1328
1314 1329
1315 1330 def hasAllDataInBuffer(self):
1316 1331 raise ValueError, "Not implemented"
1317 1332
1318 1333
1319 1334 def setBlockDimension(self):
1320 1335 raise ValueError, "Not implemented"
1321 1336
1322 1337
1323 1338 def writeBlock(self):
1324 1339 raise ValueError, "No implemented"
1325 1340
1326 1341
1327 1342 def putData(self):
1328 1343 raise ValueError, "No implemented"
1329 1344
1330 1345
1331 1346 def getProcessFlags(self):
1332 1347
1333 1348 processFlags = 0
1334 1349
1335 1350 dtype_index = get_dtype_index(self.dtype)
1336 1351 procflag_dtype = get_procflag_dtype(dtype_index)
1337 1352
1338 1353 processFlags += procflag_dtype
1339 1354
1340 1355 if self.dataOut.flagDecodeData:
1341 1356 processFlags += PROCFLAG.DECODE_DATA
1342 1357
1343 1358 if self.dataOut.flagDeflipData:
1344 1359 processFlags += PROCFLAG.DEFLIP_DATA
1345 1360
1346 1361 if self.dataOut.code is not None:
1347 1362 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1348 1363
1349 1364 if self.dataOut.nCohInt > 1:
1350 1365 processFlags += PROCFLAG.COHERENT_INTEGRATION
1351 1366
1352 1367 if self.dataOut.type == "Spectra":
1353 1368 if self.dataOut.nIncohInt > 1:
1354 1369 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1355 1370
1356 1371 if self.dataOut.data_dc is not None:
1357 1372 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1358 1373
1359 1374 if self.dataOut.flagShiftFFT:
1360 1375 processFlags += PROCFLAG.SHIFT_FFT_DATA
1361 1376
1362 1377 return processFlags
1363 1378
1364 1379 def setBasicHeader(self):
1365 1380
1366 1381 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1367 1382 self.basicHeaderObj.version = self.versionFile
1368 1383 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1369 1384
1370 1385 utc = numpy.floor(self.dataOut.utctime)
1371 1386 milisecond = (self.dataOut.utctime - utc)* 1000.0
1372 1387
1373 1388 self.basicHeaderObj.utc = utc
1374 1389 self.basicHeaderObj.miliSecond = milisecond
1375 1390 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1376 1391 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1377 1392 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1378 1393
1379 1394 def setFirstHeader(self):
1380 1395 """
1381 1396 Obtiene una copia del First Header
1382 1397
1383 1398 Affected:
1384 1399
1385 1400 self.basicHeaderObj
1386 1401 self.systemHeaderObj
1387 1402 self.radarControllerHeaderObj
1388 1403 self.processingHeaderObj self.
1389 1404
1390 1405 Return:
1391 1406 None
1392 1407 """
1393 1408
1394 1409 raise ValueError, "No implemented"
1395 1410
1396 1411 def __writeFirstHeader(self):
1397 1412 """
1398 1413 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1399 1414
1400 1415 Affected:
1401 1416 __dataType
1402 1417
1403 1418 Return:
1404 1419 None
1405 1420 """
1406 1421
1407 1422 # CALCULAR PARAMETROS
1408 1423
1409 1424 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1410 1425 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1411 1426
1412 1427 self.basicHeaderObj.write(self.fp)
1413 1428 self.systemHeaderObj.write(self.fp)
1414 1429 self.radarControllerHeaderObj.write(self.fp)
1415 1430 self.processingHeaderObj.write(self.fp)
1416 1431
1417 1432 def __setNewBlock(self):
1418 1433 """
1419 1434 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1420 1435
1421 1436 Return:
1422 1437 0 : si no pudo escribir nada
1423 1438 1 : Si escribio el Basic el First Header
1424 1439 """
1425 1440 if self.fp == None:
1426 1441 self.setNextFile()
1427 1442
1428 1443 if self.flagIsNewFile:
1429 1444 return 1
1430 1445
1431 1446 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1432 1447 self.basicHeaderObj.write(self.fp)
1433 1448 return 1
1434 1449
1435 1450 if not( self.setNextFile() ):
1436 1451 return 0
1437 1452
1438 1453 return 1
1439 1454
1440 1455
1441 1456 def writeNextBlock(self):
1442 1457 """
1443 1458 Selecciona el bloque siguiente de datos y los escribe en un file
1444 1459
1445 1460 Return:
1446 1461 0 : Si no hizo pudo escribir el bloque de datos
1447 1462 1 : Si no pudo escribir el bloque de datos
1448 1463 """
1449 1464 if not( self.__setNewBlock() ):
1450 1465 return 0
1451 1466
1452 1467 self.writeBlock()
1453 1468
1454 1469 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1455 1470 self.processingHeaderObj.dataBlocksPerFile)
1456 1471
1457 1472 return 1
1458 1473
1459 1474 def setNextFile(self):
1460 1475 """
1461 1476 Determina el siguiente file que sera escrito
1462 1477
1463 1478 Affected:
1464 1479 self.filename
1465 1480 self.subfolder
1466 1481 self.fp
1467 1482 self.setFile
1468 1483 self.flagIsNewFile
1469 1484
1470 1485 Return:
1471 1486 0 : Si el archivo no puede ser escrito
1472 1487 1 : Si el archivo esta listo para ser escrito
1473 1488 """
1474 1489 ext = self.ext
1475 1490 path = self.path
1476 1491
1477 1492 if self.fp != None:
1478 1493 self.fp.close()
1479 1494
1480 1495 timeTuple = time.localtime( self.dataOut.utctime)
1481 1496 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1482 1497
1483 1498 fullpath = os.path.join( path, subfolder )
1484 1499 setFile = self.setFile
1485 1500
1486 1501 if not( os.path.exists(fullpath) ):
1487 1502 os.mkdir(fullpath)
1488 1503 setFile = -1 #inicializo mi contador de seteo
1489 1504 else:
1490 1505 filesList = os.listdir( fullpath )
1491 1506 if len( filesList ) > 0:
1492 1507 filesList = sorted( filesList, key=str.lower )
1493 1508 filen = filesList[-1]
1494 1509 # el filename debera tener el siguiente formato
1495 1510 # 0 1234 567 89A BCDE (hex)
1496 1511 # x YYYY DDD SSS .ext
1497 1512 if isNumber( filen[8:11] ):
1498 1513 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1499 1514 else:
1500 1515 setFile = -1
1501 1516 else:
1502 1517 setFile = -1 #inicializo mi contador de seteo
1503 1518
1504 1519 setFile += 1
1505 1520
1506 1521 #If this is a new day it resets some values
1507 1522 if self.dataOut.datatime.date() > self.fileDate:
1508 1523 setFile = 0
1509 1524 self.nTotalBlocks = 0
1510 1525
1511 1526 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1512 1527
1513 1528 filename = os.path.join( path, subfolder, filen )
1514 1529
1515 1530 fp = open( filename,'wb' )
1516 1531
1517 1532 self.blockIndex = 0
1518 1533
1519 1534 #guardando atributos
1520 1535 self.filename = filename
1521 1536 self.subfolder = subfolder
1522 1537 self.fp = fp
1523 1538 self.setFile = setFile
1524 1539 self.flagIsNewFile = 1
1525 1540 self.fileDate = self.dataOut.datatime.date()
1526 1541
1527 1542 self.setFirstHeader()
1528 1543
1529 1544 print '[Writing] Opening file: %s'%self.filename
1530 1545
1531 1546 self.__writeFirstHeader()
1532 1547
1533 1548 return 1
1534 1549
1535 1550 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1536 1551 """
1537 1552 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1538 1553
1539 1554 Inputs:
1540 1555 path : directory where data will be saved
1541 1556 profilesPerBlock : number of profiles per block
1542 1557 set : initial file set
1543 1558 datatype : An integer number that defines data type:
1544 1559 0 : int8 (1 byte)
1545 1560 1 : int16 (2 bytes)
1546 1561 2 : int32 (4 bytes)
1547 1562 3 : int64 (8 bytes)
1548 1563 4 : float32 (4 bytes)
1549 1564 5 : double64 (8 bytes)
1550 1565
1551 1566 Return:
1552 1567 0 : Si no realizo un buen seteo
1553 1568 1 : Si realizo un buen seteo
1554 1569 """
1555 1570
1556 1571 if ext == None:
1557 1572 ext = self.ext
1558 1573
1559 1574 self.ext = ext.lower()
1560 1575
1561 1576 self.path = path
1562 1577
1563 1578 if set is None:
1564 1579 self.setFile = -1
1565 1580 else:
1566 1581 self.setFile = set - 1
1567 1582
1568 1583 self.blocksPerFile = blocksPerFile
1569 1584
1570 1585 self.profilesPerBlock = profilesPerBlock
1571 1586
1572 1587 self.dataOut = dataOut
1573 1588 self.fileDate = self.dataOut.datatime.date()
1574 1589 #By default
1575 1590 self.dtype = self.dataOut.dtype
1576 1591
1577 1592 if datatype is not None:
1578 1593 self.dtype = get_numpy_dtype(datatype)
1579 1594
1580 1595 if not(self.setNextFile()):
1581 1596 print "[Writing] There isn't a next file"
1582 1597 return 0
1583 1598
1584 1599 self.setBlockDimension()
1585 1600
1586 1601 return 1
1587 1602
1588 1603 def run(self, dataOut, **kwargs):
1589 1604
1590 1605 if not(self.isConfig):
1591 1606
1592 1607 self.setup(dataOut, **kwargs)
1593 1608 self.isConfig = True
1594 1609
1595 1610 self.putData()
1596 1611
General Comments 0
You need to be logged in to leave comments. Login now