##// END OF EJS Templates
schain
RSS
merge con online_data_hour
José Chávez -
r1039:a86c6846b78a
parent child
Show More
Side by Side Unified
Context file:
r1039:a86c6846b78a Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1866 +1,1853
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time, datetime
14 14 import traceback
15 15 import zmq
16 16
17 17 try:
18 18 from gevent import sleep
19 19 except:
20 20 from time import sleep
21 21
22 22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24 24
25 25 LOCALTIME = True
26 26
27 27 def isNumber(cad):
28 28 """
29 29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
30 30
31 31 Excepciones:
32 32 Si un determinado string no puede ser convertido a numero
33 33 Input:
34 34 str, string al cual se le analiza para determinar si convertible a un numero o no
35 35
36 36 Return:
37 37 True : si el string es uno numerico
38 38 False : no es un string numerico
39 39 """
40 40 try:
41 41 float( cad )
42 42 return True
43 43 except:
44 44 return False
45 45
46 46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
47 47 """
48 48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
49 49
50 50 Inputs:
51 51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
52 52
53 53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
54 54 segundos contados desde 01/01/1970.
55 55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
56 56 segundos contados desde 01/01/1970.
57 57
58 58 Return:
59 59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
60 60 fecha especificado, de lo contrario retorna False.
61 61
62 62 Excepciones:
63 63 Si el archivo no existe o no puede ser abierto
64 64 Si la cabecera no puede ser leida.
65 65
66 66 """
67 67 basicHeaderObj = BasicHeader(LOCALTIME)
68 68
69 69 try:
70 70 fp = open(filename,'rb')
71 71 except IOError:
72 72 print "The file %s can't be opened" %(filename)
73 73 return 0
74 74
75 75 sts = basicHeaderObj.read(fp)
76 76 fp.close()
77 77
78 78 if not(sts):
79 79 print "Skipping the file %s because it has not a valid header" %(filename)
80 80 return 0
81 81
82 82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
83 83 return 0
84 84
85 85 return 1
86 86
87 87 def isTimeInRange(thisTime, startTime, endTime):
88 88
89 89 if endTime >= startTime:
90 90 if (thisTime < startTime) or (thisTime > endTime):
91 91 return 0
92 92
93 93 return 1
94 94 else:
95 95 if (thisTime < startTime) and (thisTime > endTime):
96 96 return 0
97 97
98 98 return 1
99 99
100 100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
101 101 """
102 102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
103 103
104 104 Inputs:
105 105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
106 106
107 107 startDate : fecha inicial del rango seleccionado en formato datetime.date
108 108
109 109 endDate : fecha final del rango seleccionado en formato datetime.date
110 110
111 111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
112 112
113 113 endTime : tiempo final del rango seleccionado en formato datetime.time
114 114
115 115 Return:
116 116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
117 117 fecha especificado, de lo contrario retorna False.
118 118
119 119 Excepciones:
120 120 Si el archivo no existe o no puede ser abierto
121 121 Si la cabecera no puede ser leida.
122 122
123 123 """
124 124
125 125
126 126 try:
127 127 fp = open(filename,'rb')
128 128 except IOError:
129 129 print "The file %s can't be opened" %(filename)
130 130 return None
131 131
132 132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 133 systemHeaderObj = SystemHeader()
134 134 radarControllerHeaderObj = RadarControllerHeader()
135 135 processingHeaderObj = ProcessingHeader()
136 136
137 137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138 138
139 139 sts = firstBasicHeaderObj.read(fp)
140 140
141 141 if not(sts):
142 142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
143 143 return None
144 144
145 145 if not systemHeaderObj.read(fp):
146 146 return None
147 147
148 148 if not radarControllerHeaderObj.read(fp):
149 149 return None
150 150
151 151 if not processingHeaderObj.read(fp):
152 152 return None
153 153
154 154 filesize = os.path.getsize(filename)
155 155
156 156 offset = processingHeaderObj.blockSize + 24 #header size
157 157
158 158 if filesize <= offset:
159 159 print "[Reading] %s: This file has not enough data" %filename
160 160 return None
161 161
162 162 fp.seek(-offset, 2)
163 163
164 164 sts = lastBasicHeaderObj.read(fp)
165 165
166 166 fp.close()
167 167
168 168 thisDatetime = lastBasicHeaderObj.datatime
169 169 thisTime_last_block = thisDatetime.time()
170 170
171 171 thisDatetime = firstBasicHeaderObj.datatime
172 172 thisDate = thisDatetime.date()
173 173 thisTime_first_block = thisDatetime.time()
174 174
175 175 #General case
176 176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 177 #-----------o----------------------------o-----------
178 178 # startTime endTime
179 179
180 180 if endTime >= startTime:
181 181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 182 return None
183 183
184 184 return thisDatetime
185 185
186 186 #If endTime < startTime then endTime belongs to the next day
187 187
188 188
189 189 #<<<<<<<<<<<o o>>>>>>>>>>>
190 190 #-----------o----------------------------o-----------
191 191 # endTime startTime
192 192
193 193 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 194 return None
195 195
196 196 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 197 return None
198 198
199 199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 200 return None
201 201
202 202 return thisDatetime
203 203
204 204 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 205 """
206 206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207 207
208 208 Inputs:
209 209 folder : nombre completo del directorio.
210 210 Su formato deberia ser "/path_root/?YYYYDDD"
211 211
212 212 siendo:
213 213 YYYY : Anio (ejemplo 2015)
214 214 DDD : Dia del anio (ejemplo 305)
215 215
216 216 startDate : fecha inicial del rango seleccionado en formato datetime.date
217 217
218 218 endDate : fecha final del rango seleccionado en formato datetime.date
219 219
220 220 Return:
221 221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 222 fecha especificado, de lo contrario retorna False.
223 223 Excepciones:
224 224 Si el directorio no tiene el formato adecuado
225 225 """
226 226
227 227 basename = os.path.basename(folder)
228 228
229 229 if not isRadarFolder(basename):
230 230 print "The folder %s has not the rigth format" %folder
231 231 return 0
232 232
233 233 if startDate and endDate:
234 234 thisDate = getDateFromRadarFolder(basename)
235 235
236 236 if thisDate < startDate:
237 237 return 0
238 238
239 239 if thisDate > endDate:
240 240 return 0
241 241
242 242 return 1
243 243
244 244 def isFileInDateRange(filename, startDate=None, endDate=None):
245 245 """
246 246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
247 247
248 248 Inputs:
249 249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
250 250
251 251 Su formato deberia ser "?YYYYDDDsss"
252 252
253 253 siendo:
254 254 YYYY : Anio (ejemplo 2015)
255 255 DDD : Dia del anio (ejemplo 305)
256 256 sss : set
257 257
258 258 startDate : fecha inicial del rango seleccionado en formato datetime.date
259 259
260 260 endDate : fecha final del rango seleccionado en formato datetime.date
261 261
262 262 Return:
263 263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
264 264 fecha especificado, de lo contrario retorna False.
265 265 Excepciones:
266 266 Si el archivo no tiene el formato adecuado
267 267 """
268 268
269 269 basename = os.path.basename(filename)
270 270
271 271 if not isRadarFile(basename):
272 272 print "The filename %s has not the rigth format" %filename
273 273 return 0
274 274
275 275 if startDate and endDate:
276 276 thisDate = getDateFromRadarFile(basename)
277 277
278 278 if thisDate < startDate:
279 279 return 0
280 280
281 281 if thisDate > endDate:
282 282 return 0
283 283
284 284 return 1
285 285
286 286 def getFileFromSet(path, ext, set):
287 287 validFilelist = []
288 288 fileList = os.listdir(path)
289 289
290 290 # 0 1234 567 89A BCDE
291 291 # H YYYY DDD SSS .ext
292 292
293 293 for thisFile in fileList:
294 294 try:
295 295 year = int(thisFile[1:5])
296 296 doy = int(thisFile[5:8])
297 297 except:
298 298 continue
299 299
300 300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
301 301 continue
302 302
303 303 validFilelist.append(thisFile)
304 304
305 305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
306 306
307 307 if len(myfile)!= 0:
308 308 return myfile[0]
309 309 else:
310 310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
311 311 print 'the filename %s does not exist'%filename
312 312 print '...going to the last file: '
313 313
314 314 if validFilelist:
315 315 validFilelist = sorted( validFilelist, key=str.lower )
316 316 return validFilelist[-1]
317 317
318 318 return None
319 319
320 320 def getlastFileFromPath(path, ext):
321 321 """
322 322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
323 323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
324 324
325 325 Input:
326 326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
327 327 ext : extension de los files contenidos en una carpeta
328 328
329 329 Return:
330 330 El ultimo file de una determinada carpeta, no se considera el path.
331 331 """
332 332 validFilelist = []
333 333 fileList = os.listdir(path)
334 334
335 335 # 0 1234 567 89A BCDE
336 336 # H YYYY DDD SSS .ext
337 337
338 338 for thisFile in fileList:
339 339
340 340 year = thisFile[1:5]
341 341 if not isNumber(year):
342 342 continue
343 343
344 344 doy = thisFile[5:8]
345 345 if not isNumber(doy):
346 346 continue
347 347
348 348 year = int(year)
349 349 doy = int(doy)
350 350
351 351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
352 352 continue
353 353
354 354 validFilelist.append(thisFile)
355 355
356 356 if validFilelist:
357 357 validFilelist = sorted( validFilelist, key=str.lower )
358 358 return validFilelist[-1]
359 359
360 360 return None
361 361
362 362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 363 """
364 364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
365 365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
366 366 el path exacto de un determinado file.
367 367
368 368 Example :
369 369 nombre correcto del file es .../.../D2009307/P2009307367.ext
370 370
371 371 Entonces la funcion prueba con las siguientes combinaciones
372 372 .../.../y2009307367.ext
373 373 .../.../Y2009307367.ext
374 374 .../.../x2009307/y2009307367.ext
375 375 .../.../x2009307/Y2009307367.ext
376 376 .../.../X2009307/y2009307367.ext
377 377 .../.../X2009307/Y2009307367.ext
378 378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
379 379
380 380 Return:
381 381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
382 382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
383 383 para el filename
384 384 """
385 385 fullfilename = None
386 386 find_flag = False
387 387 filename = None
388 388
389 389 prefixDirList = [None,'d','D']
390 390 if ext.lower() == ".r": #voltage
391 391 prefixFileList = ['d','D']
392 392 elif ext.lower() == ".pdata": #spectra
393 393 prefixFileList = ['p','P']
394 394 else:
395 395 return None, filename
396 396
397 397 #barrido por las combinaciones posibles
398 398 for prefixDir in prefixDirList:
399 399 thispath = path
400 400 if prefixDir != None:
401 401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
402 402 if foldercounter == 0:
403 403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
404 404 else:
405 405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
406 406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
407 407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
408 408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
409 409
410 410 if os.path.exists( fullfilename ): #verifico que exista
411 411 find_flag = True
412 412 break
413 413 if find_flag:
414 414 break
415 415
416 416 if not(find_flag):
417 417 return None, filename
418 418
419 419 return fullfilename, filename
420 420
421 421 def isRadarFolder(folder):
422 422 try:
423 423 year = int(folder[1:5])
424 424 doy = int(folder[5:8])
425 425 except:
426 426 return 0
427 427
428 428 return 1
429 429
430 430 def isRadarFile(file):
431 431 try:
432 432 year = int(file[1:5])
433 433 doy = int(file[5:8])
434 434 set = int(file[8:11])
435 435 except:
436 436 return 0
437 437
438 438 return 1
439 439
440 440 def getDateFromRadarFile(file):
441 441 try:
442 442 year = int(file[1:5])
443 443 doy = int(file[5:8])
444 444 set = int(file[8:11])
445 445 except:
446 446 return None
447 447
448 448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
449 449 return thisDate
450 450
451 451 def getDateFromRadarFolder(folder):
452 452 try:
453 453 year = int(folder[1:5])
454 454 doy = int(folder[5:8])
455 455 except:
456 456 return None
457 457
458 458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
459 459 return thisDate
460 460
461 461 class JRODataIO:
462 462
463 463 c = 3E8
464 464
465 465 isConfig = False
466 466
467 467 basicHeaderObj = None
468 468
469 469 systemHeaderObj = None
470 470
471 471 radarControllerHeaderObj = None
472 472
473 473 processingHeaderObj = None
474 474
475 475 dtype = None
476 476
477 477 pathList = []
478 478
479 479 filenameList = []
480 480
481 481 filename = None
482 482
483 483 ext = None
484 484
485 485 flagIsNewFile = 1
486 486
487 487 flagDiscontinuousBlock = 0
488 488
489 489 flagIsNewBlock = 0
490 490
491 491 fp = None
492 492
493 493 firstHeaderSize = 0
494 494
495 495 basicHeaderSize = 24
496 496
497 497 versionFile = 1103
498 498
499 499 fileSize = None
500 500
501 501 # ippSeconds = None
502 502
503 503 fileSizeByHeader = None
504 504
505 505 fileIndex = None
506 506
507 507 profileIndex = None
508 508
509 509 blockIndex = None
510 510
511 511 nTotalBlocks = None
512 512
513 513 maxTimeStep = 30
514 514
515 515 lastUTTime = None
516 516
517 517 datablock = None
518 518
519 519 dataOut = None
520 520
521 521 blocksize = None
522 522
523 523 getByBlock = False
524 524
525 525 def __init__(self):
526 526
527 527 raise NotImplementedError
528 528
529 529 def run(self):
530 530
531 531 raise NotImplementedError
532 532
533 533 def getDtypeWidth(self):
534 534
535 535 dtype_index = get_dtype_index(self.dtype)
536 536 dtype_width = get_dtype_width(dtype_index)
537 537
538 538 return dtype_width
539 539
540 540 def getAllowedArgs(self):
541 541 return inspect.getargspec(self.run).args
542 542
543 543 class JRODataReader(JRODataIO):
544 544
545 545 firstTime = True
546 546 online = 0
547 547
548 548 realtime = 0
549 549
550 550 nReadBlocks = 0
551 551
552 552 delay = 10 #number of seconds waiting a new file
553 553
554 554 nTries = 3 #quantity tries
555 555
556 556 nFiles = 3 #number of files for searching
557 557
558 558 path = None
559 559
560 560 foldercounter = 0
561 561
562 562 flagNoMoreFiles = 0
563 563
564 564 datetimeList = []
565 565
566 566 __isFirstTimeOnline = 1
567 567
568 568 __printInfo = True
569 569
570 570 profileIndex = None
571 571
572 572 nTxs = 1
573 573
574 574 txIndex = None
575 575
576 576 #Added--------------------
577 577
578 578 selBlocksize = None
579 579
580 580 selBlocktime = None
581 581
582 582 onlineWithDate = False
583 583 def __init__(self):
584 584
585 585 """
586 586 This class is used to find data files
587 587
588 588 Example:
589 589 reader = JRODataReader()
590 590 fileList = reader.findDataFiles()
591 591
592 592 """
593 593 pass
594 594
595 595
596 596 def createObjByDefault(self):
597 597 """
598 598
599 599 """
600 600 raise NotImplementedError
601 601
602 602 def getBlockDimension(self):
603 603
604 604 raise NotImplementedError
605 605
606 606 def __searchFilesOffLine(self,
607 607 path,
608 608 startDate=None,
609 609 endDate=None,
610 610 startTime=datetime.time(0,0,0),
611 611 endTime=datetime.time(23,59,59),
612 612 set=None,
613 613 expLabel='',
614 614 ext='.r',
615 615 queue=None,
616 616 cursor=None,
617 617 skip=None,
618 618 walk=True):
619 619 self.filenameList = []
620 620 self.datetimeList = []
621 621
622 622 pathList = []
623 623
624 624 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
625 625
626 626 if dateList == []:
627 627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
628 628 return None, None
629 629
630 630 if len(dateList) > 1:
631 631 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
632 632 else:
633 633 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
634 634
635 635 filenameList = []
636 636 datetimeList = []
637 637
638 638 for thisPath in pathList:
639 639 # thisPath = pathList[pathDict[file]]
640 640
641 641 fileList = glob.glob1(thisPath, "*%s" %ext)
642 642 fileList.sort()
643 643
644 644 skippedFileList = []
645 645
646 646 if cursor is not None and skip is not None:
647 647 # if cursor*skip > len(fileList):
648 648 if skip == 0:
649 649 if queue is not None:
650 650 queue.put(len(fileList))
651 651 skippedFileList = []
652 652 else:
653 653 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
654 654
655 655 else:
656 656 skippedFileList = fileList
657 657
658 658 for file in skippedFileList:
659 659
660 660 filename = os.path.join(thisPath,file)
661 661
662 662 if not isFileInDateRange(filename, startDate, endDate):
663 663 continue
664 664
665 665 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
666 666
667 667 if not(thisDatetime):
668 668 continue
669 669
670 670 filenameList.append(filename)
671 671 datetimeList.append(thisDatetime)
672 672
673 673 if not(filenameList):
674 674 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
675 675 return None, None
676 676
677 677 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
678 678 print
679 679
680 680 for i in range(len(filenameList)):
681 681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
682 682
683 683 self.filenameList = filenameList
684 684 self.datetimeList = datetimeList
685 685 return pathList, filenameList
686 686
687 687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
688 688
689 689 """
690 690 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
691 691 devuelve el archivo encontrado ademas de otros datos.
692 692
693 693 Input:
694 694 path : carpeta donde estan contenidos los files que contiene data
695 695
696 696 expLabel : Nombre del subexperimento (subfolder)
697 697
698 698 ext : extension de los files
699 699
700 700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
701 701
702 702 Return:
703 703 directory : eL directorio donde esta el file encontrado
704 704 filename : el ultimo file de una determinada carpeta
705 705 year : el anho
706 706 doy : el numero de dia del anho
707 707 set : el set del archivo
708 708
709 709
710 710 """
711 711 pathList = None
712 712 filenameList = None
713 713 if not os.path.isdir(path):
714 714 return None, None, None, None, None, None
715 715
716 716 dirList = []
717 717
718 718 if not walk:
719 719 fullpath = path
720 720 foldercounter = 0
721 721 else:
722 722 # Filtra solo los directorios
723 723 for thisPath in os.listdir(path):
724 724 if not os.path.isdir(os.path.join(path,thisPath)):
725 725 continue
726 726 if not isRadarFolder(thisPath):
727 727 continue
728 728
729 729 dirList.append(thisPath)
730 730
731 731 if not(dirList):
732 732 return None, None, None, None, None, None
733 733
734 734 dirList = sorted( dirList, key=str.lower )
735 735
736 736 doypath = dirList[-1]
737 737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
738 738 fullpath = os.path.join(path, doypath, expLabel)
739 739
740 740
741 741 print "[Reading] %s folder was found: " %(fullpath )
742 742
743 743 if set == None:
744 744 filename = getlastFileFromPath(fullpath, ext)
745 745 else:
746 746 filename = getFileFromSet(fullpath, ext, set)
747 747
748 748 if not(filename):
749 749 return None, None, None, None, None, None
750 750
751 751 print "[Reading] %s file was found" %(filename)
752 752
753 753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
754 754 return None, None, None, None, None, None
755 755
756 756 year = int( filename[1:5] )
757 757 doy = int( filename[5:8] )
758 758 set = int( filename[8:11] )
759 759
760 760 return fullpath, foldercounter, filename, year, doy, set
761 761
762 762 def __setNextFileOffline(self):
763 763
764 764 idFile = self.fileIndex
765 765
766 766 while (True):
767 767 idFile += 1
768 768 if not(idFile < len(self.filenameList)):
769 769 self.flagNoMoreFiles = 1
770 770 # print "[Reading] No more Files"
771 771 return 0
772 772
773 773 filename = self.filenameList[idFile]
774 774
775 775 if not(self.__verifyFile(filename)):
776 776 continue
777 777
778 778 fileSize = os.path.getsize(filename)
779 779 fp = open(filename,'rb')
780 780 break
781 781
782 782 self.flagIsNewFile = 1
783 783 self.fileIndex = idFile
784 784 self.filename = filename
785 785 self.fileSize = fileSize
786 786 self.fp = fp
787 787
788 788 #print "[Reading] Setting the file: %s"%self.filename
789 789
790 790 return 1
791 791
792 792 def __setNextFileOnline(self):
793 793 """
794 794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 796 siguientes.
797 797
798 798 Affected:
799 799 self.flagIsNewFile
800 800 self.filename
801 801 self.fileSize
802 802 self.fp
803 803 self.set
804 804 self.flagNoMoreFiles
805 805
806 806 Return:
807 807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 808 1 : si el file fue abierto con exito y esta listo a ser leido
809 809
810 810 Excepciones:
811 811 Si un determinado file no puede ser abierto
812 812 """
813 813
814 814 nFiles = 0
815 815 fileOk_flag = False
816 816 firstTime_flag = True
817 817
818 818 self.set += 1
819 819
820 820 if self.set > 999:
821 821 self.set = 0
822 822 self.foldercounter += 1
823 823
824 824 #busca el 1er file disponible
825 825 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
826 826 if fullfilename:
827 827 if self.__verifyFile(fullfilename, False):
828 828 fileOk_flag = True
829 829
830 830 #si no encuentra un file entonces espera y vuelve a buscar
831 831 if not(fileOk_flag):
832 832 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
833 833
834 834 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
835 835 tries = self.nTries
836 836 else:
837 837 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
838 838
839 839 for nTries in range( tries ):
840 840 if firstTime_flag:
841 841 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
842 842 sleep( self.delay )
843 843 else:
844 844 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
845 845
846 846 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
847 847 if fullfilename:
848 848 if self.__verifyFile(fullfilename):
849 849 fileOk_flag = True
850 850 break
851 851
852 852 if fileOk_flag:
853 853 break
854 854
855 855 firstTime_flag = False
856 856
857 857 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
858 858 self.set += 1
859 859
860 860 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
861 861 self.set = 0
862 862 self.doy += 1
863 863 self.foldercounter = 0
864 864
865 865 if fileOk_flag:
866 866 self.fileSize = os.path.getsize( fullfilename )
867 867 self.filename = fullfilename
868 868 self.flagIsNewFile = 1
869 869 if self.fp != None: self.fp.close()
870 870 self.fp = open(fullfilename, 'rb')
871 871 self.flagNoMoreFiles = 0
872 872 # print '[Reading] Setting the file: %s' % fullfilename
873 873 else:
874 874 self.fileSize = 0
875 875 self.filename = None
876 876 self.flagIsNewFile = 0
877 877 self.fp = None
878 878 self.flagNoMoreFiles = 1
879 879 # print '[Reading] No more files to read'
880 880
881 881 return fileOk_flag
882 882
883 883 def setNextFile(self):
884 884 if self.fp != None:
885 885 self.fp.close()
886 886 if self.online:
887 887 newFile = self.__setNextFileOnline()
888 888 else:
889 889 newFile = self.__setNextFileOffline()
890 890 if not(newFile):
891 891 if self.onlineWithDate is True:
892 892 self.onlineWithDate=False
893 893 self.online = True
894 894 self.firstTime = False
895 895 self.setup(
896 896 path=self.path,
897 897 startDate=self.startDate,
898 898 endDate=self.endDate,
899 899 startTime=self.startTime ,
900 900 endTime=self.endTime,
901 901 set=self.set,
902 902 expLabel=self.expLabel,
903 903 ext=self.ext,
904 904 online=self.online,
905 905 delay=self.delay,
906 906 walk=self.walk,
907 907 getblock=self.getblock,
908 908 nTxs=self.nTxs,
909 909 realtime=self.realtime,
910 910 blocksize=self.blocksize,
911 911 blocktime=self.blocktime
912 912 )
913 913 return 1
914 914 print '[Reading] No more files to read'
915 915 return 0
916 916
917 917 if self.verbose:
918 918 print '[Reading] Setting the file: %s' % self.filename
919 919
920 920 self.__readFirstHeader()
921 921 self.nReadBlocks = 0
922 922 return 1
923 923
924 924 def __waitNewBlock(self):
925 925 """
926 926 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
927 927
928 928 Si el modo de lectura es OffLine siempre retorn 0
929 929 """
930 930 if not self.online:
931 931 return 0
932 932
933 933 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
934 934 return 0
935 935
936 936 currentPointer = self.fp.tell()
937 937
938 938 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
939 939
940 940 for nTries in range( self.nTries ):
941 941
942 942 self.fp.close()
943 943 self.fp = open( self.filename, 'rb' )
944 944 self.fp.seek( currentPointer )
945 945
946 946 self.fileSize = os.path.getsize( self.filename )
947 947 currentSize = self.fileSize - currentPointer
948 948
949 949 if ( currentSize >= neededSize ):
950 950 self.basicHeaderObj.read(self.fp)
951 951 return 1
952 952
953 953 if self.fileSize == self.fileSizeByHeader:
954 954 # self.flagEoF = True
955 955 return 0
956 956
957 957 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
958 958 sleep( self.delay )
959 959
960 960
961 961 return 0
962 962
963 963 def waitDataBlock(self,pointer_location):
964 964
965 965 currentPointer = pointer_location
966 966
967 967 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
968 968
969 969 for nTries in range( self.nTries ):
970 970 self.fp.close()
971 971 self.fp = open( self.filename, 'rb' )
972 972 self.fp.seek( currentPointer )
973 973
974 974 self.fileSize = os.path.getsize( self.filename )
975 975 currentSize = self.fileSize - currentPointer
976 976
977 977 if ( currentSize >= neededSize ):
978 978 return 1
979 979
980 980 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
981 981 sleep( self.delay )
982 982
983 983 return 0
984 984
985 985 def __jumpToLastBlock(self):
986 986
987 987 if not(self.__isFirstTimeOnline):
988 988 return
989 989
990 990 csize = self.fileSize - self.fp.tell()
991 991 blocksize = self.processingHeaderObj.blockSize
992 992
993 993 #salta el primer bloque de datos
994 994 if csize > self.processingHeaderObj.blockSize:
995 995 self.fp.seek(self.fp.tell() + blocksize)
996 996 else:
997 997 return
998 998
999 999 csize = self.fileSize - self.fp.tell()
1000 1000 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1001 1001 while True:
1002 1002
1003 1003 if self.fp.tell()<self.fileSize:
1004 1004 self.fp.seek(self.fp.tell() + neededsize)
1005 1005 else:
1006 1006 self.fp.seek(self.fp.tell() - neededsize)
1007 1007 break
1008 1008
1009 1009 # csize = self.fileSize - self.fp.tell()
1010 1010 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1011 1011 # factor = int(csize/neededsize)
1012 1012 # if factor > 0:
1013 1013 # self.fp.seek(self.fp.tell() + factor*neededsize)
1014 1014
1015 1015 self.flagIsNewFile = 0
1016 1016 self.__isFirstTimeOnline = 0
1017 1017
1018 1018 def __setNewBlock(self):
1019 1019 #if self.server is None:
1020 1020 if self.fp == None:
1021 1021 return 0
1022 1022
1023 1023 # if self.online:
1024 1024 # self.__jumpToLastBlock()
1025 1025
1026 1026 if self.flagIsNewFile:
1027 1027 self.lastUTTime = self.basicHeaderObj.utc
1028 1028 return 1
1029 1029
1030 1030 if self.realtime:
1031 1031 self.flagDiscontinuousBlock = 1
1032 1032 if not(self.setNextFile()):
1033 1033 return 0
1034 1034 else:
1035 1035 return 1
1036 1036 #if self.server is None:
1037 1037 currentSize = self.fileSize - self.fp.tell()
1038 1038 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1039 1039 if (currentSize >= neededSize):
1040 1040 self.basicHeaderObj.read(self.fp)
1041 1041 self.lastUTTime = self.basicHeaderObj.utc
1042 1042 return 1
1043 1043 # else:
1044 1044 # self.basicHeaderObj.read(self.zHeader)
1045 1045 # self.lastUTTime = self.basicHeaderObj.utc
1046 1046 # return 1
1047 1047 if self.__waitNewBlock():
1048 1048 self.lastUTTime = self.basicHeaderObj.utc
1049 1049 return 1
1050 1050 #if self.server is None:
1051 1051 if not(self.setNextFile()):
1052 1052 return 0
1053 1053
1054 1054 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1055 1055 self.lastUTTime = self.basicHeaderObj.utc
1056 1056
1057 1057 self.flagDiscontinuousBlock = 0
1058 1058
1059 1059 if deltaTime > self.maxTimeStep:
1060 1060 self.flagDiscontinuousBlock = 1
1061 1061
1062 1062 return 1
1063 1063
1064 1064 def readNextBlock(self):
1065 1065
1066 1066 #Skip block out of startTime and endTime
1067 1067 while True:
1068 1068 if not(self.__setNewBlock()):
1069 1069 print 'returning'
1070 1070 return 0
1071 1071 if not(self.readBlock()):
1072 1072 return 0
1073 1073 self.getBasicHeader()
1074 1074 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1075 1075
1076 1076 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1077 1077 self.processingHeaderObj.dataBlocksPerFile,
1078 1078 self.dataOut.datatime.ctime())
1079 1079 continue
1080 1080
1081 1081 break
1082 1082
1083 1083 if self.verbose:
1084 1084 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1085 1085 self.processingHeaderObj.dataBlocksPerFile,
1086 1086 self.dataOut.datatime.ctime())
1087 1087 return 1
1088 1088
1089 1089 def __readFirstHeader(self):
1090 1090
1091 1091 self.basicHeaderObj.read(self.fp)
1092 1092 self.systemHeaderObj.read(self.fp)
1093 1093 self.radarControllerHeaderObj.read(self.fp)
1094 1094 self.processingHeaderObj.read(self.fp)
1095 1095
1096 1096 self.firstHeaderSize = self.basicHeaderObj.size
1097 1097
1098 1098 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1099 1099 if datatype == 0:
1100 1100 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1101 1101 elif datatype == 1:
1102 1102 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1103 1103 elif datatype == 2:
1104 1104 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1105 1105 elif datatype == 3:
1106 1106 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1107 1107 elif datatype == 4:
1108 1108 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1109 1109 elif datatype == 5:
1110 1110 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1111 1111 else:
1112 1112 raise ValueError, 'Data type was not defined'
1113 1113
1114 1114 self.dtype = datatype_str
1115 1115 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1116 1116 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1117 1117 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1118 1118 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1119 1119 self.getBlockDimension()
1120 1120
1121 1121 def __verifyFile(self, filename, msgFlag=True):
1122 1122
1123 1123 msg = None
1124 1124
1125 1125 try:
1126 1126 fp = open(filename, 'rb')
1127 1127 except IOError:
1128 1128
1129 1129 if msgFlag:
1130 1130 print "[Reading] File %s can't be opened" % (filename)
1131 1131
1132 1132 return False
1133 1133
1134 1134 currentPosition = fp.tell()
1135 1135 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1136 1136
1137 1137 if neededSize == 0:
1138 1138 basicHeaderObj = BasicHeader(LOCALTIME)
1139 1139 systemHeaderObj = SystemHeader()
1140 1140 radarControllerHeaderObj = RadarControllerHeader()
1141 1141 processingHeaderObj = ProcessingHeader()
1142 1142
1143 1143 if not( basicHeaderObj.read(fp) ):
1144 1144 fp.close()
1145 1145 return False
1146 1146
1147 1147 if not( systemHeaderObj.read(fp) ):
1148 1148 fp.close()
1149 1149 return False
1150 1150
1151 1151 if not( radarControllerHeaderObj.read(fp) ):
1152 1152 fp.close()
1153 1153 return False
1154 1154
1155 1155 if not( processingHeaderObj.read(fp) ):
1156 1156 fp.close()
1157 1157 return False
1158 1158
1159 1159 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1160 1160 else:
1161 1161 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1162 1162
1163 1163 fp.close()
1164 1164
1165 1165 fileSize = os.path.getsize(filename)
1166 1166 currentSize = fileSize - currentPosition
1167 1167
1168 1168 if currentSize < neededSize:
1169 1169 if msgFlag and (msg != None):
1170 1170 print msg
1171 1171 return False
1172 1172
1173 1173 return True
1174 1174
1175 1175 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1176 1176
1177 1177 path_empty = True
1178 1178
1179 1179 dateList = []
1180 1180 pathList = []
1181 1181
1182 1182 multi_path = path.split(',')
1183 1183
1184 1184 if not walk:
1185 1185
1186 1186 for single_path in multi_path:
1187 1187
1188 1188 if not os.path.isdir(single_path):
1189 1189 continue
1190 1190
1191 1191 fileList = glob.glob1(single_path, "*"+ext)
1192 1192
1193 1193 if not fileList:
1194 1194 continue
1195 1195
1196 1196 path_empty = False
1197 1197
1198 1198 fileList.sort()
1199 1199
1200 1200 for thisFile in fileList:
1201 1201
1202 1202 if not os.path.isfile(os.path.join(single_path, thisFile)):
1203 1203 continue
1204 1204
1205 1205 if not isRadarFile(thisFile):
1206 1206 continue
1207 1207
1208 1208 if not isFileInDateRange(thisFile, startDate, endDate):
1209 1209 continue
1210 1210
1211 1211 thisDate = getDateFromRadarFile(thisFile)
1212 1212
1213 1213 if thisDate in dateList:
1214 1214 continue
1215 1215
1216 1216 dateList.append(thisDate)
1217 1217 pathList.append(single_path)
1218 1218
1219 1219 else:
1220 1220 for single_path in multi_path:
1221 1221
1222 1222 if not os.path.isdir(single_path):
1223 1223 continue
1224 1224
1225 1225 dirList = []
1226 1226
1227 1227 for thisPath in os.listdir(single_path):
1228 1228
1229 1229 if not os.path.isdir(os.path.join(single_path,thisPath)):
1230 1230 continue
1231 1231
1232 1232 if not isRadarFolder(thisPath):
1233 1233 continue
1234 1234
1235 1235 if not isFolderInDateRange(thisPath, startDate, endDate):
1236 1236 continue
1237 1237
1238 1238 dirList.append(thisPath)
1239 1239
1240 1240 if not dirList:
1241 1241 continue
1242 1242
1243 1243 dirList.sort()
1244 1244
1245 1245 for thisDir in dirList:
1246 1246
1247 1247 datapath = os.path.join(single_path, thisDir, expLabel)
1248 1248 fileList = glob.glob1(datapath, "*"+ext)
1249 1249
1250 1250 if not fileList:
1251 1251 continue
1252 1252
1253 1253 path_empty = False
1254 1254
1255 1255 thisDate = getDateFromRadarFolder(thisDir)
1256 1256
1257 1257 pathList.append(datapath)
1258 1258 dateList.append(thisDate)
1259 1259
1260 1260 dateList.sort()
1261 1261
1262 1262 if walk:
1263 1263 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1264 1264 else:
1265 1265 pattern_path = multi_path[0]
1266 1266
1267 1267 if path_empty:
1268 1268 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1269 1269 else:
1270 1270 if not dateList:
1271 1271 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1272 1272
1273 1273 if include_path:
1274 1274 return dateList, pathList
1275 1275
1276 1276 return dateList
1277 1277
1278 1278 def setup(self,
1279 1279 path=None,
1280 1280 startDate=None,
1281 1281 endDate=None,
1282 1282 startTime=datetime.time(0,0,0),
1283 1283 endTime=datetime.time(23,59,59),
1284 1284 set=None,
1285 1285 expLabel = "",
1286 1286 ext = None,
1287 1287 online = False,
1288 1288 delay = 60,
1289 1289 walk = True,
1290 1290 getblock = False,
1291 1291 nTxs = 1,
1292 1292 realtime=False,
1293 1293 blocksize=None,
1294 1294 blocktime=None):
1295 1295
1296 1296 if path == None:
1297 1297 raise ValueError, "[Reading] The path is not valid"
1298 1298
1299 1299
1300 1300 if ext == None:
1301 1301 ext = self.ext
1302 1302
1303 1303 self.path = path
1304 1304 self.startDate = startDate
1305 1305 self.endDate = endDate
1306 1306 self.startTime = startTime
1307 1307 self.endTime = endTime
1308 1308 self.set = set
1309 1309 self.expLabel = expLabel
1310 1310 self.ext = ext
1311 1311 self.online = online
1312 1312 self.delay = delay
1313 1313 self.walk = walk
1314 1314 self.getblock = getblock
1315 1315 self.nTxs = nTxs
1316 1316 self.realtime = realtime
1317 1317 self.blocksize = blocksize
1318 1318 self.blocktime = blocktime
1319 1319
1320 1320
1321 1321 if self.firstTime is True:
1322 1322 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1323 1323 startTime=startTime, endTime=endTime,
1324 1324 set=set, expLabel=expLabel, ext=ext,
1325 1325 walk=walk)
1326 1326 filenameList = filenameList[:-1]
1327 1327
1328 1328 if pathList is not None and filenameList is not None and online:
1329 1329 self.onlineWithDate = True
1330 1330 online = False
1331 1331 self.fileIndex = -1
1332 1332 self.pathList = pathList
1333 1333 self.filenameList = filenameList
1334 1334 file_name = os.path.basename(filenameList[-1])
1335 1335 basename, ext = os.path.splitext(file_name)
1336 1336 last_set = int(basename[-3:])
1337 1337
1338 1338 if online:
1339 1339 print "[Reading] Searching files in online mode..."
1340 1340
1341 1341 for nTries in range(self.nTries):
1342 1342 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1343 1343 expLabel=expLabel,
1344 1344 ext=ext,
1345 1345 walk=walk,
1346 1346 startDate=startDate,
1347 1347 startTime=startTime,
1348 1348 set=set)
1349 1349
1350 1350 if fullpath:
1351 1351 break
1352 1352 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1353 1353 sleep( self.delay )
1354 1354
1355 1355 if not(fullpath):
1356 1356 print "[Reading] There 'isn't any valid file in %s" % path
1357 1357 return
1358 1358
1359 1359 self.year = year
1360 1360 self.doy = doy
1361 1361 self.set = set - 1
1362 1362 self.path = path
1363 1363 self.foldercounter = foldercounter
1364 1364 last_set = None
1365 1365
1366 1366 else:
1367 1367 print "[Reading] Searching files in offline mode ..."
1368 1368 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1369 1369 startTime=startTime, endTime=endTime,
1370 1370 set=set, expLabel=expLabel, ext=ext,
1371 1371 walk=walk)
1372 1372
1373 1373 if not(pathList):
1374 1374 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1375 1375 # datetime.datetime.combine(startDate,startTime).ctime(),
1376 1376 # datetime.datetime.combine(endDate,endTime).ctime())
1377 1377
1378 1378 # sys.exit(-1)
1379 1379
1380 1380 self.fileIndex = -1
1381 1381 self.pathList = []
1382 1382 self.filenameList = []
1383 1383 return
1384 1384
1385 1385 self.fileIndex = -1
1386 1386 self.pathList = pathList
1387 1387 self.filenameList = filenameList
1388 1388 file_name = os.path.basename(filenameList[-1])
1389 1389 basename, ext = os.path.splitext(file_name)
1390 1390 last_set = int(basename[-3:])
1391 1391
1392 1392
1393 1393 self.online = online
1394 1394 self.realtime = realtime
1395 1395 self.delay = delay
1396 1396 ext = ext.lower()
1397 1397 self.ext = ext
1398 1398 self.getByBlock = getblock
1399 1399 self.nTxs = nTxs
1400 1400 self.startTime = startTime
1401 1401 self.endTime = endTime
1402 1402
1403 1403
1404 1404 #Added-----------------
1405 1405 self.selBlocksize = blocksize
1406 1406 self.selBlocktime = blocktime
1407 1407
1408 1408
1409 1409 if not(self.setNextFile()):
1410 1410 if (startDate!=None) and (endDate!=None):
1411 1411 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1412 1412 elif startDate != None:
1413 1413 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1414 1414 else:
1415 1415 print "[Reading] No files"
1416 1416
1417 1417 self.fileIndex = -1
1418 1418 self.pathList = []
1419 1419 self.filenameList = []
1420 1420 return
1421 1421
1422 1422 # self.getBasicHeader()
1423 1423
1424 1424 if last_set != None:
1425 1425 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1426 1426 return
1427 1427
1428 1428 def getBasicHeader(self):
1429 1429
1430 1430 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1431 1431
1432 1432 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1433 1433
1434 1434 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1435 1435
1436 1436 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1437 1437
1438 1438 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1439 1439
1440 1440 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1441 1441
1442 1442 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1443 1443
1444 1444 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1445 1445
1446 1446
1447 1447 def getFirstHeader(self):
1448 1448
1449 1449 raise NotImplementedError
1450 1450
1451 1451 def getData(self):
1452 1452
1453 1453 raise NotImplementedError
1454 1454
1455 1455 def hasNotDataInBuffer(self):
1456 1456
1457 1457 raise NotImplementedError
1458 1458
1459 1459 def readBlock(self):
1460 1460
1461 1461 raise NotImplementedError
1462 1462
1463 1463 def isEndProcess(self):
1464 1464
1465 1465 return self.flagNoMoreFiles
1466 1466
1467 1467 def printReadBlocks(self):
1468 1468
1469 1469 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1470 1470
1471 1471 def printTotalBlocks(self):
1472 1472
1473 1473 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1474 1474
1475 1475 def printNumberOfBlock(self):
1476 1476
1477 1477 if self.flagIsNewBlock:
1478 1478 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1479 1479 self.processingHeaderObj.dataBlocksPerFile,
1480 1480 self.dataOut.datatime.ctime())
1481 1481
1482 1482 def printInfo(self):
1483 1483
1484 1484 if self.__printInfo == False:
1485 1485 return
1486 1486
1487 1487 self.basicHeaderObj.printInfo()
1488 1488 self.systemHeaderObj.printInfo()
1489 1489 self.radarControllerHeaderObj.printInfo()
1490 1490 self.processingHeaderObj.printInfo()
1491 1491
1492 1492 self.__printInfo = False
1493 1493
1494
1495 1494 def run(self,
1496 1495 path=None,
1497 1496 startDate=None,
1498 1497 endDate=None,
1499 1498 startTime=datetime.time(0,0,0),
1500 1499 endTime=datetime.time(23,59,59),
1501 1500 set=None,
1502 1501 expLabel = "",
1503 1502 ext = None,
1504 1503 online = False,
1505 1504 delay = 60,
1506 1505 walk = True,
1507 1506 getblock = False,
1508 1507 nTxs = 1,
1509 1508 realtime=False,
1510 1509 blocksize=None,
1511 1510 blocktime=None,
1512 1511 queue=None,
1513 1512 skip=None,
1514 1513 cursor=None,
1515 1514 warnings=True,
1516 1515 server=None,
1517 1516 verbose=True, **kwargs):
1518 1517
1519 1518 if not(self.isConfig):
1520 <<<<<<< HEAD
1521 1519 # self.dataOut = dataOut
1522 1520 self.setup( path=path,
1523 1521 startDate=startDate,
1524 1522 endDate=endDate,
1525 1523 startTime=startTime,
1526 1524 endTime=endTime,
1527 1525 set=set,
1528 1526 expLabel=expLabel,
1529 1527 ext=ext,
1530 1528 online=online,
1531 1529 delay=delay,
1532 1530 walk=walk,
1533 1531 getblock=getblock,
1534 1532 nTxs=nTxs,
1535 1533 realtime=realtime,
1536 1534 blocksize=blocksize,
1537 1535 blocktime=blocktime,
1538 1536 queue=queue,
1539 1537 skip=skip,
1540 1538 cursor=cursor,
1541 1539 warnings=warnings,
1542 1540 server=server,
1543 verbose=verbose)
1544 =======
1545
1546 # self.dataOut = dataOut
1547 self.setup(**kwargs)
1548 >>>>>>> online_data_hour
1541 verbose=verbose, **kwargs)
1549 1542 self.isConfig = True
1550 1543 if server is None:
1551 1544 self.getData()
1552 1545 else:
1553 1546 self.getFromServer()
1554 1547
1555 1548 class JRODataWriter(JRODataIO):
1556 1549
1557 1550 """
1558 1551 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1559 1552 de los datos siempre se realiza por bloques.
1560 1553 """
1561 1554
1562 1555 blockIndex = 0
1563 1556
1564 1557 path = None
1565 1558
1566 1559 setFile = None
1567 1560
1568 1561 profilesPerBlock = None
1569 1562
1570 1563 blocksPerFile = None
1571 1564
1572 1565 nWriteBlocks = 0
1573 1566
1574 1567 fileDate = None
1575 1568
1576 1569 def __init__(self, dataOut=None):
1577 1570 raise NotImplementedError
1578 1571
1579 1572
1580 1573 def hasAllDataInBuffer(self):
1581 1574 raise NotImplementedError
1582 1575
1583 1576
1584 1577 def setBlockDimension(self):
1585 1578 raise NotImplementedError
1586 1579
1587 1580
1588 1581 def writeBlock(self):
1589 1582 raise NotImplementedError
1590 1583
1591 1584
1592 1585 def putData(self):
1593 1586 raise NotImplementedError
1594 1587
1595 1588
1596 1589 def getProcessFlags(self):
1597 1590
1598 1591 processFlags = 0
1599 1592
1600 1593 dtype_index = get_dtype_index(self.dtype)
1601 1594 procflag_dtype = get_procflag_dtype(dtype_index)
1602 1595
1603 1596 processFlags += procflag_dtype
1604 1597
1605 1598 if self.dataOut.flagDecodeData:
1606 1599 processFlags += PROCFLAG.DECODE_DATA
1607 1600
1608 1601 if self.dataOut.flagDeflipData:
1609 1602 processFlags += PROCFLAG.DEFLIP_DATA
1610 1603
1611 1604 if self.dataOut.code is not None:
1612 1605 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1613 1606
1614 1607 if self.dataOut.nCohInt > 1:
1615 1608 processFlags += PROCFLAG.COHERENT_INTEGRATION
1616 1609
1617 1610 if self.dataOut.type == "Spectra":
1618 1611 if self.dataOut.nIncohInt > 1:
1619 1612 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1620 1613
1621 1614 if self.dataOut.data_dc is not None:
1622 1615 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1623 1616
1624 1617 if self.dataOut.flagShiftFFT:
1625 1618 processFlags += PROCFLAG.SHIFT_FFT_DATA
1626 1619
1627 1620 return processFlags
1628 1621
1629 1622 def setBasicHeader(self):
1630 1623
1631 1624 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1632 1625 self.basicHeaderObj.version = self.versionFile
1633 1626 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1634 1627
1635 1628 utc = numpy.floor(self.dataOut.utctime)
1636 1629 milisecond = (self.dataOut.utctime - utc)* 1000.0
1637 1630
1638 1631 self.basicHeaderObj.utc = utc
1639 1632 self.basicHeaderObj.miliSecond = milisecond
1640 1633 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1641 1634 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1642 1635 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1643 1636
1644 1637 def setFirstHeader(self):
1645 1638 """
1646 1639 Obtiene una copia del First Header
1647 1640
1648 1641 Affected:
1649 1642
1650 1643 self.basicHeaderObj
1651 1644 self.systemHeaderObj
1652 1645 self.radarControllerHeaderObj
1653 1646 self.processingHeaderObj self.
1654 1647
1655 1648 Return:
1656 1649 None
1657 1650 """
1658 1651
1659 1652 raise NotImplementedError
1660 1653
1661 1654 def __writeFirstHeader(self):
1662 1655 """
1663 1656 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1664 1657
1665 1658 Affected:
1666 1659 __dataType
1667 1660
1668 1661 Return:
1669 1662 None
1670 1663 """
1671 <<<<<<< HEAD
1672
1673 # CALCULAR PARAMETROS
1674
1675 =======
1676 1664
1677 1665 # CALCULAR PARAMETROS
1678 1666
1679 >>>>>>> online_data_hour
1680 1667 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1681 1668 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1682 1669
1683 1670 self.basicHeaderObj.write(self.fp)
1684 1671 self.systemHeaderObj.write(self.fp)
1685 1672 self.radarControllerHeaderObj.write(self.fp)
1686 1673 self.processingHeaderObj.write(self.fp)
1687 1674
1688 1675 def __setNewBlock(self):
1689 1676 """
1690 1677 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1691 1678
1692 1679 Return:
1693 1680 0 : si no pudo escribir nada
1694 1681 1 : Si escribio el Basic el First Header
1695 1682 """
1696 1683 if self.fp == None:
1697 1684 self.setNextFile()
1698 1685
1699 1686 if self.flagIsNewFile:
1700 1687 return 1
1701 1688
1702 1689 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1703 1690 self.basicHeaderObj.write(self.fp)
1704 1691 return 1
1705 1692
1706 1693 if not( self.setNextFile() ):
1707 1694 return 0
1708 1695
1709 1696 return 1
1710 1697
1711 1698
1712 1699 def writeNextBlock(self):
1713 1700 """
1714 1701 Selecciona el bloque siguiente de datos y los escribe en un file
1715 1702
1716 1703 Return:
1717 1704 0 : Si no hizo pudo escribir el bloque de datos
1718 1705 1 : Si no pudo escribir el bloque de datos
1719 1706 """
1720 1707 if not( self.__setNewBlock() ):
1721 1708 return 0
1722 1709
1723 1710 self.writeBlock()
1724 1711
1725 1712 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1726 1713 self.processingHeaderObj.dataBlocksPerFile)
1727 1714
1728 1715 return 1
1729 1716
1730 1717 def setNextFile(self):
1731 1718 """
1732 1719 Determina el siguiente file que sera escrito
1733 1720
1734 1721 Affected:
1735 1722 self.filename
1736 1723 self.subfolder
1737 1724 self.fp
1738 1725 self.setFile
1739 1726 self.flagIsNewFile
1740 1727
1741 1728 Return:
1742 1729 0 : Si el archivo no puede ser escrito
1743 1730 1 : Si el archivo esta listo para ser escrito
1744 1731 """
1745 1732 ext = self.ext
1746 1733 path = self.path
1747 1734
1748 1735 if self.fp != None:
1749 1736 self.fp.close()
1750 1737
1751 1738 timeTuple = time.localtime( self.dataOut.utctime)
1752 1739 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1753 1740
1754 1741 fullpath = os.path.join( path, subfolder )
1755 1742 setFile = self.setFile
1756 1743
1757 1744 if not( os.path.exists(fullpath) ):
1758 1745 os.mkdir(fullpath)
1759 1746 setFile = -1 #inicializo mi contador de seteo
1760 1747 else:
1761 1748 filesList = os.listdir( fullpath )
1762 1749 if len( filesList ) > 0:
1763 1750 filesList = sorted( filesList, key=str.lower )
1764 1751 filen = filesList[-1]
1765 1752 # el filename debera tener el siguiente formato
1766 1753 # 0 1234 567 89A BCDE (hex)
1767 1754 # x YYYY DDD SSS .ext
1768 1755 if isNumber( filen[8:11] ):
1769 1756 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1770 1757 else:
1771 1758 setFile = -1
1772 1759 else:
1773 1760 setFile = -1 #inicializo mi contador de seteo
1774 1761
1775 1762 setFile += 1
1776 1763
1777 1764 #If this is a new day it resets some values
1778 1765 if self.dataOut.datatime.date() > self.fileDate:
1779 1766 setFile = 0
1780 1767 self.nTotalBlocks = 0
1781 1768
1782 1769 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1783 1770
1784 1771 filename = os.path.join( path, subfolder, filen )
1785 1772
1786 1773 fp = open( filename,'wb' )
1787 1774
1788 1775 self.blockIndex = 0
1789 1776
1790 1777 #guardando atributos
1791 1778 self.filename = filename
1792 1779 self.subfolder = subfolder
1793 1780 self.fp = fp
1794 1781 self.setFile = setFile
1795 1782 self.flagIsNewFile = 1
1796 1783 self.fileDate = self.dataOut.datatime.date()
1797 1784
1798 1785 self.setFirstHeader()
1799 1786
1800 1787 print '[Writing] Opening file: %s'%self.filename
1801 1788
1802 1789 self.__writeFirstHeader()
1803 1790
1804 1791 return 1
1805 1792
1806 1793 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1807 1794 """
1808 1795 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1809 1796
1810 1797 Inputs:
1811 1798 path : directory where data will be saved
1812 1799 profilesPerBlock : number of profiles per block
1813 1800 set : initial file set
1814 1801 datatype : An integer number that defines data type:
1815 1802 0 : int8 (1 byte)
1816 1803 1 : int16 (2 bytes)
1817 1804 2 : int32 (4 bytes)
1818 1805 3 : int64 (8 bytes)
1819 1806 4 : float32 (4 bytes)
1820 1807 5 : double64 (8 bytes)
1821 1808
1822 1809 Return:
1823 1810 0 : Si no realizo un buen seteo
1824 1811 1 : Si realizo un buen seteo
1825 1812 """
1826 1813
1827 1814 if ext == None:
1828 1815 ext = self.ext
1829 1816
1830 1817 self.ext = ext.lower()
1831 1818
1832 1819 self.path = path
1833 1820
1834 1821 if set is None:
1835 1822 self.setFile = -1
1836 1823 else:
1837 1824 self.setFile = set - 1
1838 1825
1839 1826 self.blocksPerFile = blocksPerFile
1840 1827
1841 1828 self.profilesPerBlock = profilesPerBlock
1842 1829
1843 1830 self.dataOut = dataOut
1844 1831 self.fileDate = self.dataOut.datatime.date()
1845 1832 #By default
1846 1833 self.dtype = self.dataOut.dtype
1847 1834
1848 1835 if datatype is not None:
1849 1836 self.dtype = get_numpy_dtype(datatype)
1850 1837
1851 1838 if not(self.setNextFile()):
1852 1839 print "[Writing] There isn't a next file"
1853 1840 return 0
1854 1841
1855 1842 self.setBlockDimension()
1856 1843
1857 1844 return 1
1858 1845
1859 1846 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1860 1847
1861 1848 if not(self.isConfig):
1862 1849
1863 1850 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1864 1851 self.isConfig = True
1865 1852
1866 1853 self.putData()
General Comments 0
You need to be logged in to leave comments. Login now