##// END OF EJS Templates
schain
RSS
base primer
José Chávez -
r1038:2da89ff4f915
parent child
Show More
Side by Side Unified
Context file:
r1038:2da89ff4f915 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1976 +1,1866
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time, datetime
14 14 import traceback
15 15 import zmq
16 16
17 17 try:
18 18 from gevent import sleep
19 19 except:
20 20 from time import sleep
21 21
22 22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24 24
25 25 LOCALTIME = True
26 26
27 27 def isNumber(cad):
28 28 """
29 29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
30 30
31 31 Excepciones:
32 32 Si un determinado string no puede ser convertido a numero
33 33 Input:
34 34 str, string al cual se le analiza para determinar si convertible a un numero o no
35 35
36 36 Return:
37 37 True : si el string es uno numerico
38 38 False : no es un string numerico
39 39 """
40 40 try:
41 41 float( cad )
42 42 return True
43 43 except:
44 44 return False
45 45
46 46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
47 47 """
48 48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
49 49
50 50 Inputs:
51 51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
52 52
53 53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
54 54 segundos contados desde 01/01/1970.
55 55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
56 56 segundos contados desde 01/01/1970.
57 57
58 58 Return:
59 59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
60 60 fecha especificado, de lo contrario retorna False.
61 61
62 62 Excepciones:
63 63 Si el archivo no existe o no puede ser abierto
64 64 Si la cabecera no puede ser leida.
65 65
66 66 """
67 67 basicHeaderObj = BasicHeader(LOCALTIME)
68 68
69 69 try:
70 70 fp = open(filename,'rb')
71 71 except IOError:
72 72 print "The file %s can't be opened" %(filename)
73 73 return 0
74 74
75 75 sts = basicHeaderObj.read(fp)
76 76 fp.close()
77 77
78 78 if not(sts):
79 79 print "Skipping the file %s because it has not a valid header" %(filename)
80 80 return 0
81 81
82 82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
83 83 return 0
84 84
85 85 return 1
86 86
87 87 def isTimeInRange(thisTime, startTime, endTime):
88 88
89 89 if endTime >= startTime:
90 90 if (thisTime < startTime) or (thisTime > endTime):
91 91 return 0
92 92
93 93 return 1
94 94 else:
95 95 if (thisTime < startTime) and (thisTime > endTime):
96 96 return 0
97 97
98 98 return 1
99 99
100 100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
101 101 """
102 102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
103 103
104 104 Inputs:
105 105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
106 106
107 107 startDate : fecha inicial del rango seleccionado en formato datetime.date
108 108
109 109 endDate : fecha final del rango seleccionado en formato datetime.date
110 110
111 111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
112 112
113 113 endTime : tiempo final del rango seleccionado en formato datetime.time
114 114
115 115 Return:
116 116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
117 117 fecha especificado, de lo contrario retorna False.
118 118
119 119 Excepciones:
120 120 Si el archivo no existe o no puede ser abierto
121 121 Si la cabecera no puede ser leida.
122 122
123 123 """
124 124
125 125
126 126 try:
127 127 fp = open(filename,'rb')
128 128 except IOError:
129 129 print "The file %s can't be opened" %(filename)
130 130 return None
131 131
132 132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 133 systemHeaderObj = SystemHeader()
134 134 radarControllerHeaderObj = RadarControllerHeader()
135 135 processingHeaderObj = ProcessingHeader()
136 136
137 137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138 138
139 139 sts = firstBasicHeaderObj.read(fp)
140 140
141 141 if not(sts):
142 142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
143 143 return None
144 144
145 145 if not systemHeaderObj.read(fp):
146 146 return None
147 147
148 148 if not radarControllerHeaderObj.read(fp):
149 149 return None
150 150
151 151 if not processingHeaderObj.read(fp):
152 152 return None
153 153
154 154 filesize = os.path.getsize(filename)
155 155
156 156 offset = processingHeaderObj.blockSize + 24 #header size
157 157
158 158 if filesize <= offset:
159 159 print "[Reading] %s: This file has not enough data" %filename
160 160 return None
161 161
162 162 fp.seek(-offset, 2)
163 163
164 164 sts = lastBasicHeaderObj.read(fp)
165 165
166 166 fp.close()
167 167
168 168 thisDatetime = lastBasicHeaderObj.datatime
169 169 thisTime_last_block = thisDatetime.time()
170 170
171 171 thisDatetime = firstBasicHeaderObj.datatime
172 172 thisDate = thisDatetime.date()
173 173 thisTime_first_block = thisDatetime.time()
174 174
175 175 #General case
176 176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 177 #-----------o----------------------------o-----------
178 178 # startTime endTime
179 179
180 180 if endTime >= startTime:
181 181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 182 return None
183 183
184 184 return thisDatetime
185 185
186 186 #If endTime < startTime then endTime belongs to the next day
187 187
188 188
189 189 #<<<<<<<<<<<o o>>>>>>>>>>>
190 190 #-----------o----------------------------o-----------
191 191 # endTime startTime
192 192
193 193 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 194 return None
195 195
196 196 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 197 return None
198 198
199 199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 200 return None
201 201
202 202 return thisDatetime
203 203
204 204 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 205 """
206 206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207 207
208 208 Inputs:
209 209 folder : nombre completo del directorio.
210 210 Su formato deberia ser "/path_root/?YYYYDDD"
211 211
212 212 siendo:
213 213 YYYY : Anio (ejemplo 2015)
214 214 DDD : Dia del anio (ejemplo 305)
215 215
216 216 startDate : fecha inicial del rango seleccionado en formato datetime.date
217 217
218 218 endDate : fecha final del rango seleccionado en formato datetime.date
219 219
220 220 Return:
221 221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 222 fecha especificado, de lo contrario retorna False.
223 223 Excepciones:
224 224 Si el directorio no tiene el formato adecuado
225 225 """
226 226
227 227 basename = os.path.basename(folder)
228 228
229 229 if not isRadarFolder(basename):
230 230 print "The folder %s has not the rigth format" %folder
231 231 return 0
232 232
233 233 if startDate and endDate:
234 234 thisDate = getDateFromRadarFolder(basename)
235 235
236 236 if thisDate < startDate:
237 237 return 0
238 238
239 239 if thisDate > endDate:
240 240 return 0
241 241
242 242 return 1
243 243
244 244 def isFileInDateRange(filename, startDate=None, endDate=None):
245 245 """
246 246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
247 247
248 248 Inputs:
249 249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
250 250
251 251 Su formato deberia ser "?YYYYDDDsss"
252 252
253 253 siendo:
254 254 YYYY : Anio (ejemplo 2015)
255 255 DDD : Dia del anio (ejemplo 305)
256 256 sss : set
257 257
258 258 startDate : fecha inicial del rango seleccionado en formato datetime.date
259 259
260 260 endDate : fecha final del rango seleccionado en formato datetime.date
261 261
262 262 Return:
263 263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
264 264 fecha especificado, de lo contrario retorna False.
265 265 Excepciones:
266 266 Si el archivo no tiene el formato adecuado
267 267 """
268 268
269 269 basename = os.path.basename(filename)
270 270
271 271 if not isRadarFile(basename):
272 272 print "The filename %s has not the rigth format" %filename
273 273 return 0
274 274
275 275 if startDate and endDate:
276 276 thisDate = getDateFromRadarFile(basename)
277 277
278 278 if thisDate < startDate:
279 279 return 0
280 280
281 281 if thisDate > endDate:
282 282 return 0
283 283
284 284 return 1
285 285
286 286 def getFileFromSet(path, ext, set):
287 287 validFilelist = []
288 288 fileList = os.listdir(path)
289 289
290 290 # 0 1234 567 89A BCDE
291 291 # H YYYY DDD SSS .ext
292 292
293 293 for thisFile in fileList:
294 294 try:
295 295 year = int(thisFile[1:5])
296 296 doy = int(thisFile[5:8])
297 297 except:
298 298 continue
299 299
300 300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
301 301 continue
302 302
303 303 validFilelist.append(thisFile)
304 304
305 305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
306 306
307 307 if len(myfile)!= 0:
308 308 return myfile[0]
309 309 else:
310 310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
311 311 print 'the filename %s does not exist'%filename
312 312 print '...going to the last file: '
313 313
314 314 if validFilelist:
315 315 validFilelist = sorted( validFilelist, key=str.lower )
316 316 return validFilelist[-1]
317 317
318 318 return None
319 319
320 320 def getlastFileFromPath(path, ext):
321 321 """
322 322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
323 323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
324 324
325 325 Input:
326 326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
327 327 ext : extension de los files contenidos en una carpeta
328 328
329 329 Return:
330 330 El ultimo file de una determinada carpeta, no se considera el path.
331 331 """
332 332 validFilelist = []
333 333 fileList = os.listdir(path)
334 334
335 335 # 0 1234 567 89A BCDE
336 336 # H YYYY DDD SSS .ext
337 337
338 338 for thisFile in fileList:
339 339
340 340 year = thisFile[1:5]
341 341 if not isNumber(year):
342 342 continue
343 343
344 344 doy = thisFile[5:8]
345 345 if not isNumber(doy):
346 346 continue
347 347
348 348 year = int(year)
349 349 doy = int(doy)
350 350
351 351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
352 352 continue
353 353
354 354 validFilelist.append(thisFile)
355 355
356 356 if validFilelist:
357 357 validFilelist = sorted( validFilelist, key=str.lower )
358 358 return validFilelist[-1]
359 359
360 360 return None
361 361
362 362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 363 """
364 364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
365 365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
366 366 el path exacto de un determinado file.
367 367
368 368 Example :
369 369 nombre correcto del file es .../.../D2009307/P2009307367.ext
370 370
371 371 Entonces la funcion prueba con las siguientes combinaciones
372 372 .../.../y2009307367.ext
373 373 .../.../Y2009307367.ext
374 374 .../.../x2009307/y2009307367.ext
375 375 .../.../x2009307/Y2009307367.ext
376 376 .../.../X2009307/y2009307367.ext
377 377 .../.../X2009307/Y2009307367.ext
378 378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
379 379
380 380 Return:
381 381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
382 382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
383 383 para el filename
384 384 """
385 385 fullfilename = None
386 386 find_flag = False
387 387 filename = None
388 388
389 389 prefixDirList = [None,'d','D']
390 390 if ext.lower() == ".r": #voltage
391 391 prefixFileList = ['d','D']
392 392 elif ext.lower() == ".pdata": #spectra
393 393 prefixFileList = ['p','P']
394 394 else:
395 395 return None, filename
396 396
397 397 #barrido por las combinaciones posibles
398 398 for prefixDir in prefixDirList:
399 399 thispath = path
400 400 if prefixDir != None:
401 401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
402 402 if foldercounter == 0:
403 403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
404 404 else:
405 405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
406 406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
407 407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
408 408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
409 409
410 410 if os.path.exists( fullfilename ): #verifico que exista
411 411 find_flag = True
412 412 break
413 413 if find_flag:
414 414 break
415 415
416 416 if not(find_flag):
417 417 return None, filename
418 418
419 419 return fullfilename, filename
420 420
421 421 def isRadarFolder(folder):
422 422 try:
423 423 year = int(folder[1:5])
424 424 doy = int(folder[5:8])
425 425 except:
426 426 return 0
427 427
428 428 return 1
429 429
430 430 def isRadarFile(file):
431 431 try:
432 432 year = int(file[1:5])
433 433 doy = int(file[5:8])
434 434 set = int(file[8:11])
435 435 except:
436 436 return 0
437 437
438 438 return 1
439 439
440 440 def getDateFromRadarFile(file):
441 441 try:
442 442 year = int(file[1:5])
443 443 doy = int(file[5:8])
444 444 set = int(file[8:11])
445 445 except:
446 446 return None
447 447
448 448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
449 449 return thisDate
450 450
451 451 def getDateFromRadarFolder(folder):
452 452 try:
453 453 year = int(folder[1:5])
454 454 doy = int(folder[5:8])
455 455 except:
456 456 return None
457 457
458 458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
459 459 return thisDate
460 460
461 461 class JRODataIO:
462 462
463 463 c = 3E8
464 464
465 465 isConfig = False
466 466
467 467 basicHeaderObj = None
468 468
469 469 systemHeaderObj = None
470 470
471 471 radarControllerHeaderObj = None
472 472
473 473 processingHeaderObj = None
474 474
475 475 dtype = None
476 476
477 477 pathList = []
478 478
479 479 filenameList = []
480 480
481 481 filename = None
482 482
483 483 ext = None
484 484
485 485 flagIsNewFile = 1
486 486
487 487 flagDiscontinuousBlock = 0
488 488
489 489 flagIsNewBlock = 0
490 490
491 491 fp = None
492 492
493 493 firstHeaderSize = 0
494 494
495 495 basicHeaderSize = 24
496 496
497 497 versionFile = 1103
498 498
499 499 fileSize = None
500 500
501 501 # ippSeconds = None
502 502
503 503 fileSizeByHeader = None
504 504
505 505 fileIndex = None
506 506
507 507 profileIndex = None
508 508
509 509 blockIndex = None
510 510
511 511 nTotalBlocks = None
512 512
513 513 maxTimeStep = 30
514 514
515 515 lastUTTime = None
516 516
517 517 datablock = None
518 518
519 519 dataOut = None
520 520
521 521 blocksize = None
522 522
523 523 getByBlock = False
524 524
525 525 def __init__(self):
526 526
527 527 raise NotImplementedError
528 528
529 529 def run(self):
530 530
531 531 raise NotImplementedError
532 532
533 533 def getDtypeWidth(self):
534 534
535 535 dtype_index = get_dtype_index(self.dtype)
536 536 dtype_width = get_dtype_width(dtype_index)
537 537
538 538 return dtype_width
539 539
540 540 def getAllowedArgs(self):
541 541 return inspect.getargspec(self.run).args
542 542
543 543 class JRODataReader(JRODataIO):
544 544
545 545 firstTime = True
546 546 online = 0
547 547
548 548 realtime = 0
549 549
550 550 nReadBlocks = 0
551 551
552 552 delay = 10 #number of seconds waiting a new file
553 553
554 554 nTries = 3 #quantity tries
555 555
556 556 nFiles = 3 #number of files for searching
557 557
558 558 path = None
559 559
560 560 foldercounter = 0
561 561
562 562 flagNoMoreFiles = 0
563 563
564 564 datetimeList = []
565 565
566 566 __isFirstTimeOnline = 1
567 567
568 568 __printInfo = True
569 569
570 570 profileIndex = None
571 571
572 572 nTxs = 1
573 573
574 574 txIndex = None
575 575
576 576 #Added--------------------
577 577
578 578 selBlocksize = None
579 579
580 580 selBlocktime = None
581 581
582 582 onlineWithDate = False
583 583 def __init__(self):
584 584
585 585 """
586 586 This class is used to find data files
587 587
588 588 Example:
589 589 reader = JRODataReader()
590 590 fileList = reader.findDataFiles()
591 591
592 592 """
593 593 pass
594 594
595 595
596 596 def createObjByDefault(self):
597 597 """
598 598
599 599 """
600 600 raise NotImplementedError
601 601
602 602 def getBlockDimension(self):
603 603
604 604 raise NotImplementedError
605 605
606 606 def __searchFilesOffLine(self,
607 607 path,
608 608 startDate=None,
609 609 endDate=None,
610 610 startTime=datetime.time(0,0,0),
611 611 endTime=datetime.time(23,59,59),
612 612 set=None,
613 613 expLabel='',
614 614 ext='.r',
615 615 queue=None,
616 616 cursor=None,
617 617 skip=None,
618 618 walk=True):
619 619 self.filenameList = []
620 620 self.datetimeList = []
621 621
622 622 pathList = []
623 623
624 624 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
625 625
626 626 if dateList == []:
627 627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
628 628 return None, None
629 629
630 630 if len(dateList) > 1:
631 631 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
632 632 else:
633 633 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
634 634
635 635 filenameList = []
636 636 datetimeList = []
637 637
638 638 for thisPath in pathList:
639 639 # thisPath = pathList[pathDict[file]]
640 640
641 641 fileList = glob.glob1(thisPath, "*%s" %ext)
642 642 fileList.sort()
643 643
644 644 skippedFileList = []
645 645
646 646 if cursor is not None and skip is not None:
647 647 # if cursor*skip > len(fileList):
648 648 if skip == 0:
649 649 if queue is not None:
650 650 queue.put(len(fileList))
651 651 skippedFileList = []
652 652 else:
653 653 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
654 654
655 655 else:
656 656 skippedFileList = fileList
657 657
658 658 for file in skippedFileList:
659 659
660 660 filename = os.path.join(thisPath,file)
661 661
662 662 if not isFileInDateRange(filename, startDate, endDate):
663 663 continue
664 664
665 665 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
666 666
667 667 if not(thisDatetime):
668 668 continue
669 669
670 670 filenameList.append(filename)
671 671 datetimeList.append(thisDatetime)
672 672
673 673 if not(filenameList):
674 674 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
675 675 return None, None
676 676
677 677 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
678 678 print
679 679
680 680 for i in range(len(filenameList)):
681 681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
682 682
683 683 self.filenameList = filenameList
684 684 self.datetimeList = datetimeList
685 685 return pathList, filenameList
686 686
687 687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
688 688
689 689 """
690 690 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
691 691 devuelve el archivo encontrado ademas de otros datos.
692 692
693 693 Input:
694 694 path : carpeta donde estan contenidos los files que contiene data
695 695
696 696 expLabel : Nombre del subexperimento (subfolder)
697 697
698 698 ext : extension de los files
699 699
700 700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
701 701
702 702 Return:
703 703 directory : eL directorio donde esta el file encontrado
704 704 filename : el ultimo file de una determinada carpeta
705 705 year : el anho
706 706 doy : el numero de dia del anho
707 707 set : el set del archivo
708 708
709 709
710 710 """
711 711 pathList = None
712 712 filenameList = None
713 713 if not os.path.isdir(path):
714 714 return None, None, None, None, None, None
715 715
716 716 dirList = []
717 717
718 718 if not walk:
719 719 fullpath = path
720 720 foldercounter = 0
721 721 else:
722 722 # Filtra solo los directorios
723 723 for thisPath in os.listdir(path):
724 724 if not os.path.isdir(os.path.join(path,thisPath)):
725 725 continue
726 726 if not isRadarFolder(thisPath):
727 727 continue
728 728
729 729 dirList.append(thisPath)
730 730
731 731 if not(dirList):
732 732 return None, None, None, None, None, None
733 733
734 734 dirList = sorted( dirList, key=str.lower )
735 735
736 736 doypath = dirList[-1]
737 737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
738 738 fullpath = os.path.join(path, doypath, expLabel)
739 739
740 740
741 741 print "[Reading] %s folder was found: " %(fullpath )
742 742
743 743 if set == None:
744 744 filename = getlastFileFromPath(fullpath, ext)
745 745 else:
746 746 filename = getFileFromSet(fullpath, ext, set)
747 747
748 748 if not(filename):
749 749 return None, None, None, None, None, None
750 750
751 751 print "[Reading] %s file was found" %(filename)
752 752
753 753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
754 754 return None, None, None, None, None, None
755 755
756 756 year = int( filename[1:5] )
757 757 doy = int( filename[5:8] )
758 758 set = int( filename[8:11] )
759 759
760 760 return fullpath, foldercounter, filename, year, doy, set
761 761
762 762 def __setNextFileOffline(self):
763 763
764 764 idFile = self.fileIndex
765 765
766 766 while (True):
767 767 idFile += 1
768 768 if not(idFile < len(self.filenameList)):
769 769 self.flagNoMoreFiles = 1
770 770 # print "[Reading] No more Files"
771 771 return 0
772 772
773 773 filename = self.filenameList[idFile]
774 774
775 775 if not(self.__verifyFile(filename)):
776 776 continue
777 777
778 778 fileSize = os.path.getsize(filename)
779 779 fp = open(filename,'rb')
780 780 break
781 781
782 782 self.flagIsNewFile = 1
783 783 self.fileIndex = idFile
784 784 self.filename = filename
785 785 self.fileSize = fileSize
786 786 self.fp = fp
787 787
788 788 #print "[Reading] Setting the file: %s"%self.filename
789 789
790 790 return 1
791 791
792 792 def __setNextFileOnline(self):
793 793 """
794 794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 796 siguientes.
797 797
798 798 Affected:
799 799 self.flagIsNewFile
800 800 self.filename
801 801 self.fileSize
802 802 self.fp
803 803 self.set
804 804 self.flagNoMoreFiles
805 805
806 806 Return:
807 807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 808 1 : si el file fue abierto con exito y esta listo a ser leido
809 809
810 810 Excepciones:
811 811 Si un determinado file no puede ser abierto
812 812 """
813 813
814 814 nFiles = 0
815 815 fileOk_flag = False
816 816 firstTime_flag = True
817 817
818 818 self.set += 1
819 819
820 820 if self.set > 999:
821 821 self.set = 0
822 822 self.foldercounter += 1
823 823
824 824 #busca el 1er file disponible
825 825 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
826 826 if fullfilename:
827 827 if self.__verifyFile(fullfilename, False):
828 828 fileOk_flag = True
829 829
830 830 #si no encuentra un file entonces espera y vuelve a buscar
831 831 if not(fileOk_flag):
832 832 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
833 833
834 834 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
835 835 tries = self.nTries
836 836 else:
837 837 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
838 838
839 839 for nTries in range( tries ):
840 840 if firstTime_flag:
841 841 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
842 842 sleep( self.delay )
843 843 else:
844 844 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
845 845
846 846 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
847 847 if fullfilename:
848 848 if self.__verifyFile(fullfilename):
849 849 fileOk_flag = True
850 850 break
851 851
852 852 if fileOk_flag:
853 853 break
854 854
855 855 firstTime_flag = False
856 856
857 857 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
858 858 self.set += 1
859 859
860 860 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
861 861 self.set = 0
862 862 self.doy += 1
863 863 self.foldercounter = 0
864 864
865 865 if fileOk_flag:
866 866 self.fileSize = os.path.getsize( fullfilename )
867 867 self.filename = fullfilename
868 868 self.flagIsNewFile = 1
869 869 if self.fp != None: self.fp.close()
870 870 self.fp = open(fullfilename, 'rb')
871 871 self.flagNoMoreFiles = 0
872 872 # print '[Reading] Setting the file: %s' % fullfilename
873 873 else:
874 874 self.fileSize = 0
875 875 self.filename = None
876 876 self.flagIsNewFile = 0
877 877 self.fp = None
878 878 self.flagNoMoreFiles = 1
879 879 # print '[Reading] No more files to read'
880 880
881 881 return fileOk_flag
882 882
883 883 def setNextFile(self):
884 884 if self.fp != None:
885 885 self.fp.close()
886 886 if self.online:
887 887 newFile = self.__setNextFileOnline()
888 888 else:
889 889 newFile = self.__setNextFileOffline()
890 890 if not(newFile):
891 891 if self.onlineWithDate is True:
892 892 self.onlineWithDate=False
893 893 self.online = True
894 894 self.firstTime = False
895 895 self.setup(
896 896 path=self.path,
897 897 startDate=self.startDate,
898 898 endDate=self.endDate,
899 899 startTime=self.startTime ,
900 900 endTime=self.endTime,
901 901 set=self.set,
902 902 expLabel=self.expLabel,
903 903 ext=self.ext,
904 904 online=self.online,
905 905 delay=self.delay,
906 906 walk=self.walk,
907 907 getblock=self.getblock,
908 908 nTxs=self.nTxs,
909 909 realtime=self.realtime,
910 910 blocksize=self.blocksize,
911 911 blocktime=self.blocktime
912 912 )
913 913 return 1
914 914 print '[Reading] No more files to read'
915 915 return 0
916 916
917 917 if self.verbose:
918 918 print '[Reading] Setting the file: %s' % self.filename
919 919
920 920 self.__readFirstHeader()
921 921 self.nReadBlocks = 0
922 922 return 1
923 923
924 924 def __waitNewBlock(self):
925 925 """
926 926 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
927 927
928 928 Si el modo de lectura es OffLine siempre retorn 0
929 929 """
930 930 if not self.online:
931 931 return 0
932 932
933 933 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
934 934 return 0
935 935
936 936 currentPointer = self.fp.tell()
937 937
938 938 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
939 939
940 940 for nTries in range( self.nTries ):
941 941
942 942 self.fp.close()
943 943 self.fp = open( self.filename, 'rb' )
944 944 self.fp.seek( currentPointer )
945 945
946 946 self.fileSize = os.path.getsize( self.filename )
947 947 currentSize = self.fileSize - currentPointer
948 948
949 949 if ( currentSize >= neededSize ):
950 950 self.basicHeaderObj.read(self.fp)
951 951 return 1
952 952
953 953 if self.fileSize == self.fileSizeByHeader:
954 954 # self.flagEoF = True
955 955 return 0
956 956
957 957 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
958 958 sleep( self.delay )
959 959
960 960
961 961 return 0
962 962
963 963 def waitDataBlock(self,pointer_location):
964 964
965 965 currentPointer = pointer_location
966 966
967 967 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
968 968
969 969 for nTries in range( self.nTries ):
970 970 self.fp.close()
971 971 self.fp = open( self.filename, 'rb' )
972 972 self.fp.seek( currentPointer )
973 973
974 974 self.fileSize = os.path.getsize( self.filename )
975 975 currentSize = self.fileSize - currentPointer
976 976
977 977 if ( currentSize >= neededSize ):
978 978 return 1
979 979
980 980 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
981 981 sleep( self.delay )
982 982
983 983 return 0
984 984
985 985 def __jumpToLastBlock(self):
986 986
987 987 if not(self.__isFirstTimeOnline):
988 988 return
989 989
990 990 csize = self.fileSize - self.fp.tell()
991 991 blocksize = self.processingHeaderObj.blockSize
992 992
993 993 #salta el primer bloque de datos
994 994 if csize > self.processingHeaderObj.blockSize:
995 995 self.fp.seek(self.fp.tell() + blocksize)
996 996 else:
997 997 return
998 998
999 999 csize = self.fileSize - self.fp.tell()
1000 1000 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1001 1001 while True:
1002 1002
1003 1003 if self.fp.tell()<self.fileSize:
1004 1004 self.fp.seek(self.fp.tell() + neededsize)
1005 1005 else:
1006 1006 self.fp.seek(self.fp.tell() - neededsize)
1007 1007 break
1008 1008
1009 1009 # csize = self.fileSize - self.fp.tell()
1010 1010 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1011 1011 # factor = int(csize/neededsize)
1012 1012 # if factor > 0:
1013 1013 # self.fp.seek(self.fp.tell() + factor*neededsize)
1014 1014
1015 1015 self.flagIsNewFile = 0
1016 1016 self.__isFirstTimeOnline = 0
1017 1017
1018 1018 def __setNewBlock(self):
1019 1019 #if self.server is None:
1020 1020 if self.fp == None:
1021 1021 return 0
1022 1022
1023 1023 # if self.online:
1024 1024 # self.__jumpToLastBlock()
1025 1025
1026 1026 if self.flagIsNewFile:
1027 1027 self.lastUTTime = self.basicHeaderObj.utc
1028 1028 return 1
1029 1029
1030 1030 if self.realtime:
1031 1031 self.flagDiscontinuousBlock = 1
1032 1032 if not(self.setNextFile()):
1033 1033 return 0
1034 1034 else:
1035 1035 return 1
1036 1036 #if self.server is None:
1037 1037 currentSize = self.fileSize - self.fp.tell()
1038 1038 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1039 1039 if (currentSize >= neededSize):
1040 1040 self.basicHeaderObj.read(self.fp)
1041 1041 self.lastUTTime = self.basicHeaderObj.utc
1042 1042 return 1
1043 1043 # else:
1044 1044 # self.basicHeaderObj.read(self.zHeader)
1045 1045 # self.lastUTTime = self.basicHeaderObj.utc
1046 1046 # return 1
1047 1047 if self.__waitNewBlock():
1048 1048 self.lastUTTime = self.basicHeaderObj.utc
1049 1049 return 1
1050 1050 #if self.server is None:
1051 1051 if not(self.setNextFile()):
1052 1052 return 0
1053 1053
1054 1054 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1055 1055 self.lastUTTime = self.basicHeaderObj.utc
1056 1056
1057 1057 self.flagDiscontinuousBlock = 0
1058 1058
1059 1059 if deltaTime > self.maxTimeStep:
1060 1060 self.flagDiscontinuousBlock = 1
1061 1061
1062 1062 return 1
1063 1063
1064 1064 def readNextBlock(self):
1065 1065
1066 1066 #Skip block out of startTime and endTime
1067 1067 while True:
1068 1068 if not(self.__setNewBlock()):
1069 1069 print 'returning'
1070 1070 return 0
1071 1071 if not(self.readBlock()):
1072 1072 return 0
1073 1073 self.getBasicHeader()
1074 1074 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1075 1075
1076 1076 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1077 1077 self.processingHeaderObj.dataBlocksPerFile,
1078 1078 self.dataOut.datatime.ctime())
1079 1079 continue
1080 1080
1081 1081 break
1082 1082
1083 1083 if self.verbose:
1084 1084 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1085 1085 self.processingHeaderObj.dataBlocksPerFile,
1086 1086 self.dataOut.datatime.ctime())
1087 1087 return 1
1088 1088
1089 1089 def __readFirstHeader(self):
1090 1090
1091 1091 self.basicHeaderObj.read(self.fp)
1092 1092 self.systemHeaderObj.read(self.fp)
1093 1093 self.radarControllerHeaderObj.read(self.fp)
1094 1094 self.processingHeaderObj.read(self.fp)
1095 1095
1096 1096 self.firstHeaderSize = self.basicHeaderObj.size
1097 1097
1098 1098 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1099 1099 if datatype == 0:
1100 1100 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1101 1101 elif datatype == 1:
1102 1102 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1103 1103 elif datatype == 2:
1104 1104 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1105 1105 elif datatype == 3:
1106 1106 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1107 1107 elif datatype == 4:
1108 1108 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1109 1109 elif datatype == 5:
1110 1110 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1111 1111 else:
1112 1112 raise ValueError, 'Data type was not defined'
1113 1113
1114 1114 self.dtype = datatype_str
1115 1115 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1116 1116 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1117 1117 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1118 1118 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1119 1119 self.getBlockDimension()
1120 1120
1121 1121 def __verifyFile(self, filename, msgFlag=True):
1122 1122
1123 1123 msg = None
1124 1124
1125 1125 try:
1126 1126 fp = open(filename, 'rb')
1127 1127 except IOError:
1128 1128
1129 1129 if msgFlag:
1130 1130 print "[Reading] File %s can't be opened" % (filename)
1131 1131
1132 1132 return False
1133 1133
1134 1134 currentPosition = fp.tell()
1135 1135 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1136 1136
1137 1137 if neededSize == 0:
1138 1138 basicHeaderObj = BasicHeader(LOCALTIME)
1139 1139 systemHeaderObj = SystemHeader()
1140 1140 radarControllerHeaderObj = RadarControllerHeader()
1141 1141 processingHeaderObj = ProcessingHeader()
1142 1142
1143 1143 if not( basicHeaderObj.read(fp) ):
1144 1144 fp.close()
1145 1145 return False
1146 1146
1147 1147 if not( systemHeaderObj.read(fp) ):
1148 1148 fp.close()
1149 1149 return False
1150 1150
1151 1151 if not( radarControllerHeaderObj.read(fp) ):
1152 1152 fp.close()
1153 1153 return False
1154 1154
1155 1155 if not( processingHeaderObj.read(fp) ):
1156 1156 fp.close()
1157 1157 return False
1158 1158
1159 1159 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1160 1160 else:
1161 1161 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1162 1162
1163 1163 fp.close()
1164 1164
1165 1165 fileSize = os.path.getsize(filename)
1166 1166 currentSize = fileSize - currentPosition
1167 1167
1168 1168 if currentSize < neededSize:
1169 1169 if msgFlag and (msg != None):
1170 1170 print msg
1171 1171 return False
1172 1172
1173 1173 return True
1174 1174
1175 1175 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1176 1176
1177 1177 path_empty = True
1178 1178
1179 1179 dateList = []
1180 1180 pathList = []
1181 1181
1182 1182 multi_path = path.split(',')
1183 1183
1184 1184 if not walk:
1185 1185
1186 1186 for single_path in multi_path:
1187 1187
1188 1188 if not os.path.isdir(single_path):
1189 1189 continue
1190 1190
1191 1191 fileList = glob.glob1(single_path, "*"+ext)
1192 1192
1193 1193 if not fileList:
1194 1194 continue
1195 1195
1196 1196 path_empty = False
1197 1197
1198 1198 fileList.sort()
1199 1199
1200 1200 for thisFile in fileList:
1201 1201
1202 1202 if not os.path.isfile(os.path.join(single_path, thisFile)):
1203 1203 continue
1204 1204
1205 1205 if not isRadarFile(thisFile):
1206 1206 continue
1207 1207
1208 1208 if not isFileInDateRange(thisFile, startDate, endDate):
1209 1209 continue
1210 1210
1211 1211 thisDate = getDateFromRadarFile(thisFile)
1212 1212
1213 1213 if thisDate in dateList:
1214 1214 continue
1215 1215
1216 1216 dateList.append(thisDate)
1217 1217 pathList.append(single_path)
1218 1218
1219 1219 else:
1220 1220 for single_path in multi_path:
1221 1221
1222 1222 if not os.path.isdir(single_path):
1223 1223 continue
1224 1224
1225 1225 dirList = []
1226 1226
1227 1227 for thisPath in os.listdir(single_path):
1228 1228
1229 1229 if not os.path.isdir(os.path.join(single_path,thisPath)):
1230 1230 continue
1231 1231
1232 1232 if not isRadarFolder(thisPath):
1233 1233 continue
1234 1234
1235 1235 if not isFolderInDateRange(thisPath, startDate, endDate):
1236 1236 continue
1237 1237
1238 1238 dirList.append(thisPath)
1239 1239
1240 1240 if not dirList:
1241 1241 continue
1242 1242
1243 1243 dirList.sort()
1244 1244
1245 1245 for thisDir in dirList:
1246 1246
1247 1247 datapath = os.path.join(single_path, thisDir, expLabel)
1248 1248 fileList = glob.glob1(datapath, "*"+ext)
1249 1249
1250 1250 if not fileList:
1251 1251 continue
1252 1252
1253 1253 path_empty = False
1254 1254
1255 1255 thisDate = getDateFromRadarFolder(thisDir)
1256 1256
1257 1257 pathList.append(datapath)
1258 1258 dateList.append(thisDate)
1259 1259
1260 1260 dateList.sort()
1261 1261
1262 1262 if walk:
1263 1263 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1264 1264 else:
1265 1265 pattern_path = multi_path[0]
1266 1266
1267 1267 if path_empty:
1268 1268 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1269 1269 else:
1270 1270 if not dateList:
1271 1271 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1272 1272
1273 1273 if include_path:
1274 1274 return dateList, pathList
1275 1275
1276 1276 return dateList
1277 1277
1278 1278 def setup(self,
1279 1279 path=None,
1280 startDate=None,
1281 endDate=None,
1282 startTime=datetime.time(0,0,0),
1283 endTime=datetime.time(23,59,59),
1284 set=None,
1285 expLabel = "",
1286 ext = None,
1280 startDate=None,
1281 endDate=None,
1282 startTime=datetime.time(0,0,0),
1283 endTime=datetime.time(23,59,59),
1284 set=None,
1285 expLabel = "",
1286 ext = None,
1287 1287 online = False,
1288 1288 delay = 60,
1289 1289 walk = True,
1290 1290 getblock = False,
1291 1291 nTxs = 1,
1292 1292 realtime=False,
1293 1293 blocksize=None,
1294 blocktime=None,
1295 queue=None,
1296 skip=None,
1297 cursor=None,
1298 warnings=True,
1299 verbose=True,
1300 server=None):
1301 if server is not None:
1302 if 'tcp://' in server:
1303 address = server
1304 else:
1305 address = 'ipc:///tmp/%s' % server
1306 self.server = address
1307 self.context = zmq.Context()
1308 self.receiver = self.context.socket(zmq.PULL)
1309 self.receiver.connect(self.server)
1310 time.sleep(0.5)
1311 print '[Starting] ReceiverData from {}'.format(self.server)
1312 else:
1313 self.server = None
1314 if path == None:
1315 raise ValueError, "[Reading] The path is not valid"
1316
1317 if ext == None:
1318 ext = self.ext
1319
1320 if online:
1321 print "[Reading] Searching files in online mode..."
1322
1323 for nTries in range( self.nTries ):
1324 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1325
1326 if fullpath:
1327 break
1328
1329 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1330 sleep( self.delay )
1331
1332 if not(fullpath):
1333 print "[Reading] There 'isn't any valid file in %s" % path
1334 return
1335
1336 self.year = year
1337 self.doy = doy
1338 self.set = set - 1
1339 self.path = path
1340 self.foldercounter = foldercounter
1341 last_set = None
1342 else:
1343 print "[Reading] Searching files in offline mode ..."
1344 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1345 startTime=startTime, endTime=endTime,
1346 set=set, expLabel=expLabel, ext=ext,
1347 walk=walk, cursor=cursor,
1348 skip=skip, queue=queue)
1349
1350 <<<<<<< HEAD
1351 if not(pathList):
1352 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1353 # datetime.datetime.combine(startDate,startTime).ctime(),
1354 # datetime.datetime.combine(endDate,endTime).ctime())
1355
1356 # sys.exit(-1)
1294 blocktime=None):
1357 1295
1358 self.fileIndex = -1
1359 self.pathList = []
1360 self.filenameList = []
1361 return
1362
1363 =======
1364 1296 if path == None:
1365 1297 raise ValueError, "[Reading] The path is not valid"
1366 1298
1367 1299
1368 1300 if ext == None:
1369 1301 ext = self.ext
1370 1302
1371 1303 self.path = path
1372 1304 self.startDate = startDate
1373 1305 self.endDate = endDate
1374 1306 self.startTime = startTime
1375 1307 self.endTime = endTime
1376 1308 self.set = set
1377 1309 self.expLabel = expLabel
1378 1310 self.ext = ext
1379 1311 self.online = online
1380 1312 self.delay = delay
1381 1313 self.walk = walk
1382 1314 self.getblock = getblock
1383 1315 self.nTxs = nTxs
1384 1316 self.realtime = realtime
1385 1317 self.blocksize = blocksize
1386 1318 self.blocktime = blocktime
1387 1319
1388 1320
1389 1321 if self.firstTime is True:
1390 1322 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1391 1323 startTime=startTime, endTime=endTime,
1392 1324 set=set, expLabel=expLabel, ext=ext,
1393 1325 walk=walk)
1394 1326 filenameList = filenameList[:-1]
1395 1327
1396 1328 if pathList is not None and filenameList is not None and online:
1397 1329 self.onlineWithDate = True
1398 1330 online = False
1399 1331 self.fileIndex = -1
1400 1332 self.pathList = pathList
1401 1333 self.filenameList = filenameList
1402 1334 file_name = os.path.basename(filenameList[-1])
1403 1335 basename, ext = os.path.splitext(file_name)
1404 1336 last_set = int(basename[-3:])
1405 1337
1406 1338 if online:
1407 1339 print "[Reading] Searching files in online mode..."
1408 1340
1409 1341 for nTries in range(self.nTries):
1410 1342 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1411 1343 expLabel=expLabel,
1412 1344 ext=ext,
1413 1345 walk=walk,
1414 1346 startDate=startDate,
1415 1347 startTime=startTime,
1416 1348 set=set)
1417 1349
1418 1350 if fullpath:
1419 1351 break
1420 1352 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1421 1353 sleep( self.delay )
1422 1354
1423 1355 if not(fullpath):
1424 1356 print "[Reading] There 'isn't any valid file in %s" % path
1425 1357 return
1426 1358
1427 1359 self.year = year
1428 1360 self.doy = doy
1429 1361 self.set = set - 1
1430 1362 self.path = path
1431 1363 self.foldercounter = foldercounter
1432 1364 last_set = None
1433 1365
1434 1366 else:
1435 1367 print "[Reading] Searching files in offline mode ..."
1436 1368 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1437 1369 startTime=startTime, endTime=endTime,
1438 1370 set=set, expLabel=expLabel, ext=ext,
1439 1371 walk=walk)
1440 1372
1441 1373 if not(pathList):
1442 1374 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1443 1375 # datetime.datetime.combine(startDate,startTime).ctime(),
1444 1376 # datetime.datetime.combine(endDate,endTime).ctime())
1445 1377
1446 1378 # sys.exit(-1)
1447 1379
1448 1380 self.fileIndex = -1
1449 1381 self.pathList = []
1450 1382 self.filenameList = []
1451 1383 return
1452 1384
1453 1385 self.fileIndex = -1
1454 1386 self.pathList = pathList
1455 1387 self.filenameList = filenameList
1456 1388 file_name = os.path.basename(filenameList[-1])
1457 1389 basename, ext = os.path.splitext(file_name)
1458 1390 last_set = int(basename[-3:])
1459 1391
1460 1392
1461 1393 self.online = online
1462 1394 self.realtime = realtime
1463 1395 self.delay = delay
1464 1396 ext = ext.lower()
1465 1397 self.ext = ext
1466 1398 self.getByBlock = getblock
1467 1399 self.nTxs = nTxs
1468 1400 self.startTime = startTime
1469 1401 self.endTime = endTime
1470 1402
1471 1403
1472 1404 #Added-----------------
1473 1405 self.selBlocksize = blocksize
1474 1406 self.selBlocktime = blocktime
1475 1407
1476 1408
1477 1409 if not(self.setNextFile()):
1478 1410 if (startDate!=None) and (endDate!=None):
1479 1411 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1480 1412 elif startDate != None:
1481 1413 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1482 1414 else:
1483 1415 print "[Reading] No files"
1484 1416
1485 >>>>>>> online_data_hour
1486 1417 self.fileIndex = -1
1487 self.pathList = pathList
1488 self.filenameList = filenameList
1489 file_name = os.path.basename(filenameList[-1])
1490 basename, ext = os.path.splitext(file_name)
1491 last_set = int(basename[-3:])
1492
1493 self.online = online
1494 self.realtime = realtime
1495 self.delay = delay
1496 ext = ext.lower()
1497 self.ext = ext
1498 self.getByBlock = getblock
1499 self.nTxs = nTxs
1500 self.startTime = startTime
1501 self.endTime = endTime
1502
1503 #Added-----------------
1504 self.selBlocksize = blocksize
1505 self.selBlocktime = blocktime
1506
1507 # Verbose-----------
1508 self.verbose = verbose
1509 self.warnings = warnings
1510
1511 <<<<<<< HEAD
1512 if not(self.setNextFile()):
1513 if (startDate!=None) and (endDate!=None):
1514 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1515 elif startDate != None:
1516 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1517 else:
1518 print "[Reading] No files"
1519
1520 self.fileIndex = -1
1521 self.pathList = []
1522 self.filenameList = []
1523 return
1524
1525 # self.getBasicHeader()
1418 self.pathList = []
1419 self.filenameList = []
1420 return
1526 1421
1527 if last_set != None:
1528 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1529 return
1530 =======
1531 1422 # self.getBasicHeader()
1532 1423
1533 1424 if last_set != None:
1534 1425 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1535 1426 return
1536 >>>>>>> online_data_hour
1537 1427
1538 1428 def getBasicHeader(self):
1539 1429
1540 1430 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1541 1431
1542 1432 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1543 1433
1544 1434 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1545 1435
1546 1436 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1547 1437
1548 1438 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1549 1439
1550 1440 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1551 1441
1552 1442 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1553 1443
1554 1444 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1555 1445
1556 1446
1557 1447 def getFirstHeader(self):
1558 1448
1559 1449 raise NotImplementedError
1560 1450
1561 1451 def getData(self):
1562 1452
1563 1453 raise NotImplementedError
1564 1454
1565 1455 def hasNotDataInBuffer(self):
1566 1456
1567 1457 raise NotImplementedError
1568 1458
1569 1459 def readBlock(self):
1570 1460
1571 1461 raise NotImplementedError
1572 1462
1573 1463 def isEndProcess(self):
1574 1464
1575 1465 return self.flagNoMoreFiles
1576 1466
1577 1467 def printReadBlocks(self):
1578 1468
1579 1469 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1580 1470
1581 1471 def printTotalBlocks(self):
1582 1472
1583 1473 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1584 1474
1585 1475 def printNumberOfBlock(self):
1586 1476
1587 1477 if self.flagIsNewBlock:
1588 1478 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1589 1479 self.processingHeaderObj.dataBlocksPerFile,
1590 1480 self.dataOut.datatime.ctime())
1591 1481
1592 1482 def printInfo(self):
1593 1483
1594 1484 if self.__printInfo == False:
1595 1485 return
1596 1486
1597 1487 self.basicHeaderObj.printInfo()
1598 1488 self.systemHeaderObj.printInfo()
1599 1489 self.radarControllerHeaderObj.printInfo()
1600 1490 self.processingHeaderObj.printInfo()
1601 1491
1602 1492 self.__printInfo = False
1603 1493
1604 1494
1605 1495 def run(self,
1606 1496 path=None,
1607 1497 startDate=None,
1608 1498 endDate=None,
1609 1499 startTime=datetime.time(0,0,0),
1610 1500 endTime=datetime.time(23,59,59),
1611 1501 set=None,
1612 1502 expLabel = "",
1613 1503 ext = None,
1614 1504 online = False,
1615 1505 delay = 60,
1616 1506 walk = True,
1617 1507 getblock = False,
1618 1508 nTxs = 1,
1619 1509 realtime=False,
1620 1510 blocksize=None,
1621 1511 blocktime=None,
1622 1512 queue=None,
1623 1513 skip=None,
1624 1514 cursor=None,
1625 1515 warnings=True,
1626 1516 server=None,
1627 1517 verbose=True, **kwargs):
1628 1518
1629 1519 if not(self.isConfig):
1630 1520 <<<<<<< HEAD
1631 1521 # self.dataOut = dataOut
1632 1522 self.setup( path=path,
1633 1523 startDate=startDate,
1634 1524 endDate=endDate,
1635 1525 startTime=startTime,
1636 1526 endTime=endTime,
1637 1527 set=set,
1638 1528 expLabel=expLabel,
1639 1529 ext=ext,
1640 1530 online=online,
1641 1531 delay=delay,
1642 1532 walk=walk,
1643 1533 getblock=getblock,
1644 1534 nTxs=nTxs,
1645 1535 realtime=realtime,
1646 1536 blocksize=blocksize,
1647 1537 blocktime=blocktime,
1648 1538 queue=queue,
1649 1539 skip=skip,
1650 1540 cursor=cursor,
1651 1541 warnings=warnings,
1652 1542 server=server,
1653 1543 verbose=verbose)
1654 1544 =======
1655 1545
1656 1546 # self.dataOut = dataOut
1657 1547 self.setup(**kwargs)
1658 1548 >>>>>>> online_data_hour
1659 1549 self.isConfig = True
1660 1550 if server is None:
1661 1551 self.getData()
1662 1552 else:
1663 1553 self.getFromServer()
1664 1554
1665 1555 class JRODataWriter(JRODataIO):
1666 1556
1667 1557 """
1668 1558 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1669 1559 de los datos siempre se realiza por bloques.
1670 1560 """
1671 1561
1672 1562 blockIndex = 0
1673 1563
1674 1564 path = None
1675 1565
1676 1566 setFile = None
1677 1567
1678 1568 profilesPerBlock = None
1679 1569
1680 1570 blocksPerFile = None
1681 1571
1682 1572 nWriteBlocks = 0
1683 1573
1684 1574 fileDate = None
1685 1575
1686 1576 def __init__(self, dataOut=None):
1687 1577 raise NotImplementedError
1688 1578
1689 1579
1690 1580 def hasAllDataInBuffer(self):
1691 1581 raise NotImplementedError
1692 1582
1693 1583
1694 1584 def setBlockDimension(self):
1695 1585 raise NotImplementedError
1696 1586
1697 1587
1698 1588 def writeBlock(self):
1699 1589 raise NotImplementedError
1700 1590
1701 1591
1702 1592 def putData(self):
1703 1593 raise NotImplementedError
1704 1594
1705 1595
1706 1596 def getProcessFlags(self):
1707 1597
1708 1598 processFlags = 0
1709 1599
1710 1600 dtype_index = get_dtype_index(self.dtype)
1711 1601 procflag_dtype = get_procflag_dtype(dtype_index)
1712 1602
1713 1603 processFlags += procflag_dtype
1714 1604
1715 1605 if self.dataOut.flagDecodeData:
1716 1606 processFlags += PROCFLAG.DECODE_DATA
1717 1607
1718 1608 if self.dataOut.flagDeflipData:
1719 1609 processFlags += PROCFLAG.DEFLIP_DATA
1720 1610
1721 1611 if self.dataOut.code is not None:
1722 1612 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1723 1613
1724 1614 if self.dataOut.nCohInt > 1:
1725 1615 processFlags += PROCFLAG.COHERENT_INTEGRATION
1726 1616
1727 1617 if self.dataOut.type == "Spectra":
1728 1618 if self.dataOut.nIncohInt > 1:
1729 1619 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1730 1620
1731 1621 if self.dataOut.data_dc is not None:
1732 1622 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1733 1623
1734 1624 if self.dataOut.flagShiftFFT:
1735 1625 processFlags += PROCFLAG.SHIFT_FFT_DATA
1736 1626
1737 1627 return processFlags
1738 1628
1739 1629 def setBasicHeader(self):
1740 1630
1741 1631 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1742 1632 self.basicHeaderObj.version = self.versionFile
1743 1633 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1744 1634
1745 1635 utc = numpy.floor(self.dataOut.utctime)
1746 1636 milisecond = (self.dataOut.utctime - utc)* 1000.0
1747 1637
1748 1638 self.basicHeaderObj.utc = utc
1749 1639 self.basicHeaderObj.miliSecond = milisecond
1750 1640 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1751 1641 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1752 1642 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1753 1643
1754 1644 def setFirstHeader(self):
1755 1645 """
1756 1646 Obtiene una copia del First Header
1757 1647
1758 1648 Affected:
1759 1649
1760 1650 self.basicHeaderObj
1761 1651 self.systemHeaderObj
1762 1652 self.radarControllerHeaderObj
1763 1653 self.processingHeaderObj self.
1764 1654
1765 1655 Return:
1766 1656 None
1767 1657 """
1768 1658
1769 1659 raise NotImplementedError
1770 1660
1771 1661 def __writeFirstHeader(self):
1772 1662 """
1773 1663 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1774 1664
1775 1665 Affected:
1776 1666 __dataType
1777 1667
1778 1668 Return:
1779 1669 None
1780 1670 """
1781 1671 <<<<<<< HEAD
1782 1672
1783 1673 # CALCULAR PARAMETROS
1784 1674
1785 1675 =======
1786 1676
1787 1677 # CALCULAR PARAMETROS
1788 1678
1789 1679 >>>>>>> online_data_hour
1790 1680 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1791 1681 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1792 1682
1793 1683 self.basicHeaderObj.write(self.fp)
1794 1684 self.systemHeaderObj.write(self.fp)
1795 1685 self.radarControllerHeaderObj.write(self.fp)
1796 1686 self.processingHeaderObj.write(self.fp)
1797 1687
1798 1688 def __setNewBlock(self):
1799 1689 """
1800 1690 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1801 1691
1802 1692 Return:
1803 1693 0 : si no pudo escribir nada
1804 1694 1 : Si escribio el Basic el First Header
1805 1695 """
1806 1696 if self.fp == None:
1807 1697 self.setNextFile()
1808 1698
1809 1699 if self.flagIsNewFile:
1810 1700 return 1
1811 1701
1812 1702 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1813 1703 self.basicHeaderObj.write(self.fp)
1814 1704 return 1
1815 1705
1816 1706 if not( self.setNextFile() ):
1817 1707 return 0
1818 1708
1819 1709 return 1
1820 1710
1821 1711
1822 1712 def writeNextBlock(self):
1823 1713 """
1824 1714 Selecciona el bloque siguiente de datos y los escribe en un file
1825 1715
1826 1716 Return:
1827 1717 0 : Si no hizo pudo escribir el bloque de datos
1828 1718 1 : Si no pudo escribir el bloque de datos
1829 1719 """
1830 1720 if not( self.__setNewBlock() ):
1831 1721 return 0
1832 1722
1833 1723 self.writeBlock()
1834 1724
1835 1725 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1836 1726 self.processingHeaderObj.dataBlocksPerFile)
1837 1727
1838 1728 return 1
1839 1729
1840 1730 def setNextFile(self):
1841 1731 """
1842 1732 Determina el siguiente file que sera escrito
1843 1733
1844 1734 Affected:
1845 1735 self.filename
1846 1736 self.subfolder
1847 1737 self.fp
1848 1738 self.setFile
1849 1739 self.flagIsNewFile
1850 1740
1851 1741 Return:
1852 1742 0 : Si el archivo no puede ser escrito
1853 1743 1 : Si el archivo esta listo para ser escrito
1854 1744 """
1855 1745 ext = self.ext
1856 1746 path = self.path
1857 1747
1858 1748 if self.fp != None:
1859 1749 self.fp.close()
1860 1750
1861 1751 timeTuple = time.localtime( self.dataOut.utctime)
1862 1752 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1863 1753
1864 1754 fullpath = os.path.join( path, subfolder )
1865 1755 setFile = self.setFile
1866 1756
1867 1757 if not( os.path.exists(fullpath) ):
1868 1758 os.mkdir(fullpath)
1869 1759 setFile = -1 #inicializo mi contador de seteo
1870 1760 else:
1871 1761 filesList = os.listdir( fullpath )
1872 1762 if len( filesList ) > 0:
1873 1763 filesList = sorted( filesList, key=str.lower )
1874 1764 filen = filesList[-1]
1875 1765 # el filename debera tener el siguiente formato
1876 1766 # 0 1234 567 89A BCDE (hex)
1877 1767 # x YYYY DDD SSS .ext
1878 1768 if isNumber( filen[8:11] ):
1879 1769 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1880 1770 else:
1881 1771 setFile = -1
1882 1772 else:
1883 1773 setFile = -1 #inicializo mi contador de seteo
1884 1774
1885 1775 setFile += 1
1886 1776
1887 1777 #If this is a new day it resets some values
1888 1778 if self.dataOut.datatime.date() > self.fileDate:
1889 1779 setFile = 0
1890 1780 self.nTotalBlocks = 0
1891 1781
1892 1782 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1893 1783
1894 1784 filename = os.path.join( path, subfolder, filen )
1895 1785
1896 1786 fp = open( filename,'wb' )
1897 1787
1898 1788 self.blockIndex = 0
1899 1789
1900 1790 #guardando atributos
1901 1791 self.filename = filename
1902 1792 self.subfolder = subfolder
1903 1793 self.fp = fp
1904 1794 self.setFile = setFile
1905 1795 self.flagIsNewFile = 1
1906 1796 self.fileDate = self.dataOut.datatime.date()
1907 1797
1908 1798 self.setFirstHeader()
1909 1799
1910 1800 print '[Writing] Opening file: %s'%self.filename
1911 1801
1912 1802 self.__writeFirstHeader()
1913 1803
1914 1804 return 1
1915 1805
1916 1806 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1917 1807 """
1918 1808 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1919 1809
1920 1810 Inputs:
1921 1811 path : directory where data will be saved
1922 1812 profilesPerBlock : number of profiles per block
1923 1813 set : initial file set
1924 1814 datatype : An integer number that defines data type:
1925 1815 0 : int8 (1 byte)
1926 1816 1 : int16 (2 bytes)
1927 1817 2 : int32 (4 bytes)
1928 1818 3 : int64 (8 bytes)
1929 1819 4 : float32 (4 bytes)
1930 1820 5 : double64 (8 bytes)
1931 1821
1932 1822 Return:
1933 1823 0 : Si no realizo un buen seteo
1934 1824 1 : Si realizo un buen seteo
1935 1825 """
1936 1826
1937 1827 if ext == None:
1938 1828 ext = self.ext
1939 1829
1940 1830 self.ext = ext.lower()
1941 1831
1942 1832 self.path = path
1943 1833
1944 1834 if set is None:
1945 1835 self.setFile = -1
1946 1836 else:
1947 1837 self.setFile = set - 1
1948 1838
1949 1839 self.blocksPerFile = blocksPerFile
1950 1840
1951 1841 self.profilesPerBlock = profilesPerBlock
1952 1842
1953 1843 self.dataOut = dataOut
1954 1844 self.fileDate = self.dataOut.datatime.date()
1955 1845 #By default
1956 1846 self.dtype = self.dataOut.dtype
1957 1847
1958 1848 if datatype is not None:
1959 1849 self.dtype = get_numpy_dtype(datatype)
1960 1850
1961 1851 if not(self.setNextFile()):
1962 1852 print "[Writing] There isn't a next file"
1963 1853 return 0
1964 1854
1965 1855 self.setBlockDimension()
1966 1856
1967 1857 return 1
1968 1858
1969 1859 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1970 1860
1971 1861 if not(self.isConfig):
1972 1862
1973 1863 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1974 1864 self.isConfig = True
1975 1865
1976 1866 self.putData()
General Comments 0
You need to be logged in to leave comments. Login now