##// END OF EJS Templates
schain-jc
RSS
Raise Warning when no more files
jespinoza -
r1150:083c61a33c80
parent child
Show More
Side by Side Unified
Context file:
r1150:083c61a33c80 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1824 +1,1826
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time
14 14 import datetime
15 15 import traceback
16 16 import zmq
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25 25 from schainpy.utils import log
26 import schainpy.admin
26 27
27 28 LOCALTIME = True
28 29
29 30
30 31 def isNumber(cad):
31 32 """
32 33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
33 34
34 35 Excepciones:
35 36 Si un determinado string no puede ser convertido a numero
36 37 Input:
37 38 str, string al cual se le analiza para determinar si convertible a un numero o no
38 39
39 40 Return:
40 41 True : si el string es uno numerico
41 42 False : no es un string numerico
42 43 """
43 44 try:
44 45 float(cad)
45 46 return True
46 47 except:
47 48 return False
48 49
49 50
50 51 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
51 52 """
52 53 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
53 54
54 55 Inputs:
55 56 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
56 57
57 58 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
58 59 segundos contados desde 01/01/1970.
59 60 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
60 61 segundos contados desde 01/01/1970.
61 62
62 63 Return:
63 64 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
64 65 fecha especificado, de lo contrario retorna False.
65 66
66 67 Excepciones:
67 68 Si el archivo no existe o no puede ser abierto
68 69 Si la cabecera no puede ser leida.
69 70
70 71 """
71 72 basicHeaderObj = BasicHeader(LOCALTIME)
72 73
73 74 try:
74 75 fp = open(filename, 'rb')
75 76 except IOError:
76 77 print "The file %s can't be opened" % (filename)
77 78 return 0
78 79
79 80 sts = basicHeaderObj.read(fp)
80 81 fp.close()
81 82
82 83 if not(sts):
83 84 print "Skipping the file %s because it has not a valid header" % (filename)
84 85 return 0
85 86
86 87 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
87 88 return 0
88 89
89 90 return 1
90 91
91 92
92 93 def isTimeInRange(thisTime, startTime, endTime):
93 94 if endTime >= startTime:
94 95 if (thisTime < startTime) or (thisTime > endTime):
95 96 return 0
96 97 return 1
97 98 else:
98 99 if (thisTime < startTime) and (thisTime > endTime):
99 100 return 0
100 101 return 1
101 102
102 103
103 104 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
104 105 """
105 106 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
106 107
107 108 Inputs:
108 109 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
109 110
110 111 startDate : fecha inicial del rango seleccionado en formato datetime.date
111 112
112 113 endDate : fecha final del rango seleccionado en formato datetime.date
113 114
114 115 startTime : tiempo inicial del rango seleccionado en formato datetime.time
115 116
116 117 endTime : tiempo final del rango seleccionado en formato datetime.time
117 118
118 119 Return:
119 120 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
120 121 fecha especificado, de lo contrario retorna False.
121 122
122 123 Excepciones:
123 124 Si el archivo no existe o no puede ser abierto
124 125 Si la cabecera no puede ser leida.
125 126
126 127 """
127 128
128 129 try:
129 130 fp = open(filename, 'rb')
130 131 except IOError:
131 132 print "The file %s can't be opened" % (filename)
132 133 return None
133 134
134 135 firstBasicHeaderObj = BasicHeader(LOCALTIME)
135 136 systemHeaderObj = SystemHeader()
136 137 radarControllerHeaderObj = RadarControllerHeader()
137 138 processingHeaderObj = ProcessingHeader()
138 139
139 140 lastBasicHeaderObj = BasicHeader(LOCALTIME)
140 141
141 142 sts = firstBasicHeaderObj.read(fp)
142 143
143 144 if not(sts):
144 145 print "[Reading] Skipping the file %s because it has not a valid header" % (filename)
145 146 return None
146 147
147 148 if not systemHeaderObj.read(fp):
148 149 return None
149 150
150 151 if not radarControllerHeaderObj.read(fp):
151 152 return None
152 153
153 154 if not processingHeaderObj.read(fp):
154 155 return None
155 156
156 157 filesize = os.path.getsize(filename)
157 158
158 159 offset = processingHeaderObj.blockSize + 24 # header size
159 160
160 161 if filesize <= offset:
161 162 print "[Reading] %s: This file has not enough data" % filename
162 163 return None
163 164
164 165 fp.seek(-offset, 2)
165 166
166 167 sts = lastBasicHeaderObj.read(fp)
167 168
168 169 fp.close()
169 170
170 171 thisDatetime = lastBasicHeaderObj.datatime
171 172 thisTime_last_block = thisDatetime.time()
172 173
173 174 thisDatetime = firstBasicHeaderObj.datatime
174 175 thisDate = thisDatetime.date()
175 176 thisTime_first_block = thisDatetime.time()
176 177
177 178 # General case
178 179 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
179 180 #-----------o----------------------------o-----------
180 181 # startTime endTime
181 182
182 183 if endTime >= startTime:
183 184 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
184 185 return None
185 186
186 187 return thisDatetime
187 188
188 189 # If endTime < startTime then endTime belongs to the next day
189 190
190 191 #<<<<<<<<<<<o o>>>>>>>>>>>
191 192 #-----------o----------------------------o-----------
192 193 # endTime startTime
193 194
194 195 if (thisDate == startDate) and (thisTime_last_block < startTime):
195 196 return None
196 197
197 198 if (thisDate == endDate) and (thisTime_first_block > endTime):
198 199 return None
199 200
200 201 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
201 202 return None
202 203
203 204 return thisDatetime
204 205
205 206
206 207 def isFolderInDateRange(folder, startDate=None, endDate=None):
207 208 """
208 209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
209 210
210 211 Inputs:
211 212 folder : nombre completo del directorio.
212 213 Su formato deberia ser "/path_root/?YYYYDDD"
213 214
214 215 siendo:
215 216 YYYY : Anio (ejemplo 2015)
216 217 DDD : Dia del anio (ejemplo 305)
217 218
218 219 startDate : fecha inicial del rango seleccionado en formato datetime.date
219 220
220 221 endDate : fecha final del rango seleccionado en formato datetime.date
221 222
222 223 Return:
223 224 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
224 225 fecha especificado, de lo contrario retorna False.
225 226 Excepciones:
226 227 Si el directorio no tiene el formato adecuado
227 228 """
228 229
229 230 basename = os.path.basename(folder)
230 231
231 232 if not isRadarFolder(basename):
232 233 print "The folder %s has not the rigth format" % folder
233 234 return 0
234 235
235 236 if startDate and endDate:
236 237 thisDate = getDateFromRadarFolder(basename)
237 238
238 239 if thisDate < startDate:
239 240 return 0
240 241
241 242 if thisDate > endDate:
242 243 return 0
243 244
244 245 return 1
245 246
246 247
247 248 def isFileInDateRange(filename, startDate=None, endDate=None):
248 249 """
249 250 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
250 251
251 252 Inputs:
252 253 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
253 254
254 255 Su formato deberia ser "?YYYYDDDsss"
255 256
256 257 siendo:
257 258 YYYY : Anio (ejemplo 2015)
258 259 DDD : Dia del anio (ejemplo 305)
259 260 sss : set
260 261
261 262 startDate : fecha inicial del rango seleccionado en formato datetime.date
262 263
263 264 endDate : fecha final del rango seleccionado en formato datetime.date
264 265
265 266 Return:
266 267 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
267 268 fecha especificado, de lo contrario retorna False.
268 269 Excepciones:
269 270 Si el archivo no tiene el formato adecuado
270 271 """
271 272
272 273 basename = os.path.basename(filename)
273 274
274 275 if not isRadarFile(basename):
275 276 print "The filename %s has not the rigth format" % filename
276 277 return 0
277 278
278 279 if startDate and endDate:
279 280 thisDate = getDateFromRadarFile(basename)
280 281
281 282 if thisDate < startDate:
282 283 return 0
283 284
284 285 if thisDate > endDate:
285 286 return 0
286 287
287 288 return 1
288 289
289 290
290 291 def getFileFromSet(path, ext, set):
291 292 validFilelist = []
292 293 fileList = os.listdir(path)
293 294
294 295 # 0 1234 567 89A BCDE
295 296 # H YYYY DDD SSS .ext
296 297
297 298 for thisFile in fileList:
298 299 try:
299 300 year = int(thisFile[1:5])
300 301 doy = int(thisFile[5:8])
301 302 except:
302 303 continue
303 304
304 305 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
305 306 continue
306 307
307 308 validFilelist.append(thisFile)
308 309
309 310 myfile = fnmatch.filter(
310 311 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
311 312
312 313 if len(myfile) != 0:
313 314 return myfile[0]
314 315 else:
315 316 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
316 317 print 'the filename %s does not exist' % filename
317 318 print '...going to the last file: '
318 319
319 320 if validFilelist:
320 321 validFilelist = sorted(validFilelist, key=str.lower)
321 322 return validFilelist[-1]
322 323
323 324 return None
324 325
325 326
326 327 def getlastFileFromPath(path, ext):
327 328 """
328 329 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
329 330 al final de la depuracion devuelve el ultimo file de la lista que quedo.
330 331
331 332 Input:
332 333 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
333 334 ext : extension de los files contenidos en una carpeta
334 335
335 336 Return:
336 337 El ultimo file de una determinada carpeta, no se considera el path.
337 338 """
338 339 validFilelist = []
339 340 fileList = os.listdir(path)
340 341
341 342 # 0 1234 567 89A BCDE
342 343 # H YYYY DDD SSS .ext
343 344
344 345 for thisFile in fileList:
345 346
346 347 year = thisFile[1:5]
347 348 if not isNumber(year):
348 349 continue
349 350
350 351 doy = thisFile[5:8]
351 352 if not isNumber(doy):
352 353 continue
353 354
354 355 year = int(year)
355 356 doy = int(doy)
356 357
357 358 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
358 359 continue
359 360
360 361 validFilelist.append(thisFile)
361 362
362 363 if validFilelist:
363 364 validFilelist = sorted(validFilelist, key=str.lower)
364 365 return validFilelist[-1]
365 366
366 367 return None
367 368
368 369
369 370 def checkForRealPath(path, foldercounter, year, doy, set, ext):
370 371 """
371 372 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
372 373 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
373 374 el path exacto de un determinado file.
374 375
375 376 Example :
376 377 nombre correcto del file es .../.../D2009307/P2009307367.ext
377 378
378 379 Entonces la funcion prueba con las siguientes combinaciones
379 380 .../.../y2009307367.ext
380 381 .../.../Y2009307367.ext
381 382 .../.../x2009307/y2009307367.ext
382 383 .../.../x2009307/Y2009307367.ext
383 384 .../.../X2009307/y2009307367.ext
384 385 .../.../X2009307/Y2009307367.ext
385 386 siendo para este caso, la ultima combinacion de letras, identica al file buscado
386 387
387 388 Return:
388 389 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
389 390 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
390 391 para el filename
391 392 """
392 393 fullfilename = None
393 394 find_flag = False
394 395 filename = None
395 396
396 397 prefixDirList = [None, 'd', 'D']
397 398 if ext.lower() == ".r": # voltage
398 399 prefixFileList = ['d', 'D']
399 400 elif ext.lower() == ".pdata": # spectra
400 401 prefixFileList = ['p', 'P']
401 402 else:
402 403 return None, filename
403 404
404 405 # barrido por las combinaciones posibles
405 406 for prefixDir in prefixDirList:
406 407 thispath = path
407 408 if prefixDir != None:
408 409 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
409 410 if foldercounter == 0:
410 411 thispath = os.path.join(path, "%s%04d%03d" %
411 412 (prefixDir, year, doy))
412 413 else:
413 414 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
414 415 prefixDir, year, doy, foldercounter))
415 416 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
416 417 # formo el nombre del file xYYYYDDDSSS.ext
417 418 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
418 419 fullfilename = os.path.join(
419 420 thispath, filename) # formo el path completo
420 421
421 422 if os.path.exists(fullfilename): # verifico que exista
422 423 find_flag = True
423 424 break
424 425 if find_flag:
425 426 break
426 427
427 428 if not(find_flag):
428 429 return None, filename
429 430
430 431 return fullfilename, filename
431 432
432 433
433 434 def isRadarFolder(folder):
434 435 try:
435 436 year = int(folder[1:5])
436 437 doy = int(folder[5:8])
437 438 except:
438 439 return 0
439 440
440 441 return 1
441 442
442 443
443 444 def isRadarFile(file):
444 445 try:
445 446 year = int(file[1:5])
446 447 doy = int(file[5:8])
447 448 set = int(file[8:11])
448 449 except:
449 450 return 0
450 451
451 452 return 1
452 453
453 454
454 455 def getDateFromRadarFile(file):
455 456 try:
456 457 year = int(file[1:5])
457 458 doy = int(file[5:8])
458 459 set = int(file[8:11])
459 460 except:
460 461 return None
461 462
462 463 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
463 464 return thisDate
464 465
465 466
466 467 def getDateFromRadarFolder(folder):
467 468 try:
468 469 year = int(folder[1:5])
469 470 doy = int(folder[5:8])
470 471 except:
471 472 return None
472 473
473 474 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
474 475 return thisDate
475 476
476 477
477 478 class JRODataIO:
478 479
479 480 c = 3E8
480 481
481 482 isConfig = False
482 483
483 484 basicHeaderObj = None
484 485
485 486 systemHeaderObj = None
486 487
487 488 radarControllerHeaderObj = None
488 489
489 490 processingHeaderObj = None
490 491
491 492 dtype = None
492 493
493 494 pathList = []
494 495
495 496 filenameList = []
496 497
497 498 filename = None
498 499
499 500 ext = None
500 501
501 502 flagIsNewFile = 1
502 503
503 504 flagDiscontinuousBlock = 0
504 505
505 506 flagIsNewBlock = 0
506 507
507 508 fp = None
508 509
509 510 firstHeaderSize = 0
510 511
511 512 basicHeaderSize = 24
512 513
513 514 versionFile = 1103
514 515
515 516 fileSize = None
516 517
517 518 # ippSeconds = None
518 519
519 520 fileSizeByHeader = None
520 521
521 522 fileIndex = None
522 523
523 524 profileIndex = None
524 525
525 526 blockIndex = None
526 527
527 528 nTotalBlocks = None
528 529
529 530 maxTimeStep = 30
530 531
531 532 lastUTTime = None
532 533
533 534 datablock = None
534 535
535 536 dataOut = None
536 537
537 538 blocksize = None
538 539
539 540 getByBlock = False
540 541
541 542 def __init__(self):
542 543
543 544 raise NotImplementedError
544 545
545 546 def run(self):
546 547
547 548 raise NotImplementedError
548 549
549 550 def getDtypeWidth(self):
550 551
551 552 dtype_index = get_dtype_index(self.dtype)
552 553 dtype_width = get_dtype_width(dtype_index)
553 554
554 555 return dtype_width
555 556
556 557 def getAllowedArgs(self):
557 558 if hasattr(self, '__attrs__'):
558 559 return self.__attrs__
559 560 else:
560 561 return inspect.getargspec(self.run).args
561 562
562 563
563 564 class JRODataReader(JRODataIO):
564 565
565 566 online = 0
566 567
567 568 realtime = 0
568 569
569 570 nReadBlocks = 0
570 571
571 572 delay = 10 # number of seconds waiting a new file
572 573
573 574 nTries = 3 # quantity tries
574 575
575 576 nFiles = 3 # number of files for searching
576 577
577 578 path = None
578 579
579 580 foldercounter = 0
580 581
581 582 flagNoMoreFiles = 0
582 583
583 584 datetimeList = []
584 585
585 586 __isFirstTimeOnline = 1
586 587
587 588 __printInfo = True
588 589
589 590 profileIndex = None
590 591
591 592 nTxs = 1
592 593
593 594 txIndex = None
594 595
595 596 # Added--------------------
596 597
597 598 selBlocksize = None
598 599
599 600 selBlocktime = None
600 601
601 602 def __init__(self):
602 603 """
603 604 This class is used to find data files
604 605
605 606 Example:
606 607 reader = JRODataReader()
607 608 fileList = reader.findDataFiles()
608 609
609 610 """
610 611 pass
611 612
612 613 def createObjByDefault(self):
613 614 """
614 615
615 616 """
616 617 raise NotImplementedError
617 618
618 619 def getBlockDimension(self):
619 620
620 621 raise NotImplementedError
621 622
622 623 def searchFilesOffLine(self,
623 624 path,
624 625 startDate=None,
625 626 endDate=None,
626 627 startTime=datetime.time(0, 0, 0),
627 628 endTime=datetime.time(23, 59, 59),
628 629 set=None,
629 630 expLabel='',
630 631 ext='.r',
631 632 cursor=None,
632 633 skip=None,
633 634 walk=True):
634 635
635 636 self.filenameList = []
636 637 self.datetimeList = []
637 638
638 639 pathList = []
639 640
640 641 dateList, pathList = self.findDatafiles(
641 642 path, startDate, endDate, expLabel, ext, walk, include_path=True)
642 643
643 644 if dateList == []:
644 645 return [], []
645 646
646 647 if len(dateList) > 1:
647 648 print "[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList))
648 649 else:
649 650 print "[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0])
650 651
651 652 filenameList = []
652 653 datetimeList = []
653 654
654 655 for thisPath in pathList:
655 656
656 657 fileList = glob.glob1(thisPath, "*%s" % ext)
657 658 fileList.sort()
658 659
659 660 for file in fileList:
660 661
661 662 filename = os.path.join(thisPath, file)
662 663
663 664 if not isFileInDateRange(filename, startDate, endDate):
664 665 continue
665 666
666 667 thisDatetime = isFileInTimeRange(
667 668 filename, startDate, endDate, startTime, endTime)
668 669
669 670 if not(thisDatetime):
670 671 continue
671 672
672 673 filenameList.append(filename)
673 674 datetimeList.append(thisDatetime)
674 675
675 676 if cursor is not None and skip is not None:
676 677 filenameList = filenameList[cursor * skip:cursor * skip + skip]
677 678 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
678 679
679 680 if not(filenameList):
680 681 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path)
681 682 return [], []
682 683
683 684 print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
684 685
685 686 # for i in range(len(filenameList)):
686 687 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
687 688
688 689 self.filenameList = filenameList
689 690 self.datetimeList = datetimeList
690 691
691 692 return pathList, filenameList
692 693
693 694 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
694 695 """
695 696 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
696 697 devuelve el archivo encontrado ademas de otros datos.
697 698
698 699 Input:
699 700 path : carpeta donde estan contenidos los files que contiene data
700 701
701 702 expLabel : Nombre del subexperimento (subfolder)
702 703
703 704 ext : extension de los files
704 705
705 706 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
706 707
707 708 Return:
708 709 directory : eL directorio donde esta el file encontrado
709 710 filename : el ultimo file de una determinada carpeta
710 711 year : el anho
711 712 doy : el numero de dia del anho
712 713 set : el set del archivo
713 714
714 715
715 716 """
716 717 if not os.path.isdir(path):
717 718 return None, None, None, None, None, None
718 719
719 720 dirList = []
720 721
721 722 if not walk:
722 723 fullpath = path
723 724 foldercounter = 0
724 725 else:
725 726 # Filtra solo los directorios
726 727 for thisPath in os.listdir(path):
727 728 if not os.path.isdir(os.path.join(path, thisPath)):
728 729 continue
729 730 if not isRadarFolder(thisPath):
730 731 continue
731 732
732 733 dirList.append(thisPath)
733 734
734 735 if not(dirList):
735 736 return None, None, None, None, None, None
736 737
737 738 dirList = sorted(dirList, key=str.lower)
738 739
739 740 doypath = dirList[-1]
740 741 foldercounter = int(doypath.split('_')[1]) if len(
741 742 doypath.split('_')) > 1 else 0
742 743 fullpath = os.path.join(path, doypath, expLabel)
743 744
744 745 print "[Reading] %s folder was found: " % (fullpath)
745 746
746 747 if set == None:
747 748 filename = getlastFileFromPath(fullpath, ext)
748 749 else:
749 750 filename = getFileFromSet(fullpath, ext, set)
750 751
751 752 if not(filename):
752 753 return None, None, None, None, None, None
753 754
754 755 print "[Reading] %s file was found" % (filename)
755 756
756 757 if not(self.__verifyFile(os.path.join(fullpath, filename))):
757 758 return None, None, None, None, None, None
758 759
759 760 year = int(filename[1:5])
760 761 doy = int(filename[5:8])
761 762 set = int(filename[8:11])
762 763
763 764 return fullpath, foldercounter, filename, year, doy, set
764 765
765 766 def __setNextFileOffline(self):
766 767
767 768 idFile = self.fileIndex
768 769
769 770 while (True):
770 771 idFile += 1
771 772 if not(idFile < len(self.filenameList)):
772 773 self.flagNoMoreFiles = 1
773 774 # print "[Reading] No more Files"
774 775 return 0
775 776
776 777 filename = self.filenameList[idFile]
777 778
778 779 if not(self.__verifyFile(filename)):
779 780 continue
780 781
781 782 fileSize = os.path.getsize(filename)
782 783 fp = open(filename, 'rb')
783 784 break
784 785
785 786 self.flagIsNewFile = 1
786 787 self.fileIndex = idFile
787 788 self.filename = filename
788 789 self.fileSize = fileSize
789 790 self.fp = fp
790 791
791 792 # print "[Reading] Setting the file: %s"%self.filename
792 793
793 794 return 1
794 795
795 796 def __setNextFileOnline(self):
796 797 """
797 798 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
798 799 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
799 800 siguientes.
800 801
801 802 Affected:
802 803 self.flagIsNewFile
803 804 self.filename
804 805 self.fileSize
805 806 self.fp
806 807 self.set
807 808 self.flagNoMoreFiles
808 809
809 810 Return:
810 811 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
811 812 1 : si el file fue abierto con exito y esta listo a ser leido
812 813
813 814 Excepciones:
814 815 Si un determinado file no puede ser abierto
815 816 """
816 817 nFiles = 0
817 818 fileOk_flag = False
818 819 firstTime_flag = True
819 820
820 821 self.set += 1
821 822
822 823 if self.set > 999:
823 824 self.set = 0
824 825 self.foldercounter += 1
825 826
826 827 # busca el 1er file disponible
827 828 fullfilename, filename = checkForRealPath(
828 829 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
829 830 if fullfilename:
830 831 if self.__verifyFile(fullfilename, False):
831 832 fileOk_flag = True
832 833
833 834 # si no encuentra un file entonces espera y vuelve a buscar
834 835 if not(fileOk_flag):
835 836 # busco en los siguientes self.nFiles+1 files posibles
836 837 for nFiles in range(self.nFiles + 1):
837 838
838 839 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
839 840 tries = self.nTries
840 841 else:
841 842 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
842 843
843 844 for nTries in range(tries):
844 845 if firstTime_flag:
845 846 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1)
846 847 sleep(self.delay)
847 848 else:
848 849 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
849 850
850 851 fullfilename, filename = checkForRealPath(
851 852 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
852 853 if fullfilename:
853 854 if self.__verifyFile(fullfilename):
854 855 fileOk_flag = True
855 856 break
856 857
857 858 if fileOk_flag:
858 859 break
859 860
860 861 firstTime_flag = False
861 862
862 863 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
863 864 self.set += 1
864 865
865 866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
866 867 if nFiles == (self.nFiles - 1):
867 868 self.set = 0
868 869 self.doy += 1
869 870 self.foldercounter = 0
870 871
871 872 if fileOk_flag:
872 873 self.fileSize = os.path.getsize(fullfilename)
873 874 self.filename = fullfilename
874 875 self.flagIsNewFile = 1
875 876 if self.fp != None:
876 877 self.fp.close()
877 878 self.fp = open(fullfilename, 'rb')
878 879 self.flagNoMoreFiles = 0
879 880 # print '[Reading] Setting the file: %s' % fullfilename
880 881 else:
881 882 self.fileSize = 0
882 883 self.filename = None
883 884 self.flagIsNewFile = 0
884 885 self.fp = None
885 886 self.flagNoMoreFiles = 1
886 887 # print '[Reading] No more files to read'
887 888
888 889 return fileOk_flag
889 890
890 891 def setNextFile(self):
891 892 if self.fp != None:
892 893 self.fp.close()
893 894
894 895 if self.online:
895 896 newFile = self.__setNextFileOnline()
896 897 else:
897 898 newFile = self.__setNextFileOffline()
898 899
899 900 if not(newFile):
900 901 print '[Reading] No more files to read'
901 902 return 0
902 903
903 904 if self.verbose:
904 905 print '[Reading] Setting the file: %s' % self.filename
905 906
906 907 self.__readFirstHeader()
907 908 self.nReadBlocks = 0
908 909 return 1
909 910
910 911 def __waitNewBlock(self):
911 912 """
912 913 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
913 914
914 915 Si el modo de lectura es OffLine siempre retorn 0
915 916 """
916 917 if not self.online:
917 918 return 0
918 919
919 920 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
920 921 return 0
921 922
922 923 currentPointer = self.fp.tell()
923 924
924 925 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
925 926
926 927 for nTries in range(self.nTries):
927 928
928 929 self.fp.close()
929 930 self.fp = open(self.filename, 'rb')
930 931 self.fp.seek(currentPointer)
931 932
932 933 self.fileSize = os.path.getsize(self.filename)
933 934 currentSize = self.fileSize - currentPointer
934 935
935 936 if (currentSize >= neededSize):
936 937 self.basicHeaderObj.read(self.fp)
937 938 return 1
938 939
939 940 if self.fileSize == self.fileSizeByHeader:
940 941 # self.flagEoF = True
941 942 return 0
942 943
943 944 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
944 945 sleep(self.delay)
945 946
946 947 return 0
947 948
948 949 def waitDataBlock(self, pointer_location):
949 950
950 951 currentPointer = pointer_location
951 952
952 953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
953 954
954 955 for nTries in range(self.nTries):
955 956 self.fp.close()
956 957 self.fp = open(self.filename, 'rb')
957 958 self.fp.seek(currentPointer)
958 959
959 960 self.fileSize = os.path.getsize(self.filename)
960 961 currentSize = self.fileSize - currentPointer
961 962
962 963 if (currentSize >= neededSize):
963 964 return 1
964 965
965 966 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
966 967 sleep(self.delay)
967 968
968 969 return 0
969 970
970 971 def __jumpToLastBlock(self):
971 972
972 973 if not(self.__isFirstTimeOnline):
973 974 return
974 975
975 976 csize = self.fileSize - self.fp.tell()
976 977 blocksize = self.processingHeaderObj.blockSize
977 978
978 979 # salta el primer bloque de datos
979 980 if csize > self.processingHeaderObj.blockSize:
980 981 self.fp.seek(self.fp.tell() + blocksize)
981 982 else:
982 983 return
983 984
984 985 csize = self.fileSize - self.fp.tell()
985 986 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
986 987 while True:
987 988
988 989 if self.fp.tell() < self.fileSize:
989 990 self.fp.seek(self.fp.tell() + neededsize)
990 991 else:
991 992 self.fp.seek(self.fp.tell() - neededsize)
992 993 break
993 994
994 995 # csize = self.fileSize - self.fp.tell()
995 996 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
996 997 # factor = int(csize/neededsize)
997 998 # if factor > 0:
998 999 # self.fp.seek(self.fp.tell() + factor*neededsize)
999 1000
1000 1001 self.flagIsNewFile = 0
1001 1002 self.__isFirstTimeOnline = 0
1002 1003
1003 1004 def __setNewBlock(self):
1004 1005 # if self.server is None:
1005 1006 if self.fp == None:
1006 1007 return 0
1007 1008
1008 1009 # if self.online:
1009 1010 # self.__jumpToLastBlock()
1010 1011
1011 1012 if self.flagIsNewFile:
1012 1013 self.lastUTTime = self.basicHeaderObj.utc
1013 1014 return 1
1014 1015
1015 1016 if self.realtime:
1016 1017 self.flagDiscontinuousBlock = 1
1017 1018 if not(self.setNextFile()):
1018 1019 return 0
1019 1020 else:
1020 1021 return 1
1021 1022 # if self.server is None:
1022 1023 currentSize = self.fileSize - self.fp.tell()
1023 1024 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1024 1025 if (currentSize >= neededSize):
1025 1026 self.basicHeaderObj.read(self.fp)
1026 1027 self.lastUTTime = self.basicHeaderObj.utc
1027 1028 return 1
1028 1029 # else:
1029 1030 # self.basicHeaderObj.read(self.zHeader)
1030 1031 # self.lastUTTime = self.basicHeaderObj.utc
1031 1032 # return 1
1032 1033 if self.__waitNewBlock():
1033 1034 self.lastUTTime = self.basicHeaderObj.utc
1034 1035 return 1
1035 1036 # if self.server is None:
1036 1037 if not(self.setNextFile()):
1037 1038 return 0
1038 1039
1039 1040 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1040 1041 self.lastUTTime = self.basicHeaderObj.utc
1041 1042
1042 1043 self.flagDiscontinuousBlock = 0
1043 1044
1044 1045 if deltaTime > self.maxTimeStep:
1045 1046 self.flagDiscontinuousBlock = 1
1046 1047
1047 1048 return 1
1048 1049
1049 1050 def readNextBlock(self):
1050 1051
1051 1052 # Skip block out of startTime and endTime
1052 1053 while True:
1053 1054 if not(self.__setNewBlock()):
1055 raise(schainpy.admin.SchainWarning('No more files'))
1054 1056 return 0
1055 1057
1056 1058 if not(self.readBlock()):
1057 1059 return 0
1058 1060
1059 1061 self.getBasicHeader()
1060 1062 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1061 1063 print "[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1062 1064 self.processingHeaderObj.dataBlocksPerFile,
1063 1065 self.dataOut.datatime.ctime())
1064 1066 continue
1065 1067
1066 1068 break
1067 1069
1068 1070 if self.verbose:
1069 1071 print "[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1070 1072 self.processingHeaderObj.dataBlocksPerFile,
1071 1073 self.dataOut.datatime.ctime())
1072 1074 return 1
1073 1075
1074 1076 def __readFirstHeader(self):
1075 1077
1076 1078 self.basicHeaderObj.read(self.fp)
1077 1079 self.systemHeaderObj.read(self.fp)
1078 1080 self.radarControllerHeaderObj.read(self.fp)
1079 1081 self.processingHeaderObj.read(self.fp)
1080 1082
1081 1083 self.firstHeaderSize = self.basicHeaderObj.size
1082 1084
1083 1085 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1084 1086 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1085 1087 if datatype == 0:
1086 1088 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1087 1089 elif datatype == 1:
1088 1090 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1089 1091 elif datatype == 2:
1090 1092 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1091 1093 elif datatype == 3:
1092 1094 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1093 1095 elif datatype == 4:
1094 1096 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1095 1097 elif datatype == 5:
1096 1098 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1097 1099 else:
1098 1100 raise ValueError, 'Data type was not defined'
1099 1101
1100 1102 self.dtype = datatype_str
1101 1103 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1102 1104 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1103 1105 self.firstHeaderSize + self.basicHeaderSize * \
1104 1106 (self.processingHeaderObj.dataBlocksPerFile - 1)
1105 1107 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1106 1108 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1107 1109 self.getBlockDimension()
1108 1110
1109 1111 def __verifyFile(self, filename, msgFlag=True):
1110 1112
1111 1113 msg = None
1112 1114
1113 1115 try:
1114 1116 fp = open(filename, 'rb')
1115 1117 except IOError:
1116 1118
1117 1119 if msgFlag:
1118 1120 print "[Reading] File %s can't be opened" % (filename)
1119 1121
1120 1122 return False
1121 1123
1122 1124 currentPosition = fp.tell()
1123 1125 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1124 1126
1125 1127 if neededSize == 0:
1126 1128 basicHeaderObj = BasicHeader(LOCALTIME)
1127 1129 systemHeaderObj = SystemHeader()
1128 1130 radarControllerHeaderObj = RadarControllerHeader()
1129 1131 processingHeaderObj = ProcessingHeader()
1130 1132
1131 1133 if not(basicHeaderObj.read(fp)):
1132 1134 fp.close()
1133 1135 return False
1134 1136
1135 1137 if not(systemHeaderObj.read(fp)):
1136 1138 fp.close()
1137 1139 return False
1138 1140
1139 1141 if not(radarControllerHeaderObj.read(fp)):
1140 1142 fp.close()
1141 1143 return False
1142 1144
1143 1145 if not(processingHeaderObj.read(fp)):
1144 1146 fp.close()
1145 1147 return False
1146 1148
1147 1149 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1148 1150 else:
1149 1151 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1150 1152
1151 1153 fp.close()
1152 1154
1153 1155 fileSize = os.path.getsize(filename)
1154 1156 currentSize = fileSize - currentPosition
1155 1157
1156 1158 if currentSize < neededSize:
1157 1159 if msgFlag and (msg != None):
1158 1160 print msg
1159 1161 return False
1160 1162
1161 1163 return True
1162 1164
1163 1165 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1164 1166
1165 1167 path_empty = True
1166 1168
1167 1169 dateList = []
1168 1170 pathList = []
1169 1171
1170 1172 multi_path = path.split(',')
1171 1173
1172 1174 if not walk:
1173 1175
1174 1176 for single_path in multi_path:
1175 1177
1176 1178 if not os.path.isdir(single_path):
1177 1179 continue
1178 1180
1179 1181 fileList = glob.glob1(single_path, "*" + ext)
1180 1182
1181 1183 if not fileList:
1182 1184 continue
1183 1185
1184 1186 path_empty = False
1185 1187
1186 1188 fileList.sort()
1187 1189
1188 1190 for thisFile in fileList:
1189 1191
1190 1192 if not os.path.isfile(os.path.join(single_path, thisFile)):
1191 1193 continue
1192 1194
1193 1195 if not isRadarFile(thisFile):
1194 1196 continue
1195 1197
1196 1198 if not isFileInDateRange(thisFile, startDate, endDate):
1197 1199 continue
1198 1200
1199 1201 thisDate = getDateFromRadarFile(thisFile)
1200 1202
1201 1203 if thisDate in dateList:
1202 1204 continue
1203 1205
1204 1206 dateList.append(thisDate)
1205 1207 pathList.append(single_path)
1206 1208
1207 1209 else:
1208 1210 for single_path in multi_path:
1209 1211
1210 1212 if not os.path.isdir(single_path):
1211 1213 continue
1212 1214
1213 1215 dirList = []
1214 1216
1215 1217 for thisPath in os.listdir(single_path):
1216 1218
1217 1219 if not os.path.isdir(os.path.join(single_path, thisPath)):
1218 1220 continue
1219 1221
1220 1222 if not isRadarFolder(thisPath):
1221 1223 continue
1222 1224
1223 1225 if not isFolderInDateRange(thisPath, startDate, endDate):
1224 1226 continue
1225 1227
1226 1228 dirList.append(thisPath)
1227 1229
1228 1230 if not dirList:
1229 1231 continue
1230 1232
1231 1233 dirList.sort()
1232 1234
1233 1235 for thisDir in dirList:
1234 1236
1235 1237 datapath = os.path.join(single_path, thisDir, expLabel)
1236 1238 fileList = glob.glob1(datapath, "*" + ext)
1237 1239
1238 1240 if not fileList:
1239 1241 continue
1240 1242
1241 1243 path_empty = False
1242 1244
1243 1245 thisDate = getDateFromRadarFolder(thisDir)
1244 1246
1245 1247 pathList.append(datapath)
1246 1248 dateList.append(thisDate)
1247 1249
1248 1250 dateList.sort()
1249 1251
1250 1252 if walk:
1251 1253 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1252 1254 else:
1253 1255 pattern_path = multi_path[0]
1254 1256
1255 1257 if path_empty:
1256 1258 print "[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate)
1257 1259 else:
1258 1260 if not dateList:
1259 1261 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path)
1260 1262
1261 1263 if include_path:
1262 1264 return dateList, pathList
1263 1265
1264 1266 return dateList
1265 1267
1266 1268 def setup(self,
1267 1269 path=None,
1268 1270 startDate=None,
1269 1271 endDate=None,
1270 1272 startTime=datetime.time(0, 0, 0),
1271 1273 endTime=datetime.time(23, 59, 59),
1272 1274 set=None,
1273 1275 expLabel="",
1274 1276 ext=None,
1275 1277 online=False,
1276 1278 delay=60,
1277 1279 walk=True,
1278 1280 getblock=False,
1279 1281 nTxs=1,
1280 1282 realtime=False,
1281 1283 blocksize=None,
1282 1284 blocktime=None,
1283 1285 skip=None,
1284 1286 cursor=None,
1285 1287 warnings=True,
1286 1288 verbose=True,
1287 1289 server=None,
1288 1290 format=None,
1289 1291 oneDDict=None,
1290 1292 twoDDict=None,
1291 1293 ind2DList=None):
1292 1294 if server is not None:
1293 1295 if 'tcp://' in server:
1294 1296 address = server
1295 1297 else:
1296 1298 address = 'ipc:///tmp/%s' % server
1297 1299 self.server = address
1298 1300 self.context = zmq.Context()
1299 1301 self.receiver = self.context.socket(zmq.PULL)
1300 1302 self.receiver.connect(self.server)
1301 1303 time.sleep(0.5)
1302 1304 print '[Starting] ReceiverData from {}'.format(self.server)
1303 1305 else:
1304 1306 self.server = None
1305 1307 if path == None:
1306 1308 raise ValueError, "[Reading] The path is not valid"
1307 1309
1308 1310 if ext == None:
1309 1311 ext = self.ext
1310 1312
1311 1313 if online:
1312 1314 print "[Reading] Searching files in online mode..."
1313 1315
1314 1316 for nTries in range(self.nTries):
1315 1317 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1316 1318 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1317 1319
1318 1320 if fullpath:
1319 1321 break
1320 1322
1321 1323 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1)
1322 1324 sleep(self.delay)
1323 1325
1324 1326 if not(fullpath):
1325 1327 print "[Reading] There 'isn't any valid file in %s" % path
1326 1328 return
1327 1329
1328 1330 self.year = year
1329 1331 self.doy = doy
1330 1332 self.set = set - 1
1331 1333 self.path = path
1332 1334 self.foldercounter = foldercounter
1333 1335 last_set = None
1334 1336 else:
1335 1337 print "[Reading] Searching files in offline mode ..."
1336 1338 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1337 1339 startTime=startTime, endTime=endTime,
1338 1340 set=set, expLabel=expLabel, ext=ext,
1339 1341 walk=walk, cursor=cursor,
1340 1342 skip=skip)
1341 1343
1342 1344 if not(pathList):
1343 1345 self.fileIndex = -1
1344 1346 self.pathList = []
1345 1347 self.filenameList = []
1346 1348 return
1347 1349
1348 1350 self.fileIndex = -1
1349 1351 self.pathList = pathList
1350 1352 self.filenameList = filenameList
1351 1353 file_name = os.path.basename(filenameList[-1])
1352 1354 basename, ext = os.path.splitext(file_name)
1353 1355 last_set = int(basename[-3:])
1354 1356
1355 1357 self.online = online
1356 1358 self.realtime = realtime
1357 1359 self.delay = delay
1358 1360 ext = ext.lower()
1359 1361 self.ext = ext
1360 1362 self.getByBlock = getblock
1361 1363 self.nTxs = nTxs
1362 1364 self.startTime = startTime
1363 1365 self.endTime = endTime
1364 1366 self.endDate = endDate
1365 1367 self.startDate = startDate
1366 1368 # Added-----------------
1367 1369 self.selBlocksize = blocksize
1368 1370 self.selBlocktime = blocktime
1369 1371
1370 1372 # Verbose-----------
1371 1373 self.verbose = verbose
1372 1374 self.warnings = warnings
1373 1375
1374 1376 if not(self.setNextFile()):
1375 1377 if (startDate != None) and (endDate != None):
1376 1378 print "[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime())
1377 1379 elif startDate != None:
1378 1380 print "[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime())
1379 1381 else:
1380 1382 print "[Reading] No files"
1381 1383
1382 1384 self.fileIndex = -1
1383 1385 self.pathList = []
1384 1386 self.filenameList = []
1385 1387 return
1386 1388
1387 1389 # self.getBasicHeader()
1388 1390
1389 1391 if last_set != None:
1390 1392 self.dataOut.last_block = last_set * \
1391 1393 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1392 1394 return
1393 1395
1394 1396 def getBasicHeader(self):
1395 1397
1396 1398 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1397 1399 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1398 1400
1399 1401 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1400 1402
1401 1403 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1402 1404
1403 1405 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1404 1406
1405 1407 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1406 1408
1407 1409 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1408 1410
1409 1411 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1410 1412
1411 1413 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1412 1414
1413 1415 def getFirstHeader(self):
1414 1416
1415 1417 raise NotImplementedError
1416 1418
1417 1419 def getData(self):
1418 1420
1419 1421 raise NotImplementedError
1420 1422
1421 1423 def hasNotDataInBuffer(self):
1422 1424
1423 1425 raise NotImplementedError
1424 1426
1425 1427 def readBlock(self):
1426 1428
1427 1429 raise NotImplementedError
1428 1430
1429 1431 def isEndProcess(self):
1430 1432
1431 1433 return self.flagNoMoreFiles
1432 1434
1433 1435 def printReadBlocks(self):
1434 1436
1435 1437 print "[Reading] Number of read blocks per file %04d" % self.nReadBlocks
1436 1438
1437 1439 def printTotalBlocks(self):
1438 1440
1439 1441 print "[Reading] Number of read blocks %04d" % self.nTotalBlocks
1440 1442
1441 1443 def printNumberOfBlock(self):
1442 1444 'SPAM!'
1443 1445
1444 1446 # if self.flagIsNewBlock:
1445 1447 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1446 1448 # self.processingHeaderObj.dataBlocksPerFile,
1447 1449 # self.dataOut.datatime.ctime())
1448 1450
1449 1451 def printInfo(self):
1450 1452
1451 1453 if self.__printInfo == False:
1452 1454 return
1453 1455
1454 1456 self.basicHeaderObj.printInfo()
1455 1457 self.systemHeaderObj.printInfo()
1456 1458 self.radarControllerHeaderObj.printInfo()
1457 1459 self.processingHeaderObj.printInfo()
1458 1460
1459 1461 self.__printInfo = False
1460 1462
1461 1463 def run(self,
1462 1464 path=None,
1463 1465 startDate=None,
1464 1466 endDate=None,
1465 1467 startTime=datetime.time(0, 0, 0),
1466 1468 endTime=datetime.time(23, 59, 59),
1467 1469 set=None,
1468 1470 expLabel="",
1469 1471 ext=None,
1470 1472 online=False,
1471 1473 delay=60,
1472 1474 walk=True,
1473 1475 getblock=False,
1474 1476 nTxs=1,
1475 1477 realtime=False,
1476 1478 blocksize=None,
1477 1479 blocktime=None,
1478 1480 skip=None,
1479 1481 cursor=None,
1480 1482 warnings=True,
1481 1483 server=None,
1482 1484 verbose=True,
1483 1485 format=None,
1484 1486 oneDDict=None,
1485 1487 twoDDict=None,
1486 1488 ind2DList=None, **kwargs):
1487 1489
1488 1490 if not(self.isConfig):
1489 1491 self.setup(path=path,
1490 1492 startDate=startDate,
1491 1493 endDate=endDate,
1492 1494 startTime=startTime,
1493 1495 endTime=endTime,
1494 1496 set=set,
1495 1497 expLabel=expLabel,
1496 1498 ext=ext,
1497 1499 online=online,
1498 1500 delay=delay,
1499 1501 walk=walk,
1500 1502 getblock=getblock,
1501 1503 nTxs=nTxs,
1502 1504 realtime=realtime,
1503 1505 blocksize=blocksize,
1504 1506 blocktime=blocktime,
1505 1507 skip=skip,
1506 1508 cursor=cursor,
1507 1509 warnings=warnings,
1508 1510 server=server,
1509 1511 verbose=verbose,
1510 1512 format=format,
1511 1513 oneDDict=oneDDict,
1512 1514 twoDDict=twoDDict,
1513 1515 ind2DList=ind2DList)
1514 1516 self.isConfig = True
1515 1517 if server is None:
1516 1518 self.getData()
1517 1519 else:
1518 1520 self.getFromServer()
1519 1521
1520 1522
1521 1523 class JRODataWriter(JRODataIO):
1522 1524
1523 1525 """
1524 1526 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1525 1527 de los datos siempre se realiza por bloques.
1526 1528 """
1527 1529
1528 1530 blockIndex = 0
1529 1531
1530 1532 path = None
1531 1533
1532 1534 setFile = None
1533 1535
1534 1536 profilesPerBlock = None
1535 1537
1536 1538 blocksPerFile = None
1537 1539
1538 1540 nWriteBlocks = 0
1539 1541
1540 1542 fileDate = None
1541 1543
1542 1544 def __init__(self, dataOut=None):
1543 1545 raise NotImplementedError
1544 1546
1545 1547 def hasAllDataInBuffer(self):
1546 1548 raise NotImplementedError
1547 1549
1548 1550 def setBlockDimension(self):
1549 1551 raise NotImplementedError
1550 1552
1551 1553 def writeBlock(self):
1552 1554 raise NotImplementedError
1553 1555
1554 1556 def putData(self):
1555 1557 raise NotImplementedError
1556 1558
1557 1559 def getProcessFlags(self):
1558 1560
1559 1561 processFlags = 0
1560 1562
1561 1563 dtype_index = get_dtype_index(self.dtype)
1562 1564 procflag_dtype = get_procflag_dtype(dtype_index)
1563 1565
1564 1566 processFlags += procflag_dtype
1565 1567
1566 1568 if self.dataOut.flagDecodeData:
1567 1569 processFlags += PROCFLAG.DECODE_DATA
1568 1570
1569 1571 if self.dataOut.flagDeflipData:
1570 1572 processFlags += PROCFLAG.DEFLIP_DATA
1571 1573
1572 1574 if self.dataOut.code is not None:
1573 1575 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1574 1576
1575 1577 if self.dataOut.nCohInt > 1:
1576 1578 processFlags += PROCFLAG.COHERENT_INTEGRATION
1577 1579
1578 1580 if self.dataOut.type == "Spectra":
1579 1581 if self.dataOut.nIncohInt > 1:
1580 1582 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1581 1583
1582 1584 if self.dataOut.data_dc is not None:
1583 1585 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1584 1586
1585 1587 if self.dataOut.flagShiftFFT:
1586 1588 processFlags += PROCFLAG.SHIFT_FFT_DATA
1587 1589
1588 1590 return processFlags
1589 1591
1590 1592 def setBasicHeader(self):
1591 1593
1592 1594 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1593 1595 self.basicHeaderObj.version = self.versionFile
1594 1596 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1595 1597
1596 1598 utc = numpy.floor(self.dataOut.utctime)
1597 1599 milisecond = (self.dataOut.utctime - utc) * 1000.0
1598 1600
1599 1601 self.basicHeaderObj.utc = utc
1600 1602 self.basicHeaderObj.miliSecond = milisecond
1601 1603 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1602 1604 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1603 1605 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1604 1606
1605 1607 def setFirstHeader(self):
1606 1608 """
1607 1609 Obtiene una copia del First Header
1608 1610
1609 1611 Affected:
1610 1612
1611 1613 self.basicHeaderObj
1612 1614 self.systemHeaderObj
1613 1615 self.radarControllerHeaderObj
1614 1616 self.processingHeaderObj self.
1615 1617
1616 1618 Return:
1617 1619 None
1618 1620 """
1619 1621
1620 1622 raise NotImplementedError
1621 1623
1622 1624 def __writeFirstHeader(self):
1623 1625 """
1624 1626 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1625 1627
1626 1628 Affected:
1627 1629 __dataType
1628 1630
1629 1631 Return:
1630 1632 None
1631 1633 """
1632 1634
1633 1635 # CALCULAR PARAMETROS
1634 1636
1635 1637 sizeLongHeader = self.systemHeaderObj.size + \
1636 1638 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1637 1639 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1638 1640
1639 1641 self.basicHeaderObj.write(self.fp)
1640 1642 self.systemHeaderObj.write(self.fp)
1641 1643 self.radarControllerHeaderObj.write(self.fp)
1642 1644 self.processingHeaderObj.write(self.fp)
1643 1645
1644 1646 def __setNewBlock(self):
1645 1647 """
1646 1648 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1647 1649
1648 1650 Return:
1649 1651 0 : si no pudo escribir nada
1650 1652 1 : Si escribio el Basic el First Header
1651 1653 """
1652 1654 if self.fp == None:
1653 1655 self.setNextFile()
1654 1656
1655 1657 if self.flagIsNewFile:
1656 1658 return 1
1657 1659
1658 1660 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1659 1661 self.basicHeaderObj.write(self.fp)
1660 1662 return 1
1661 1663
1662 1664 if not(self.setNextFile()):
1663 1665 return 0
1664 1666
1665 1667 return 1
1666 1668
1667 1669 def writeNextBlock(self):
1668 1670 """
1669 1671 Selecciona el bloque siguiente de datos y los escribe en un file
1670 1672
1671 1673 Return:
1672 1674 0 : Si no hizo pudo escribir el bloque de datos
1673 1675 1 : Si no pudo escribir el bloque de datos
1674 1676 """
1675 1677 if not(self.__setNewBlock()):
1676 1678 return 0
1677 1679
1678 1680 self.writeBlock()
1679 1681
1680 1682 print "[Writing] Block No. %d/%d" % (self.blockIndex,
1681 1683 self.processingHeaderObj.dataBlocksPerFile)
1682 1684
1683 1685 return 1
1684 1686
1685 1687 def setNextFile(self):
1686 1688 """
1687 1689 Determina el siguiente file que sera escrito
1688 1690
1689 1691 Affected:
1690 1692 self.filename
1691 1693 self.subfolder
1692 1694 self.fp
1693 1695 self.setFile
1694 1696 self.flagIsNewFile
1695 1697
1696 1698 Return:
1697 1699 0 : Si el archivo no puede ser escrito
1698 1700 1 : Si el archivo esta listo para ser escrito
1699 1701 """
1700 1702 ext = self.ext
1701 1703 path = self.path
1702 1704
1703 1705 if self.fp != None:
1704 1706 self.fp.close()
1705 1707
1706 1708 timeTuple = time.localtime(self.dataOut.utctime)
1707 1709 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1708 1710
1709 1711 fullpath = os.path.join(path, subfolder)
1710 1712 setFile = self.setFile
1711 1713
1712 1714 if not(os.path.exists(fullpath)):
1713 1715 os.mkdir(fullpath)
1714 1716 setFile = -1 # inicializo mi contador de seteo
1715 1717 else:
1716 1718 filesList = os.listdir(fullpath)
1717 1719 if len(filesList) > 0:
1718 1720 filesList = sorted(filesList, key=str.lower)
1719 1721 filen = filesList[-1]
1720 1722 # el filename debera tener el siguiente formato
1721 1723 # 0 1234 567 89A BCDE (hex)
1722 1724 # x YYYY DDD SSS .ext
1723 1725 if isNumber(filen[8:11]):
1724 1726 # inicializo mi contador de seteo al seteo del ultimo file
1725 1727 setFile = int(filen[8:11])
1726 1728 else:
1727 1729 setFile = -1
1728 1730 else:
1729 1731 setFile = -1 # inicializo mi contador de seteo
1730 1732
1731 1733 setFile += 1
1732 1734
1733 1735 # If this is a new day it resets some values
1734 1736 if self.dataOut.datatime.date() > self.fileDate:
1735 1737 setFile = 0
1736 1738 self.nTotalBlocks = 0
1737 1739
1738 1740 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1739 1741 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1740 1742
1741 1743 filename = os.path.join(path, subfolder, filen)
1742 1744
1743 1745 fp = open(filename, 'wb')
1744 1746
1745 1747 self.blockIndex = 0
1746 1748
1747 1749 # guardando atributos
1748 1750 self.filename = filename
1749 1751 self.subfolder = subfolder
1750 1752 self.fp = fp
1751 1753 self.setFile = setFile
1752 1754 self.flagIsNewFile = 1
1753 1755 self.fileDate = self.dataOut.datatime.date()
1754 1756
1755 1757 self.setFirstHeader()
1756 1758
1757 1759 print '[Writing] Opening file: %s' % self.filename
1758 1760
1759 1761 self.__writeFirstHeader()
1760 1762
1761 1763 return 1
1762 1764
1763 1765 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1764 1766 """
1765 1767 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1766 1768
1767 1769 Inputs:
1768 1770 path : directory where data will be saved
1769 1771 profilesPerBlock : number of profiles per block
1770 1772 set : initial file set
1771 1773 datatype : An integer number that defines data type:
1772 1774 0 : int8 (1 byte)
1773 1775 1 : int16 (2 bytes)
1774 1776 2 : int32 (4 bytes)
1775 1777 3 : int64 (8 bytes)
1776 1778 4 : float32 (4 bytes)
1777 1779 5 : double64 (8 bytes)
1778 1780
1779 1781 Return:
1780 1782 0 : Si no realizo un buen seteo
1781 1783 1 : Si realizo un buen seteo
1782 1784 """
1783 1785
1784 1786 if ext == None:
1785 1787 ext = self.ext
1786 1788
1787 1789 self.ext = ext.lower()
1788 1790
1789 1791 self.path = path
1790 1792
1791 1793 if set is None:
1792 1794 self.setFile = -1
1793 1795 else:
1794 1796 self.setFile = set - 1
1795 1797
1796 1798 self.blocksPerFile = blocksPerFile
1797 1799
1798 1800 self.profilesPerBlock = profilesPerBlock
1799 1801
1800 1802 self.dataOut = dataOut
1801 1803 self.fileDate = self.dataOut.datatime.date()
1802 1804 # By default
1803 1805 self.dtype = self.dataOut.dtype
1804 1806
1805 1807 if datatype is not None:
1806 1808 self.dtype = get_numpy_dtype(datatype)
1807 1809
1808 1810 if not(self.setNextFile()):
1809 1811 print "[Writing] There isn't a next file"
1810 1812 return 0
1811 1813
1812 1814 self.setBlockDimension()
1813 1815
1814 1816 return 1
1815 1817
1816 1818 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1817 1819
1818 1820 if not(self.isConfig):
1819 1821
1820 1822 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1821 1823 set=set, ext=ext, datatype=datatype, **kwargs)
1822 1824 self.isConfig = True
1823 1825
1824 1826 self.putData()
General Comments 0
You need to be logged in to leave comments. Login now