##// END OF EJS Templates
schain
RSS
Filtering block by time
Miguel Valdez -
r759:7c9c0d91a0d5
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1646 +1,1698
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 print "The file %s can't be opened" %(filename)
72 72 return 0
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 def isTimeInRange(thisTime, startTime, endTime):
87
88 if endTime >= startTime:
89 if (thisTime < startTime) or (thisTime > endTime):
90 return 0
91
92 return 1
93 else:
94 if (thisTime < startTime) and (thisTime > endTime):
95 return 0
96
97 return 1
98
86 99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
87 100 """
88 101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89 102
90 103 Inputs:
91 104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92 105
93 106 startDate : fecha inicial del rango seleccionado en formato datetime.date
94 107
95 108 endDate : fecha final del rango seleccionado en formato datetime.date
96 109
97 110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
98 111
99 112 endTime : tiempo final del rango seleccionado en formato datetime.time
100 113
101 114 Return:
102 115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
103 116 fecha especificado, de lo contrario retorna False.
104 117
105 118 Excepciones:
106 119 Si el archivo no existe o no puede ser abierto
107 120 Si la cabecera no puede ser leida.
108 121
109 122 """
110 123
111 124
112 125 try:
113 126 fp = open(filename,'rb')
114 127 except IOError:
115 128 print "The file %s can't be opened" %(filename)
116 129 return None
117 130
118 basicHeaderObj = BasicHeader(LOCALTIME)
119 sts = basicHeaderObj.read(fp)
120 fp.close()
131 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 systemHeaderObj = SystemHeader()
133 radarControllerHeaderObj = RadarControllerHeader()
134 processingHeaderObj = ProcessingHeader()
121 135
122 thisDatetime = basicHeaderObj.datatime
123 thisDate = thisDatetime.date()
124 thisTime = thisDatetime.time()
136 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137
138 sts = firstBasicHeaderObj.read(fp)
139 sts = systemHeaderObj.read(fp)
140 sts = radarControllerHeaderObj.read(fp)
141 sts = processingHeaderObj.read(fp)
142
143 offset = processingHeaderObj.blockSize + 24 #header size
144
145 fp.seek(-offset, 2)
146
147 sts = lastBasicHeaderObj.read(fp)
148
149 fp.close()
125 150
126 151 if not(sts):
127 152 print "Skipping the file %s because it has not a valid header" %(filename)
128 153 return None
129 154
155 thisDatetime = firstBasicHeaderObj.datatime
156 thisDate = thisDatetime.date()
157 thisTime_first_block = thisDatetime.time()
158
159 thisDatetime = lastBasicHeaderObj.datatime
160 thisTime_last_block = thisDatetime.time()
161
162
130 163 #General case
131 164 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
132 165 #-----------o----------------------------o-----------
133 166 # startTime endTime
134 167
135 168 if endTime >= startTime:
136 if (thisTime < startTime) or (thisTime > endTime):
169 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
137 170 return None
138 171
139 172 return thisDatetime
140 173
141 174 #If endTime < startTime then endTime belongs to the next day
142 175
143 176
144 177 #<<<<<<<<<<<o o>>>>>>>>>>>
145 178 #-----------o----------------------------o-----------
146 179 # endTime startTime
147 180
148 if (thisDate == startDate) and (thisTime < startTime):
181 if (thisDate == startDate) and (thisTime_last_block < startTime):
149 182 return None
150 183
151 if (thisDate == endDate) and (thisTime > endTime):
184 if (thisDate == endDate) and (thisTime_first_block > endTime):
152 185 return None
153 186
154 if (thisTime < startTime) and (thisTime > endTime):
187 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
155 188 return None
156 189
157 190 return thisDatetime
158 191
159 192 def isFolderInDateRange(folder, startDate=None, endDate=None):
160 193 """
161 194 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
162 195
163 196 Inputs:
164 197 folder : nombre completo del directorio.
165 198 Su formato deberia ser "/path_root/?YYYYDDD"
166 199
167 200 siendo:
168 201 YYYY : Anio (ejemplo 2015)
169 202 DDD : Dia del anio (ejemplo 305)
170 203
171 204 startDate : fecha inicial del rango seleccionado en formato datetime.date
172 205
173 206 endDate : fecha final del rango seleccionado en formato datetime.date
174 207
175 208 Return:
176 209 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
177 210 fecha especificado, de lo contrario retorna False.
178 211 Excepciones:
179 212 Si el directorio no tiene el formato adecuado
180 213 """
181 214
182 215 basename = os.path.basename(folder)
183 216
184 217 if not isRadarFolder(basename):
185 218 print "The folder %s has not the rigth format" %folder
186 219 return 0
187 220
188 221 if startDate and endDate:
189 222 thisDate = getDateFromRadarFolder(basename)
190 223
191 224 if thisDate < startDate:
192 225 return 0
193 226
194 227 if thisDate > endDate:
195 228 return 0
196 229
197 230 return 1
198 231
199 232 def isFileInDateRange(filename, startDate=None, endDate=None):
200 233 """
201 234 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
202 235
203 236 Inputs:
204 237 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
205 238
206 239 Su formato deberia ser "?YYYYDDDsss"
207 240
208 241 siendo:
209 242 YYYY : Anio (ejemplo 2015)
210 243 DDD : Dia del anio (ejemplo 305)
211 244 sss : set
212 245
213 246 startDate : fecha inicial del rango seleccionado en formato datetime.date
214 247
215 248 endDate : fecha final del rango seleccionado en formato datetime.date
216 249
217 250 Return:
218 251 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
219 252 fecha especificado, de lo contrario retorna False.
220 253 Excepciones:
221 254 Si el archivo no tiene el formato adecuado
222 255 """
223 256
224 257 basename = os.path.basename(filename)
225 258
226 259 if not isRadarFile(basename):
227 260 print "The filename %s has not the rigth format" %filename
228 261 return 0
229 262
230 263 if startDate and endDate:
231 264 thisDate = getDateFromRadarFile(basename)
232 265
233 266 if thisDate < startDate:
234 267 return 0
235 268
236 269 if thisDate > endDate:
237 270 return 0
238 271
239 272 return 1
240 273
241 274 def getFileFromSet(path, ext, set):
242 275 validFilelist = []
243 276 fileList = os.listdir(path)
244 277
245 278 # 0 1234 567 89A BCDE
246 279 # H YYYY DDD SSS .ext
247 280
248 281 for thisFile in fileList:
249 282 try:
250 283 year = int(thisFile[1:5])
251 284 doy = int(thisFile[5:8])
252 285 except:
253 286 continue
254 287
255 288 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
256 289 continue
257 290
258 291 validFilelist.append(thisFile)
259 292
260 293 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
261 294
262 295 if len(myfile)!= 0:
263 296 return myfile[0]
264 297 else:
265 298 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
266 299 print 'the filename %s does not exist'%filename
267 300 print '...going to the last file: '
268 301
269 302 if validFilelist:
270 303 validFilelist = sorted( validFilelist, key=str.lower )
271 304 return validFilelist[-1]
272 305
273 306 return None
274 307
275 308 def getlastFileFromPath(path, ext):
276 309 """
277 310 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
278 311 al final de la depuracion devuelve el ultimo file de la lista que quedo.
279 312
280 313 Input:
281 314 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
282 315 ext : extension de los files contenidos en una carpeta
283 316
284 317 Return:
285 318 El ultimo file de una determinada carpeta, no se considera el path.
286 319 """
287 320 validFilelist = []
288 321 fileList = os.listdir(path)
289 322
290 323 # 0 1234 567 89A BCDE
291 324 # H YYYY DDD SSS .ext
292 325
293 326 for thisFile in fileList:
294 327
295 328 year = thisFile[1:5]
296 329 if not isNumber(year):
297 330 continue
298 331
299 332 doy = thisFile[5:8]
300 333 if not isNumber(doy):
301 334 continue
302 335
303 336 year = int(year)
304 337 doy = int(doy)
305 338
306 339 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
307 340 continue
308 341
309 342 validFilelist.append(thisFile)
310 343
311 344 if validFilelist:
312 345 validFilelist = sorted( validFilelist, key=str.lower )
313 346 return validFilelist[-1]
314 347
315 348 return None
316 349
317 350 def checkForRealPath(path, foldercounter, year, doy, set, ext):
318 351 """
319 352 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
320 353 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
321 354 el path exacto de un determinado file.
322 355
323 356 Example :
324 357 nombre correcto del file es .../.../D2009307/P2009307367.ext
325 358
326 359 Entonces la funcion prueba con las siguientes combinaciones
327 360 .../.../y2009307367.ext
328 361 .../.../Y2009307367.ext
329 362 .../.../x2009307/y2009307367.ext
330 363 .../.../x2009307/Y2009307367.ext
331 364 .../.../X2009307/y2009307367.ext
332 365 .../.../X2009307/Y2009307367.ext
333 366 siendo para este caso, la ultima combinacion de letras, identica al file buscado
334 367
335 368 Return:
336 369 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
337 370 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
338 371 para el filename
339 372 """
340 373 fullfilename = None
341 374 find_flag = False
342 375 filename = None
343 376
344 377 prefixDirList = [None,'d','D']
345 378 if ext.lower() == ".r": #voltage
346 379 prefixFileList = ['d','D']
347 380 elif ext.lower() == ".pdata": #spectra
348 381 prefixFileList = ['p','P']
349 382 else:
350 383 return None, filename
351 384
352 385 #barrido por las combinaciones posibles
353 386 for prefixDir in prefixDirList:
354 387 thispath = path
355 388 if prefixDir != None:
356 389 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
357 390 if foldercounter == 0:
358 391 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
359 392 else:
360 393 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
361 394 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
362 395 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
363 396 fullfilename = os.path.join( thispath, filename ) #formo el path completo
364 397
365 398 if os.path.exists( fullfilename ): #verifico que exista
366 399 find_flag = True
367 400 break
368 401 if find_flag:
369 402 break
370 403
371 404 if not(find_flag):
372 405 return None, filename
373 406
374 407 return fullfilename, filename
375 408
376 409 def isRadarFolder(folder):
377 410 try:
378 411 year = int(folder[1:5])
379 412 doy = int(folder[5:8])
380 413 except:
381 414 return 0
382 415
383 416 return 1
384 417
385 418 def isRadarFile(file):
386 419 try:
387 420 year = int(file[1:5])
388 421 doy = int(file[5:8])
389 422 set = int(file[8:11])
390 423 except:
391 424 return 0
392 425
393 426 return 1
394 427
395 428 def getDateFromRadarFile(file):
396 429 try:
397 430 year = int(file[1:5])
398 431 doy = int(file[5:8])
399 432 set = int(file[8:11])
400 433 except:
401 434 return None
402 435
403 436 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
404 437 return thisDate
405 438
406 439 def getDateFromRadarFolder(folder):
407 440 try:
408 441 year = int(folder[1:5])
409 442 doy = int(folder[5:8])
410 443 except:
411 444 return None
412 445
413 446 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
414 447 return thisDate
415 448
416 449 class JRODataIO:
417 450
418 451 c = 3E8
419 452
420 453 isConfig = False
421 454
422 455 basicHeaderObj = None
423 456
424 457 systemHeaderObj = None
425 458
426 459 radarControllerHeaderObj = None
427 460
428 461 processingHeaderObj = None
429 462
430 463 dtype = None
431 464
432 465 pathList = []
433 466
434 467 filenameList = []
435 468
436 469 filename = None
437 470
438 471 ext = None
439 472
440 473 flagIsNewFile = 1
441 474
442 475 flagDiscontinuousBlock = 0
443 476
444 477 flagIsNewBlock = 0
445 478
446 479 fp = None
447 480
448 481 firstHeaderSize = 0
449 482
450 483 basicHeaderSize = 24
451 484
452 485 versionFile = 1103
453 486
454 487 fileSize = None
455 488
456 489 # ippSeconds = None
457 490
458 491 fileSizeByHeader = None
459 492
460 493 fileIndex = None
461 494
462 495 profileIndex = None
463 496
464 497 blockIndex = None
465 498
466 499 nTotalBlocks = None
467 500
468 501 maxTimeStep = 30
469 502
470 503 lastUTTime = None
471 504
472 505 datablock = None
473 506
474 507 dataOut = None
475 508
476 509 blocksize = None
477 510
478 511 getByBlock = False
479 512
480 513 def __init__(self):
481 514
482 515 raise NotImplementedError
483 516
484 517 def run(self):
485 518
486 519 raise NotImplementedError
487 520
488 521 def getDtypeWidth(self):
489 522
490 523 dtype_index = get_dtype_index(self.dtype)
491 524 dtype_width = get_dtype_width(dtype_index)
492 525
493 526 return dtype_width
494 527
495 528 class JRODataReader(JRODataIO):
496 529
497 530
498 531 online = 0
499 532
500 533 realtime = 0
501 534
502 535 nReadBlocks = 0
503 536
504 537 delay = 10 #number of seconds waiting a new file
505 538
506 539 nTries = 3 #quantity tries
507 540
508 541 nFiles = 3 #number of files for searching
509 542
510 543 path = None
511 544
512 545 foldercounter = 0
513 546
514 547 flagNoMoreFiles = 0
515 548
516 549 datetimeList = []
517 550
518 551 __isFirstTimeOnline = 1
519 552
520 553 __printInfo = True
521 554
522 555 profileIndex = None
523 556
524 557 nTxs = 1
525 558
526 559 txIndex = None
527 560
528 561 def __init__(self):
529 562
530 563 """
531 564 This class is used to find data files
532 565
533 566 Example:
534 567 reader = JRODataReader()
535 568 fileList = reader.findDataFiles()
536 569
537 570 """
538 571 pass
539 572
540 573
541 574 def createObjByDefault(self):
542 575 """
543 576
544 577 """
545 578 raise NotImplementedError
546 579
547 580 def getBlockDimension(self):
548 581
549 582 raise NotImplementedError
550 583
551 584 def __searchFilesOffLine(self,
552 585 path,
553 586 startDate=None,
554 587 endDate=None,
555 588 startTime=datetime.time(0,0,0),
556 589 endTime=datetime.time(23,59,59),
557 590 set=None,
558 591 expLabel='',
559 592 ext='.r',
560 593 walk=True):
561 594
562 595 self.filenameList = []
563 596 self.datetimeList = []
564 597
565 598 pathList = []
566 599
567 600 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
568 601
569 602 if dateList == []:
570 # print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
571 # datetime.datetime.combine(startDate,startTime).ctime(),
572 # datetime.datetime.combine(endDate,endTime).ctime())
573
603 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
574 604 return None, None
575 605
576 606 if len(dateList) > 1:
577 print "[Reading] Data found: total days = %d, date range = %s - %s" %(len(dateList), startDate, endDate)
607 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
578 608 else:
579 print "[Reading] Data found: date = %s" %(dateList[0])
609 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
580 610
581 611 filenameList = []
582 612 datetimeList = []
583 613
584 614 for thisPath in pathList:
585 615 # thisPath = pathList[pathDict[file]]
586 616
587 617 fileList = glob.glob1(thisPath, "*%s" %ext)
588 618 fileList.sort()
589 619
590 620 for file in fileList:
591 621
592 622 filename = os.path.join(thisPath,file)
593 623
594 624 if not isFileInDateRange(filename, startDate, endDate):
595 625 continue
596 626
597 627 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
598 628
599 629 if not(thisDatetime):
600 630 continue
601 631
602 632 filenameList.append(filename)
603 633 datetimeList.append(thisDatetime)
604 634
605 635 if not(filenameList):
606 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
636 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
607 637 return None, None
608 638
609 639 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
610 640 print
611 641
612 642 for i in range(len(filenameList)):
613 643 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
614 644
615 645 self.filenameList = filenameList
616 646 self.datetimeList = datetimeList
617 647
618 648 return pathList, filenameList
619 649
620 650 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
621 651
622 652 """
623 653 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
624 654 devuelve el archivo encontrado ademas de otros datos.
625 655
626 656 Input:
627 657 path : carpeta donde estan contenidos los files que contiene data
628 658
629 659 expLabel : Nombre del subexperimento (subfolder)
630 660
631 661 ext : extension de los files
632 662
633 663 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
634 664
635 665 Return:
636 666 directory : eL directorio donde esta el file encontrado
637 667 filename : el ultimo file de una determinada carpeta
638 668 year : el anho
639 669 doy : el numero de dia del anho
640 670 set : el set del archivo
641 671
642 672
643 673 """
644 674 if not os.path.isdir(path):
645 675 return None, None, None, None, None, None
646 676
647 677 dirList = []
648 678
649 679 if not walk:
650 680 fullpath = path
651 681 foldercounter = 0
652 682 else:
653 683 #Filtra solo los directorios
654 684 for thisPath in os.listdir(path):
655 685 if not os.path.isdir(os.path.join(path,thisPath)):
656 686 continue
657 687 if not isRadarFolder(thisPath):
658 688 continue
659 689
660 690 dirList.append(thisPath)
661 691
662 692 if not(dirList):
663 693 return None, None, None, None, None, None
664 694
665 695 dirList = sorted( dirList, key=str.lower )
666 696
667 697 doypath = dirList[-1]
668 698 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
669 699 fullpath = os.path.join(path, doypath, expLabel)
670 700
671 701
672 702 print "[Reading] %s folder was found: " %(fullpath )
673 703
674 704 if set == None:
675 705 filename = getlastFileFromPath(fullpath, ext)
676 706 else:
677 707 filename = getFileFromSet(fullpath, ext, set)
678 708
679 709 if not(filename):
680 710 return None, None, None, None, None, None
681 711
682 712 print "[Reading] %s file was found" %(filename)
683 713
684 714 if not(self.__verifyFile(os.path.join(fullpath, filename))):
685 715 return None, None, None, None, None, None
686 716
687 717 year = int( filename[1:5] )
688 718 doy = int( filename[5:8] )
689 719 set = int( filename[8:11] )
690 720
691 721 return fullpath, foldercounter, filename, year, doy, set
692 722
693 723 def __setNextFileOffline(self):
694 724
695 725 idFile = self.fileIndex
696 726
697 727 while (True):
698 728 idFile += 1
699 729 if not(idFile < len(self.filenameList)):
700 730 self.flagNoMoreFiles = 1
701 731 # print "[Reading] No more Files"
702 732 return 0
703 733
704 734 filename = self.filenameList[idFile]
705 735
706 736 if not(self.__verifyFile(filename)):
707 737 continue
708 738
709 739 fileSize = os.path.getsize(filename)
710 740 fp = open(filename,'rb')
711 741 break
712 742
713 743 self.flagIsNewFile = 1
714 744 self.fileIndex = idFile
715 745 self.filename = filename
716 746 self.fileSize = fileSize
717 747 self.fp = fp
718 748
719 749 # print "[Reading] Setting the file: %s"%self.filename
720 750
721 751 return 1
722 752
723 753 def __setNextFileOnline(self):
724 754 """
725 755 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
726 756 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
727 757 siguientes.
728 758
729 759 Affected:
730 760 self.flagIsNewFile
731 761 self.filename
732 762 self.fileSize
733 763 self.fp
734 764 self.set
735 765 self.flagNoMoreFiles
736 766
737 767 Return:
738 768 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
739 769 1 : si el file fue abierto con exito y esta listo a ser leido
740 770
741 771 Excepciones:
742 772 Si un determinado file no puede ser abierto
743 773 """
744 774 nFiles = 0
745 775 fileOk_flag = False
746 776 firstTime_flag = True
747 777
748 778 self.set += 1
749 779
750 780 if self.set > 999:
751 781 self.set = 0
752 782 self.foldercounter += 1
753 783
754 784 #busca el 1er file disponible
755 785 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
756 786 if fullfilename:
757 787 if self.__verifyFile(fullfilename, False):
758 788 fileOk_flag = True
759 789
760 790 #si no encuentra un file entonces espera y vuelve a buscar
761 791 if not(fileOk_flag):
762 792 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
763 793
764 794 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
765 795 tries = self.nTries
766 796 else:
767 797 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
768 798
769 799 for nTries in range( tries ):
770 800 if firstTime_flag:
771 801 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
772 802 sleep( self.delay )
773 803 else:
774 804 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
775 805
776 806 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
777 807 if fullfilename:
778 808 if self.__verifyFile(fullfilename):
779 809 fileOk_flag = True
780 810 break
781 811
782 812 if fileOk_flag:
783 813 break
784 814
785 815 firstTime_flag = False
786 816
787 817 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
788 818 self.set += 1
789 819
790 820 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
791 821 self.set = 0
792 822 self.doy += 1
793 823 self.foldercounter = 0
794 824
795 825 if fileOk_flag:
796 826 self.fileSize = os.path.getsize( fullfilename )
797 827 self.filename = fullfilename
798 828 self.flagIsNewFile = 1
799 829 if self.fp != None: self.fp.close()
800 830 self.fp = open(fullfilename, 'rb')
801 831 self.flagNoMoreFiles = 0
802 832 # print '[Reading] Setting the file: %s' % fullfilename
803 833 else:
804 834 self.fileSize = 0
805 835 self.filename = None
806 836 self.flagIsNewFile = 0
807 837 self.fp = None
808 838 self.flagNoMoreFiles = 1
809 839 # print '[Reading] No more files to read'
810 840
811 841 return fileOk_flag
812 842
813 843 def setNextFile(self):
814 844 if self.fp != None:
815 845 self.fp.close()
816 846
817 847 if self.online:
818 848 newFile = self.__setNextFileOnline()
819 849 else:
820 850 newFile = self.__setNextFileOffline()
821 851
822 852 if not(newFile):
823 853 print '[Reading] No more files to read'
824 854 return 0
825 855
826 856 print '[Reading] Setting the file: %s' % self.filename
827 857
828 858 self.__readFirstHeader()
829 859 self.nReadBlocks = 0
830 860 return 1
831 861
832 862 def __waitNewBlock(self):
833 863 """
834 864 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
835 865
836 866 Si el modo de lectura es OffLine siempre retorn 0
837 867 """
838 868 if not self.online:
839 869 return 0
840 870
841 871 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
842 872 return 0
843 873
844 874 currentPointer = self.fp.tell()
845 875
846 876 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
847 877
848 878 for nTries in range( self.nTries ):
849 879
850 880 self.fp.close()
851 881 self.fp = open( self.filename, 'rb' )
852 882 self.fp.seek( currentPointer )
853 883
854 884 self.fileSize = os.path.getsize( self.filename )
855 885 currentSize = self.fileSize - currentPointer
856 886
857 887 if ( currentSize >= neededSize ):
858 888 self.basicHeaderObj.read(self.fp)
859 889 return 1
860 890
861 891 if self.fileSize == self.fileSizeByHeader:
862 892 # self.flagEoF = True
863 893 return 0
864 894
865 895 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
866 896 sleep( self.delay )
867 897
868 898
869 899 return 0
870 900
871 901 def waitDataBlock(self,pointer_location):
872 902
873 903 currentPointer = pointer_location
874 904
875 905 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
876 906
877 907 for nTries in range( self.nTries ):
878 908 self.fp.close()
879 909 self.fp = open( self.filename, 'rb' )
880 910 self.fp.seek( currentPointer )
881 911
882 912 self.fileSize = os.path.getsize( self.filename )
883 913 currentSize = self.fileSize - currentPointer
884 914
885 915 if ( currentSize >= neededSize ):
886 916 return 1
887 917
888 918 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
889 919 sleep( self.delay )
890 920
891 921 return 0
892 922
893 923 def __jumpToLastBlock(self):
894 924
895 925 if not(self.__isFirstTimeOnline):
896 926 return
897 927
898 928 csize = self.fileSize - self.fp.tell()
899 929 blocksize = self.processingHeaderObj.blockSize
900 930
901 931 #salta el primer bloque de datos
902 932 if csize > self.processingHeaderObj.blockSize:
903 933 self.fp.seek(self.fp.tell() + blocksize)
904 934 else:
905 935 return
906 936
907 937 csize = self.fileSize - self.fp.tell()
908 938 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
909 939 while True:
910 940
911 941 if self.fp.tell()<self.fileSize:
912 942 self.fp.seek(self.fp.tell() + neededsize)
913 943 else:
914 944 self.fp.seek(self.fp.tell() - neededsize)
915 945 break
916 946
917 947 # csize = self.fileSize - self.fp.tell()
918 948 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
919 949 # factor = int(csize/neededsize)
920 950 # if factor > 0:
921 951 # self.fp.seek(self.fp.tell() + factor*neededsize)
922 952
923 953 self.flagIsNewFile = 0
924 954 self.__isFirstTimeOnline = 0
925 955
926 956 def __setNewBlock(self):
927 957
928 958 if self.fp == None:
929 959 return 0
930 960
931 961 # if self.online:
932 962 # self.__jumpToLastBlock()
933 963
934 964 if self.flagIsNewFile:
935 965 self.lastUTTime = self.basicHeaderObj.utc
936 966 return 1
937 967
938 968 if self.realtime:
939 969 self.flagDiscontinuousBlock = 1
940 970 if not(self.setNextFile()):
941 971 return 0
942 972 else:
943 973 return 1
944 974
945 975 currentSize = self.fileSize - self.fp.tell()
946 976 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
947 977
948 978 if (currentSize >= neededSize):
949 979 self.basicHeaderObj.read(self.fp)
950 980 self.lastUTTime = self.basicHeaderObj.utc
951 981 return 1
952 982
953 983 if self.__waitNewBlock():
954 984 self.lastUTTime = self.basicHeaderObj.utc
955 985 return 1
956 986
957 987 if not(self.setNextFile()):
958 988 return 0
959 989
960 990 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
961 991 self.lastUTTime = self.basicHeaderObj.utc
962 992
963 993 self.flagDiscontinuousBlock = 0
964 994
965 995 if deltaTime > self.maxTimeStep:
966 996 self.flagDiscontinuousBlock = 1
967 997
968 998 return 1
969 999
970 1000 def readNextBlock(self):
971 1001
972 if not(self.__setNewBlock()):
973 return 0
974
975 if not(self.readBlock()):
976 return 0
1002 #Skip block out of startTime and endTime
1003 while True:
1004 if not(self.__setNewBlock()):
1005 return 0
1006
1007 if not(self.readBlock()):
1008 return 0
1009
1010 self.getBasicHeader()
977 1011
978 self.getBasicHeader()
1012 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1013
1014 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1015 self.processingHeaderObj.dataBlocksPerFile,
1016 self.dataOut.datatime.ctime())
1017 continue
1018
1019 break
979 1020
980 1021 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
981 1022 self.processingHeaderObj.dataBlocksPerFile,
982 1023 self.dataOut.datatime.ctime())
983 1024 return 1
984 1025
985 1026 def __readFirstHeader(self):
986 1027
987 1028 self.basicHeaderObj.read(self.fp)
988 1029 self.systemHeaderObj.read(self.fp)
989 1030 self.radarControllerHeaderObj.read(self.fp)
990 1031 self.processingHeaderObj.read(self.fp)
991 1032
992 1033 self.firstHeaderSize = self.basicHeaderObj.size
993 1034
994 1035 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
995 1036 if datatype == 0:
996 1037 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
997 1038 elif datatype == 1:
998 1039 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
999 1040 elif datatype == 2:
1000 1041 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1001 1042 elif datatype == 3:
1002 1043 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1003 1044 elif datatype == 4:
1004 1045 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1005 1046 elif datatype == 5:
1006 1047 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1007 1048 else:
1008 1049 raise ValueError, 'Data type was not defined'
1009 1050
1010 1051 self.dtype = datatype_str
1011 1052 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1012 1053 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1013 1054 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1014 1055 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1015 1056 self.getBlockDimension()
1016 1057
1017 1058 def __verifyFile(self, filename, msgFlag=True):
1018 1059
1019 1060 msg = None
1020 1061
1021 1062 try:
1022 1063 fp = open(filename, 'rb')
1023 1064 except IOError:
1024 1065
1025 1066 if msgFlag:
1026 1067 print "[Reading] File %s can't be opened" % (filename)
1027 1068
1028 1069 return False
1029 1070
1030 1071 currentPosition = fp.tell()
1031 1072 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1032 1073
1033 1074 if neededSize == 0:
1034 1075 basicHeaderObj = BasicHeader(LOCALTIME)
1035 1076 systemHeaderObj = SystemHeader()
1036 1077 radarControllerHeaderObj = RadarControllerHeader()
1037 1078 processingHeaderObj = ProcessingHeader()
1038 1079
1039 1080 if not( basicHeaderObj.read(fp) ):
1040 1081 fp.close()
1041 1082 return False
1042 1083
1043 1084 if not( systemHeaderObj.read(fp) ):
1044 1085 fp.close()
1045 1086 return False
1046 1087
1047 1088 if not( radarControllerHeaderObj.read(fp) ):
1048 1089 fp.close()
1049 1090 return False
1050 1091
1051 1092 if not( processingHeaderObj.read(fp) ):
1052 1093 fp.close()
1053 1094 return False
1054 1095
1055 1096 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1056 1097 else:
1057 1098 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1058 1099
1059 1100 fp.close()
1060 1101
1061 1102 fileSize = os.path.getsize(filename)
1062 1103 currentSize = fileSize - currentPosition
1063 1104
1064 1105 if currentSize < neededSize:
1065 1106 if msgFlag and (msg != None):
1066 1107 print msg
1067 1108 return False
1068 1109
1069 1110 return True
1070 1111
1071 1112 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1072 1113
1114 path_empty = True
1115
1073 1116 dateList = []
1074 1117 pathList = []
1075 1118
1076 1119 multi_path = path.split(',')
1077 1120
1078 1121 if not walk:
1079 1122
1080 1123 for single_path in multi_path:
1081 1124
1082 1125 if not os.path.isdir(single_path):
1083 1126 continue
1084 1127
1085 1128 fileList = glob.glob1(single_path, "*"+ext)
1086 1129
1087 1130 if not fileList:
1088 1131 continue
1089 1132
1133 path_empty = False
1134
1090 1135 fileList.sort()
1091 1136
1092 1137 for thisFile in fileList:
1093 1138
1094 1139 if not os.path.isfile(os.path.join(single_path, thisFile)):
1095 1140 continue
1096 1141
1097 1142 if not isRadarFile(thisFile):
1098 1143 continue
1099 1144
1100 1145 if not isFileInDateRange(thisFile, startDate, endDate):
1101 1146 continue
1102 1147
1103 1148 thisDate = getDateFromRadarFile(thisFile)
1104 1149
1105 1150 if thisDate in dateList:
1106 1151 continue
1107 1152
1108 1153 dateList.append(thisDate)
1109 1154 pathList.append(single_path)
1110 1155
1111 1156 else:
1112 1157 for single_path in multi_path:
1113 1158
1114 1159 if not os.path.isdir(single_path):
1115 1160 continue
1116 1161
1117 1162 dirList = []
1118 1163
1119 1164 for thisPath in os.listdir(single_path):
1120 1165
1121 1166 if not os.path.isdir(os.path.join(single_path,thisPath)):
1122 1167 continue
1123 1168
1124 1169 if not isRadarFolder(thisPath):
1125 1170 continue
1126 1171
1127 1172 if not isFolderInDateRange(thisPath, startDate, endDate):
1128 1173 continue
1129 1174
1130 1175 dirList.append(thisPath)
1131 1176
1132 1177 if not dirList:
1133 1178 continue
1134 1179
1135 1180 dirList.sort()
1136 1181
1137 1182 for thisDir in dirList:
1138 1183
1139 1184 datapath = os.path.join(single_path, thisDir, expLabel)
1140 1185 fileList = glob.glob1(datapath, "*"+ext)
1141 1186
1142 if len(fileList) < 1:
1187 if not fileList:
1143 1188 continue
1144 1189
1190 path_empty = False
1191
1145 1192 thisDate = getDateFromRadarFolder(thisDir)
1146 1193
1147 1194 pathList.append(datapath)
1148 1195 dateList.append(thisDate)
1149 1196
1150 1197 dateList.sort()
1151 1198
1152 1199 if walk:
1153 1200 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1154 1201 else:
1155 1202 pattern_path = multi_path[0]
1156 1203
1157 if not dateList:
1158 print "[Reading] No *%s files in %s from %s to %s" %(ext, pattern_path, startDate, endDate)
1159
1204 if path_empty:
1205 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1206 else:
1207 if not dateList:
1208 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1209
1160 1210 if include_path:
1161 1211 return dateList, pathList
1162 1212
1163 1213 return dateList
1164 1214
1165 1215 def setup(self,
1166 1216 path=None,
1167 1217 startDate=None,
1168 1218 endDate=None,
1169 1219 startTime=datetime.time(0,0,0),
1170 1220 endTime=datetime.time(23,59,59),
1171 1221 set=None,
1172 1222 expLabel = "",
1173 1223 ext = None,
1174 1224 online = False,
1175 1225 delay = 60,
1176 1226 walk = True,
1177 1227 getblock = False,
1178 1228 nTxs = 1,
1179 1229 realtime=False):
1180 1230
1181 1231 if path == None:
1182 1232 raise ValueError, "[Reading] The path is not valid"
1183 1233
1184 1234 if ext == None:
1185 1235 ext = self.ext
1186 1236
1187 1237 if online:
1188 1238 print "[Reading] Searching files in online mode..."
1189 1239
1190 1240 for nTries in range( self.nTries ):
1191 1241 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1192 1242
1193 1243 if fullpath:
1194 1244 break
1195 1245
1196 1246 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1197 1247 sleep( self.delay )
1198 1248
1199 1249 if not(fullpath):
1200 1250 print "[Reading] There 'isn't any valid file in %s" % path
1201 1251 return
1202 1252
1203 1253 self.year = year
1204 1254 self.doy = doy
1205 1255 self.set = set - 1
1206 1256 self.path = path
1207 1257 self.foldercounter = foldercounter
1208 1258 last_set = None
1209 1259
1210 1260 else:
1211 1261 print "[Reading] Searching files in offline mode ..."
1212 1262 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1213 1263 startTime=startTime, endTime=endTime,
1214 1264 set=set, expLabel=expLabel, ext=ext,
1215 1265 walk=walk)
1216 1266
1217 1267 if not(pathList):
1218 1268 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1219 1269 # datetime.datetime.combine(startDate,startTime).ctime(),
1220 1270 # datetime.datetime.combine(endDate,endTime).ctime())
1221 1271
1222 1272 # sys.exit(-1)
1223 1273
1224 1274 self.fileIndex = -1
1225 1275 self.pathList = []
1226 1276 self.filenameList = []
1227 1277 return
1228 1278
1229 1279 self.fileIndex = -1
1230 1280 self.pathList = pathList
1231 1281 self.filenameList = filenameList
1232 1282 file_name = os.path.basename(filenameList[-1])
1233 1283 basename, ext = os.path.splitext(file_name)
1234 1284 last_set = int(basename[-3:])
1235 1285
1236 1286 self.online = online
1237 1287 self.realtime = realtime
1238 1288 self.delay = delay
1239 1289 ext = ext.lower()
1240 1290 self.ext = ext
1241 1291 self.getByBlock = getblock
1242 1292 self.nTxs = nTxs
1293 self.startTime = startTime
1294 self.endTime = endTime
1243 1295
1244 1296 if not(self.setNextFile()):
1245 1297 if (startDate!=None) and (endDate!=None):
1246 1298 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1247 1299 elif startDate != None:
1248 1300 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1249 1301 else:
1250 1302 print "[Reading] No files"
1251 1303
1252 1304 self.fileIndex = -1
1253 1305 self.pathList = []
1254 1306 self.filenameList = []
1255 1307 return
1256 1308
1257 1309 # self.getBasicHeader()
1258 1310
1259 1311 if last_set != None:
1260 1312 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1261 1313 return
1262 1314
1263 1315 def getBasicHeader(self):
1264 1316
1265 1317 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1266 1318
1267 1319 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1268 1320
1269 1321 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1270 1322
1271 1323 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1272 1324
1273 1325 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1274 1326
1275 1327 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1276 1328
1277 1329 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1278 1330
1279 1331 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1280 1332
1281 1333
1282 1334 def getFirstHeader(self):
1283 1335
1284 1336 raise NotImplementedError
1285 1337
1286 1338 def getData(self):
1287 1339
1288 1340 raise NotImplementedError
1289 1341
1290 1342 def hasNotDataInBuffer(self):
1291 1343
1292 1344 raise NotImplementedError
1293 1345
1294 1346 def readBlock(self):
1295 1347
1296 1348 raise NotImplementedError
1297 1349
1298 1350 def isEndProcess(self):
1299 1351
1300 1352 return self.flagNoMoreFiles
1301 1353
1302 1354 def printReadBlocks(self):
1303 1355
1304 1356 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1305 1357
1306 1358 def printTotalBlocks(self):
1307 1359
1308 1360 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1309 1361
1310 1362 def printNumberOfBlock(self):
1311 1363
1312 1364 if self.flagIsNewBlock:
1313 1365 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1314 1366 self.processingHeaderObj.dataBlocksPerFile,
1315 1367 self.dataOut.datatime.ctime())
1316 1368
1317 1369 def printInfo(self):
1318 1370
1319 1371 if self.__printInfo == False:
1320 1372 return
1321 1373
1322 1374 self.basicHeaderObj.printInfo()
1323 1375 self.systemHeaderObj.printInfo()
1324 1376 self.radarControllerHeaderObj.printInfo()
1325 1377 self.processingHeaderObj.printInfo()
1326 1378
1327 1379 self.__printInfo = False
1328 1380
1329 1381
1330 1382 def run(self, **kwargs):
1331 1383
1332 1384 if not(self.isConfig):
1333 1385
1334 1386 # self.dataOut = dataOut
1335 1387 self.setup(**kwargs)
1336 1388 self.isConfig = True
1337 1389
1338 1390 self.getData()
1339 1391
1340 1392 class JRODataWriter(JRODataIO):
1341 1393
1342 1394 """
1343 1395 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1344 1396 de los datos siempre se realiza por bloques.
1345 1397 """
1346 1398
1347 1399 blockIndex = 0
1348 1400
1349 1401 path = None
1350 1402
1351 1403 setFile = None
1352 1404
1353 1405 profilesPerBlock = None
1354 1406
1355 1407 blocksPerFile = None
1356 1408
1357 1409 nWriteBlocks = 0
1358 1410
1359 1411 fileDate = None
1360 1412
1361 1413 def __init__(self, dataOut=None):
1362 1414 raise NotImplementedError
1363 1415
1364 1416
1365 1417 def hasAllDataInBuffer(self):
1366 1418 raise NotImplementedError
1367 1419
1368 1420
1369 1421 def setBlockDimension(self):
1370 1422 raise NotImplementedError
1371 1423
1372 1424
1373 1425 def writeBlock(self):
1374 1426 raise NotImplementedError
1375 1427
1376 1428
1377 1429 def putData(self):
1378 1430 raise NotImplementedError
1379 1431
1380 1432
1381 1433 def getProcessFlags(self):
1382 1434
1383 1435 processFlags = 0
1384 1436
1385 1437 dtype_index = get_dtype_index(self.dtype)
1386 1438 procflag_dtype = get_procflag_dtype(dtype_index)
1387 1439
1388 1440 processFlags += procflag_dtype
1389 1441
1390 1442 if self.dataOut.flagDecodeData:
1391 1443 processFlags += PROCFLAG.DECODE_DATA
1392 1444
1393 1445 if self.dataOut.flagDeflipData:
1394 1446 processFlags += PROCFLAG.DEFLIP_DATA
1395 1447
1396 1448 if self.dataOut.code is not None:
1397 1449 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1398 1450
1399 1451 if self.dataOut.nCohInt > 1:
1400 1452 processFlags += PROCFLAG.COHERENT_INTEGRATION
1401 1453
1402 1454 if self.dataOut.type == "Spectra":
1403 1455 if self.dataOut.nIncohInt > 1:
1404 1456 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1405 1457
1406 1458 if self.dataOut.data_dc is not None:
1407 1459 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1408 1460
1409 1461 if self.dataOut.flagShiftFFT:
1410 1462 processFlags += PROCFLAG.SHIFT_FFT_DATA
1411 1463
1412 1464 return processFlags
1413 1465
1414 1466 def setBasicHeader(self):
1415 1467
1416 1468 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1417 1469 self.basicHeaderObj.version = self.versionFile
1418 1470 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1419 1471
1420 1472 utc = numpy.floor(self.dataOut.utctime)
1421 1473 milisecond = (self.dataOut.utctime - utc)* 1000.0
1422 1474
1423 1475 self.basicHeaderObj.utc = utc
1424 1476 self.basicHeaderObj.miliSecond = milisecond
1425 1477 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1426 1478 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1427 1479 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1428 1480
1429 1481 def setFirstHeader(self):
1430 1482 """
1431 1483 Obtiene una copia del First Header
1432 1484
1433 1485 Affected:
1434 1486
1435 1487 self.basicHeaderObj
1436 1488 self.systemHeaderObj
1437 1489 self.radarControllerHeaderObj
1438 1490 self.processingHeaderObj self.
1439 1491
1440 1492 Return:
1441 1493 None
1442 1494 """
1443 1495
1444 1496 raise NotImplementedError
1445 1497
1446 1498 def __writeFirstHeader(self):
1447 1499 """
1448 1500 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1449 1501
1450 1502 Affected:
1451 1503 __dataType
1452 1504
1453 1505 Return:
1454 1506 None
1455 1507 """
1456 1508
1457 1509 # CALCULAR PARAMETROS
1458 1510
1459 1511 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1460 1512 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1461 1513
1462 1514 self.basicHeaderObj.write(self.fp)
1463 1515 self.systemHeaderObj.write(self.fp)
1464 1516 self.radarControllerHeaderObj.write(self.fp)
1465 1517 self.processingHeaderObj.write(self.fp)
1466 1518
1467 1519 def __setNewBlock(self):
1468 1520 """
1469 1521 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1470 1522
1471 1523 Return:
1472 1524 0 : si no pudo escribir nada
1473 1525 1 : Si escribio el Basic el First Header
1474 1526 """
1475 1527 if self.fp == None:
1476 1528 self.setNextFile()
1477 1529
1478 1530 if self.flagIsNewFile:
1479 1531 return 1
1480 1532
1481 1533 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1482 1534 self.basicHeaderObj.write(self.fp)
1483 1535 return 1
1484 1536
1485 1537 if not( self.setNextFile() ):
1486 1538 return 0
1487 1539
1488 1540 return 1
1489 1541
1490 1542
1491 1543 def writeNextBlock(self):
1492 1544 """
1493 1545 Selecciona el bloque siguiente de datos y los escribe en un file
1494 1546
1495 1547 Return:
1496 1548 0 : Si no hizo pudo escribir el bloque de datos
1497 1549 1 : Si no pudo escribir el bloque de datos
1498 1550 """
1499 1551 if not( self.__setNewBlock() ):
1500 1552 return 0
1501 1553
1502 1554 self.writeBlock()
1503 1555
1504 1556 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1505 1557 self.processingHeaderObj.dataBlocksPerFile)
1506 1558
1507 1559 return 1
1508 1560
1509 1561 def setNextFile(self):
1510 1562 """
1511 1563 Determina el siguiente file que sera escrito
1512 1564
1513 1565 Affected:
1514 1566 self.filename
1515 1567 self.subfolder
1516 1568 self.fp
1517 1569 self.setFile
1518 1570 self.flagIsNewFile
1519 1571
1520 1572 Return:
1521 1573 0 : Si el archivo no puede ser escrito
1522 1574 1 : Si el archivo esta listo para ser escrito
1523 1575 """
1524 1576 ext = self.ext
1525 1577 path = self.path
1526 1578
1527 1579 if self.fp != None:
1528 1580 self.fp.close()
1529 1581
1530 1582 timeTuple = time.localtime( self.dataOut.utctime)
1531 1583 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1532 1584
1533 1585 fullpath = os.path.join( path, subfolder )
1534 1586 setFile = self.setFile
1535 1587
1536 1588 if not( os.path.exists(fullpath) ):
1537 1589 os.mkdir(fullpath)
1538 1590 setFile = -1 #inicializo mi contador de seteo
1539 1591 else:
1540 1592 filesList = os.listdir( fullpath )
1541 1593 if len( filesList ) > 0:
1542 1594 filesList = sorted( filesList, key=str.lower )
1543 1595 filen = filesList[-1]
1544 1596 # el filename debera tener el siguiente formato
1545 1597 # 0 1234 567 89A BCDE (hex)
1546 1598 # x YYYY DDD SSS .ext
1547 1599 if isNumber( filen[8:11] ):
1548 1600 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1549 1601 else:
1550 1602 setFile = -1
1551 1603 else:
1552 1604 setFile = -1 #inicializo mi contador de seteo
1553 1605
1554 1606 setFile += 1
1555 1607
1556 1608 #If this is a new day it resets some values
1557 1609 if self.dataOut.datatime.date() > self.fileDate:
1558 1610 setFile = 0
1559 1611 self.nTotalBlocks = 0
1560 1612
1561 1613 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1562 1614
1563 1615 filename = os.path.join( path, subfolder, filen )
1564 1616
1565 1617 fp = open( filename,'wb' )
1566 1618
1567 1619 self.blockIndex = 0
1568 1620
1569 1621 #guardando atributos
1570 1622 self.filename = filename
1571 1623 self.subfolder = subfolder
1572 1624 self.fp = fp
1573 1625 self.setFile = setFile
1574 1626 self.flagIsNewFile = 1
1575 1627 self.fileDate = self.dataOut.datatime.date()
1576 1628
1577 1629 self.setFirstHeader()
1578 1630
1579 1631 print '[Writing] Opening file: %s'%self.filename
1580 1632
1581 1633 self.__writeFirstHeader()
1582 1634
1583 1635 return 1
1584 1636
1585 1637 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1586 1638 """
1587 1639 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1588 1640
1589 1641 Inputs:
1590 1642 path : directory where data will be saved
1591 1643 profilesPerBlock : number of profiles per block
1592 1644 set : initial file set
1593 1645 datatype : An integer number that defines data type:
1594 1646 0 : int8 (1 byte)
1595 1647 1 : int16 (2 bytes)
1596 1648 2 : int32 (4 bytes)
1597 1649 3 : int64 (8 bytes)
1598 1650 4 : float32 (4 bytes)
1599 1651 5 : double64 (8 bytes)
1600 1652
1601 1653 Return:
1602 1654 0 : Si no realizo un buen seteo
1603 1655 1 : Si realizo un buen seteo
1604 1656 """
1605 1657
1606 1658 if ext == None:
1607 1659 ext = self.ext
1608 1660
1609 1661 self.ext = ext.lower()
1610 1662
1611 1663 self.path = path
1612 1664
1613 1665 if set is None:
1614 1666 self.setFile = -1
1615 1667 else:
1616 1668 self.setFile = set - 1
1617 1669
1618 1670 self.blocksPerFile = blocksPerFile
1619 1671
1620 1672 self.profilesPerBlock = profilesPerBlock
1621 1673
1622 1674 self.dataOut = dataOut
1623 1675 self.fileDate = self.dataOut.datatime.date()
1624 1676 #By default
1625 1677 self.dtype = self.dataOut.dtype
1626 1678
1627 1679 if datatype is not None:
1628 1680 self.dtype = get_numpy_dtype(datatype)
1629 1681
1630 1682 if not(self.setNextFile()):
1631 1683 print "[Writing] There isn't a next file"
1632 1684 return 0
1633 1685
1634 1686 self.setBlockDimension()
1635 1687
1636 1688 return 1
1637 1689
1638 1690 def run(self, dataOut, **kwargs):
1639 1691
1640 1692 if not(self.isConfig):
1641 1693
1642 1694 self.setup(dataOut, **kwargs)
1643 1695 self.isConfig = True
1644 1696
1645 1697 self.putData()
1646 1698
General Comments 0
You need to be logged in to leave comments. Login now