##// END OF EJS Templates
schain-jc
RSS
isFileinTimeRange: startTime can be greater than endTime
Miguel Valdez -
r648:aa41cfeddebd
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1629 +1,1585
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 traceback.print_exc()
72 72 raise IOError, "The file %s can't be opened" %(filename)
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 86 def isFileInTimeRange(filename, startTime, endTime):
87 87 """
88 88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89 89
90 90 Inputs:
91 91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92 92
93 93 startTime : tiempo inicial del rango seleccionado en formato datetime.time
94 94
95 95 endTime : tiempo final del rango seleccionado en formato datetime.time
96 96
97 97 Return:
98 98 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
99 99 fecha especificado, de lo contrario retorna False.
100 100
101 101 Excepciones:
102 102 Si el archivo no existe o no puede ser abierto
103 103 Si la cabecera no puede ser leida.
104 104
105 105 """
106 106
107 107
108 108 try:
109 109 fp = open(filename,'rb')
110 110 except IOError:
111 111 traceback.print_exc()
112 112 raise IOError, "The file %s can't be opened" %(filename)
113 113
114 114 basicHeaderObj = BasicHeader(LOCALTIME)
115 115 sts = basicHeaderObj.read(fp)
116 116 fp.close()
117 117
118 118 thisDatetime = basicHeaderObj.datatime
119 119 thisTime = thisDatetime.time()
120 120
121 121 if not(sts):
122 122 print "Skipping the file %s because it has not a valid header" %(filename)
123 123 return None
124 124
125 if not ((startTime <= thisTime) and (endTime > thisTime)):
125 #General case
126 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
127 #-----------o----------------------------o-----------
128 # startTime endTime
129
130 if endTime >= startTime:
131 if (thisTime < startTime) or (thisTime > endTime):
132 return None
133
134 return thisDatetime
135
136 #If endTime < startTime then endTime belongs to the next day
137
138
139 #<<<<<<<<<<<o o>>>>>>>>>>>
140 #-----------o----------------------------o-----------
141 # endTime startTime
142
143 if (thisTime < startTime) and (thisTime > endTime):
126 144 return None
127 145
128 146 return thisDatetime
129 147
130 148 def isFolderInDateRange(folder, startDate=None, endDate=None):
131 149 """
132 150 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
133 151
134 152 Inputs:
135 153 folder : nombre completo del directorio.
136 154 Su formato deberia ser "/path_root/?YYYYDDD"
137 155
138 156 siendo:
139 157 YYYY : Anio (ejemplo 2015)
140 158 DDD : Dia del anio (ejemplo 305)
141 159
142 160 startDate : fecha inicial del rango seleccionado en formato datetime.date
143 161
144 162 endDate : fecha final del rango seleccionado en formato datetime.date
145 163
146 164 Return:
147 165 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
148 166 fecha especificado, de lo contrario retorna False.
149 167 Excepciones:
150 168 Si el directorio no tiene el formato adecuado
151 169 """
152 170
153 171 basename = os.path.basename(folder)
154 172
155 173 if not isRadarFolder(basename):
156 174 raise IOError, "The folder %s has not the rigth format" %folder
157 175
158 176 if startDate and endDate:
159 177 thisDate = getDateFromRadarFolder(basename)
160 178
161 179 if thisDate < startDate:
162 180 return 0
163 181
164 182 if thisDate > endDate:
165 183 return 0
166 184
167 185 return 1
168 186
169 187 def isFileInDateRange(filename, startDate=None, endDate=None):
170 188 """
171 189 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
172 190
173 191 Inputs:
174 192 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
175 193
176 194 Su formato deberia ser "?YYYYDDDsss"
177 195
178 196 siendo:
179 197 YYYY : Anio (ejemplo 2015)
180 198 DDD : Dia del anio (ejemplo 305)
181 199 sss : set
182 200
183 201 startDate : fecha inicial del rango seleccionado en formato datetime.date
184 202
185 203 endDate : fecha final del rango seleccionado en formato datetime.date
186 204
187 205 Return:
188 206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
189 207 fecha especificado, de lo contrario retorna False.
190 208 Excepciones:
191 209 Si el archivo no tiene el formato adecuado
192 210 """
193 211
194 212 basename = os.path.basename(filename)
195 213
196 214 if not isRadarFile(basename):
197 215 raise IOError, "The filename %s has not the rigth format" %filename
198 216
199 217 if startDate and endDate:
200 218 thisDate = getDateFromRadarFile(basename)
201 219
202 220 if thisDate < startDate:
203 221 return 0
204 222
205 223 if thisDate > endDate:
206 224 return 0
207 225
208 226 return 1
209 227
210 228 def getFileFromSet(path, ext, set):
211 229 validFilelist = []
212 230 fileList = os.listdir(path)
213 231
214 232 # 0 1234 567 89A BCDE
215 233 # H YYYY DDD SSS .ext
216 234
217 235 for thisFile in fileList:
218 236 try:
219 237 year = int(thisFile[1:5])
220 238 doy = int(thisFile[5:8])
221 239 except:
222 240 continue
223 241
224 242 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
225 243 continue
226 244
227 245 validFilelist.append(thisFile)
228 246
229 247 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
230 248
231 249 if len(myfile)!= 0:
232 250 return myfile[0]
233 251 else:
234 252 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
235 253 print 'the filename %s does not exist'%filename
236 254 print '...going to the last file: '
237 255
238 256 if validFilelist:
239 257 validFilelist = sorted( validFilelist, key=str.lower )
240 258 return validFilelist[-1]
241 259
242 260 return None
243 261
244 262 def getlastFileFromPath(path, ext):
245 263 """
246 264 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
247 265 al final de la depuracion devuelve el ultimo file de la lista que quedo.
248 266
249 267 Input:
250 268 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
251 269 ext : extension de los files contenidos en una carpeta
252 270
253 271 Return:
254 272 El ultimo file de una determinada carpeta, no se considera el path.
255 273 """
256 274 validFilelist = []
257 275 fileList = os.listdir(path)
258 276
259 277 # 0 1234 567 89A BCDE
260 278 # H YYYY DDD SSS .ext
261 279
262 280 for thisFile in fileList:
263 281
264 282 year = thisFile[1:5]
265 283 if not isNumber(year):
266 284 continue
267 285
268 286 doy = thisFile[5:8]
269 287 if not isNumber(doy):
270 288 continue
271 289
272 290 year = int(year)
273 291 doy = int(doy)
274 292
275 293 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
276 294 continue
277 295
278 296 validFilelist.append(thisFile)
279 297
280 298 if validFilelist:
281 299 validFilelist = sorted( validFilelist, key=str.lower )
282 300 return validFilelist[-1]
283 301
284 302 return None
285 303
286 304 def checkForRealPath(path, foldercounter, year, doy, set, ext):
287 305 """
288 306 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
289 307 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
290 308 el path exacto de un determinado file.
291 309
292 310 Example :
293 311 nombre correcto del file es .../.../D2009307/P2009307367.ext
294 312
295 313 Entonces la funcion prueba con las siguientes combinaciones
296 314 .../.../y2009307367.ext
297 315 .../.../Y2009307367.ext
298 316 .../.../x2009307/y2009307367.ext
299 317 .../.../x2009307/Y2009307367.ext
300 318 .../.../X2009307/y2009307367.ext
301 319 .../.../X2009307/Y2009307367.ext
302 320 siendo para este caso, la ultima combinacion de letras, identica al file buscado
303 321
304 322 Return:
305 323 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
306 324 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
307 325 para el filename
308 326 """
309 327 fullfilename = None
310 328 find_flag = False
311 329 filename = None
312 330
313 331 prefixDirList = [None,'d','D']
314 332 if ext.lower() == ".r": #voltage
315 333 prefixFileList = ['d','D']
316 334 elif ext.lower() == ".pdata": #spectra
317 335 prefixFileList = ['p','P']
318 336 else:
319 337 return None, filename
320 338
321 339 #barrido por las combinaciones posibles
322 340 for prefixDir in prefixDirList:
323 341 thispath = path
324 342 if prefixDir != None:
325 343 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
326 344 if foldercounter == 0:
327 345 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
328 346 else:
329 347 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
330 348 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
331 349 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
332 350 fullfilename = os.path.join( thispath, filename ) #formo el path completo
333 351
334 352 if os.path.exists( fullfilename ): #verifico que exista
335 353 find_flag = True
336 354 break
337 355 if find_flag:
338 356 break
339 357
340 358 if not(find_flag):
341 359 return None, filename
342 360
343 361 return fullfilename, filename
344 362
345 363 def isRadarFolder(folder):
346 364 try:
347 365 year = int(folder[1:5])
348 366 doy = int(folder[5:8])
349 367 except:
350 368 return 0
351 369
352 370 return 1
353 371
354 372 def isRadarFile(file):
355 373 try:
356 374 year = int(file[1:5])
357 375 doy = int(file[5:8])
358 376 set = int(file[8:11])
359 377 except:
360 378 return 0
361 379
362 380 return 1
363 381
364 382 def getDateFromRadarFile(file):
365 383 try:
366 384 year = int(file[1:5])
367 385 doy = int(file[5:8])
368 386 set = int(file[8:11])
369 387 except:
370 388 return None
371 389
372 390 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
373 391 return thisDate
374 392
375 393 def getDateFromRadarFolder(folder):
376 394 try:
377 395 year = int(folder[1:5])
378 396 doy = int(folder[5:8])
379 397 except:
380 398 return None
381 399
382 400 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
383 401 return thisDate
384 402
385 403 class JRODataIO:
386 404
387 405 c = 3E8
388 406
389 407 isConfig = False
390 408
391 409 basicHeaderObj = None
392 410
393 411 systemHeaderObj = None
394 412
395 413 radarControllerHeaderObj = None
396 414
397 415 processingHeaderObj = None
398 416
399 417 online = 0
400 418
401 419 dtype = None
402 420
403 421 pathList = []
404 422
405 423 filenameList = []
406 424
407 425 filename = None
408 426
409 427 ext = None
410 428
411 429 flagIsNewFile = 1
412 430
413 431 flagDiscontinuousBlock = 0
414 432
415 433 flagIsNewBlock = 0
416 434
417 435 fp = None
418 436
419 437 firstHeaderSize = 0
420 438
421 439 basicHeaderSize = 24
422 440
423 441 versionFile = 1103
424 442
425 443 fileSize = None
426 444
427 445 # ippSeconds = None
428 446
429 447 fileSizeByHeader = None
430 448
431 449 fileIndex = None
432 450
433 451 profileIndex = None
434 452
435 453 blockIndex = None
436 454
437 455 nTotalBlocks = None
438 456
439 457 maxTimeStep = 30
440 458
441 459 lastUTTime = None
442 460
443 461 datablock = None
444 462
445 463 dataOut = None
446 464
447 465 blocksize = None
448 466
449 467 getByBlock = False
450 468
451 469 def __init__(self):
452 470
453 471 raise ValueError, "Not implemented"
454 472
455 473 def run(self):
456 474
457 475 raise ValueError, "Not implemented"
458 476
459 477 def getDtypeWidth(self):
460 478
461 479 dtype_index = get_dtype_index(self.dtype)
462 480 dtype_width = get_dtype_width(dtype_index)
463 481
464 482 return dtype_width
465 483
466 484 class JRODataReader(JRODataIO):
467 485
468 486 nReadBlocks = 0
469 487
470 488 delay = 10 #number of seconds waiting a new file
471 489
472 490 nTries = 3 #quantity tries
473 491
474 492 nFiles = 3 #number of files for searching
475 493
476 494 path = None
477 495
478 496 foldercounter = 0
479 497
480 498 flagNoMoreFiles = 0
481 499
482 500 datetimeList = []
483 501
484 502 __isFirstTimeOnline = 1
485 503
486 504 __printInfo = True
487 505
488 506 profileIndex = None
489 507
490 508 nTxs = 1
491 509
492 510 txIndex = None
493 511
494 512 def __init__(self):
495 513
496 514 """
497 515
498 516 """
499 517
500 518 # raise NotImplementedError, "This method has not been implemented"
501 519
502 520
503 521 def createObjByDefault(self):
504 522 """
505 523
506 524 """
507 525 raise NotImplementedError, "This method has not been implemented"
508 526
509 527 def getBlockDimension(self):
510 528
511 529 raise NotImplementedError, "No implemented"
512 530
513 531 def __searchFilesOffLine(self,
514 532 path,
515 533 startDate=None,
516 534 endDate=None,
517 535 startTime=datetime.time(0,0,0),
518 536 endTime=datetime.time(23,59,59),
519 537 set=None,
520 538 expLabel='',
521 539 ext='.r',
522 540 walk=True):
523 541
524 542 self.filenameList = []
525 543 self.datetimeList = []
526 544
527 545 pathList = []
528 546
529 # if not walk:
530 # #pathList.append(path)
531 # multi_path = path.split(',')
532 # for single_path in multi_path:
533 #
534 # if not os.path.isdir(single_path):
535 # continue
536 #
537 # pathList.append(single_path)
538 #
539 # else:
540 # #dirList = []
541 # multi_path = path.split(',')
542 # for single_path in multi_path:
543 #
544 # if not os.path.isdir(single_path):
545 # continue
546 #
547 # dirList = []
548 # for thisPath in os.listdir(single_path):
549 # if not os.path.isdir(os.path.join(single_path,thisPath)):
550 # continue
551 # if not isRadarFolder(thisPath):
552 # continue
553 #
554 # dirList.append(thisPath)
555 #
556 # if not(dirList):
557 # return None, None
558 #
559 # if startDate and endDate:
560 # thisDate = startDate
561 #
562 # while(thisDate <= endDate):
563 # year = thisDate.timetuple().tm_year
564 # doy = thisDate.timetuple().tm_yday
565 #
566 # matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
567 # if len(matchlist) == 0:
568 # thisDate += datetime.timedelta(1)
569 # continue
570 # for match in matchlist:
571 # pathList.append(os.path.join(single_path,match,expLabel))
572 #
573 # thisDate += datetime.timedelta(1)
574 # else:
575 # for thiDir in dirList:
576 # pathList.append(os.path.join(single_path,thiDir,expLabel))
577
578 547 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
579 548
580 549 if dateList == []:
581 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
550 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
551 datetime.datetime.combine(startDate,startTime).ctime(),
552 datetime.datetime.combine(endDate,endTime).ctime())
553
582 554 return None, None
583 555
584 556 if len(dateList) > 1:
585 print "%d dates with data were found for the date range: %s - %s" %(len(dateList), startDate, endDate)
557 print "[Reading] %d dates with data were found for the date range: %s - %s" %(len(dateList), startDate, endDate)
586 558 else:
587 print "data was found for the date %s" %(dateList[0])
559 print "[Reading] data was found for the date %s" %(dateList[0])
588 560
589 561 filenameList = []
590 562 datetimeList = []
591 # pathDict = {}
592 # filenameList_to_sort = []
593 #
594 # for i in range(len(pathList)):
595 #
596 # thisPath = pathList[i]
597 #
598 # fileList = glob.glob1(thisPath, "*%s" %ext)
599 # if len(fileList) < 1:
600 # continue
601 # fileList.sort()
602 # pathDict.setdefault(fileList[0])
603 # pathDict[fileList[0]] = i
604 # filenameList_to_sort.append(fileList[0])
605 #
606 # filenameList_to_sort.sort()
607 563
608 564 for thisPath in pathList:
609 565 # thisPath = pathList[pathDict[file]]
610 566
611 567 fileList = glob.glob1(thisPath, "*%s" %ext)
612 568 fileList.sort()
613 569
614 570 for file in fileList:
615 571
616 572 filename = os.path.join(thisPath,file)
617 573
618 574 if not isFileInDateRange(filename, startDate, endDate):
619 575 continue
620 576
621 577 thisDatetime = isFileInTimeRange(filename, startTime, endTime)
622 578
623 579 if not(thisDatetime):
624 580 continue
625 581
626 582 filenameList.append(filename)
627 583 datetimeList.append(thisDatetime)
628 584
629 585 if not(filenameList):
630 print "Any file was found for the time range %s - %s" %(startTime, endTime)
586 print "[Reading] Any file was found int time range %s - %s" %(startTime.ctime(), endTime.ctime())
631 587 return None, None
632 588
633 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
589 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
634 590 print
635 591
636 592 for i in range(len(filenameList)):
637 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
593 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
638 594
639 595 self.filenameList = filenameList
640 596 self.datetimeList = datetimeList
641 597
642 598 return pathList, filenameList
643 599
644 600 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
645 601
646 602 """
647 603 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
648 604 devuelve el archivo encontrado ademas de otros datos.
649 605
650 606 Input:
651 607 path : carpeta donde estan contenidos los files que contiene data
652 608
653 609 expLabel : Nombre del subexperimento (subfolder)
654 610
655 611 ext : extension de los files
656 612
657 613 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
658 614
659 615 Return:
660 616 directory : eL directorio donde esta el file encontrado
661 617 filename : el ultimo file de una determinada carpeta
662 618 year : el anho
663 619 doy : el numero de dia del anho
664 620 set : el set del archivo
665 621
666 622
667 623 """
668 624 dirList = []
669 625
670 626 if not walk:
671 627 fullpath = path
672 628 foldercounter = 0
673 629 else:
674 630 #Filtra solo los directorios
675 631 for thisPath in os.listdir(path):
676 632 if not os.path.isdir(os.path.join(path,thisPath)):
677 633 continue
678 634 if not isRadarFolder(thisPath):
679 635 continue
680 636
681 637 dirList.append(thisPath)
682 638
683 639 if not(dirList):
684 640 return None, None, None, None, None, None
685 641
686 642 dirList = sorted( dirList, key=str.lower )
687 643
688 644 doypath = dirList[-1]
689 645 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
690 646 fullpath = os.path.join(path, doypath, expLabel)
691 647
692 648
693 print "%s folder was found: " %(fullpath )
649 print "[Reading] %s folder was found: " %(fullpath )
694 650
695 651 if set == None:
696 652 filename = getlastFileFromPath(fullpath, ext)
697 653 else:
698 654 filename = getFileFromSet(fullpath, ext, set)
699 655
700 656 if not(filename):
701 657 return None, None, None, None, None, None
702 658
703 print "%s file was found" %(filename)
659 print "[Reading] %s file was found" %(filename)
704 660
705 661 if not(self.__verifyFile(os.path.join(fullpath, filename))):
706 662 return None, None, None, None, None, None
707 663
708 664 year = int( filename[1:5] )
709 665 doy = int( filename[5:8] )
710 666 set = int( filename[8:11] )
711 667
712 668 return fullpath, foldercounter, filename, year, doy, set
713 669
714 670 def __setNextFileOffline(self):
715 671
716 672 idFile = self.fileIndex
717 673
718 674 while (True):
719 675 idFile += 1
720 676 if not(idFile < len(self.filenameList)):
721 677 self.flagNoMoreFiles = 1
722 678 # print "[Reading] No more Files"
723 679 return 0
724 680
725 681 filename = self.filenameList[idFile]
726 682
727 683 if not(self.__verifyFile(filename)):
728 684 continue
729 685
730 686 fileSize = os.path.getsize(filename)
731 687 fp = open(filename,'rb')
732 688 break
733 689
734 690 self.flagIsNewFile = 1
735 691 self.fileIndex = idFile
736 692 self.filename = filename
737 693 self.fileSize = fileSize
738 694 self.fp = fp
739 695
740 696 # print "[Reading] Setting the file: %s"%self.filename
741 697
742 698 return 1
743 699
744 700 def __setNextFileOnline(self):
745 701 """
746 702 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
747 703 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
748 704 siguientes.
749 705
750 706 Affected:
751 707 self.flagIsNewFile
752 708 self.filename
753 709 self.fileSize
754 710 self.fp
755 711 self.set
756 712 self.flagNoMoreFiles
757 713
758 714 Return:
759 715 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
760 716 1 : si el file fue abierto con exito y esta listo a ser leido
761 717
762 718 Excepciones:
763 719 Si un determinado file no puede ser abierto
764 720 """
765 721 nFiles = 0
766 722 fileOk_flag = False
767 723 firstTime_flag = True
768 724
769 725 self.set += 1
770 726
771 727 if self.set > 999:
772 728 self.set = 0
773 729 self.foldercounter += 1
774 730
775 731 #busca el 1er file disponible
776 732 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
777 733 if fullfilename:
778 734 if self.__verifyFile(fullfilename, False):
779 735 fileOk_flag = True
780 736
781 737 #si no encuentra un file entonces espera y vuelve a buscar
782 738 if not(fileOk_flag):
783 739 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
784 740
785 741 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
786 742 tries = self.nTries
787 743 else:
788 744 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
789 745
790 746 for nTries in range( tries ):
791 747 if firstTime_flag:
792 748 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
793 749 sleep( self.delay )
794 750 else:
795 751 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
796 752
797 753 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
798 754 if fullfilename:
799 755 if self.__verifyFile(fullfilename):
800 756 fileOk_flag = True
801 757 break
802 758
803 759 if fileOk_flag:
804 760 break
805 761
806 762 firstTime_flag = False
807 763
808 764 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
809 765 self.set += 1
810 766
811 767 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
812 768 self.set = 0
813 769 self.doy += 1
814 770 self.foldercounter = 0
815 771
816 772 if fileOk_flag:
817 773 self.fileSize = os.path.getsize( fullfilename )
818 774 self.filename = fullfilename
819 775 self.flagIsNewFile = 1
820 776 if self.fp != None: self.fp.close()
821 777 self.fp = open(fullfilename, 'rb')
822 778 self.flagNoMoreFiles = 0
823 779 # print '[Reading] Setting the file: %s' % fullfilename
824 780 else:
825 781 self.fileSize = 0
826 782 self.filename = None
827 783 self.flagIsNewFile = 0
828 784 self.fp = None
829 785 self.flagNoMoreFiles = 1
830 786 # print '[Reading] No more files to read'
831 787
832 788 return fileOk_flag
833 789
834 790 def setNextFile(self):
835 791 if self.fp != None:
836 792 self.fp.close()
837 793
838 794 if self.online:
839 795 newFile = self.__setNextFileOnline()
840 796 else:
841 797 newFile = self.__setNextFileOffline()
842 798
843 799 if not(newFile):
844 800 print '[Reading] No more files to read'
845 801 return 0
846 802
847 803 print '[Reading] Setting the file: %s' % self.filename
848 804
849 805 self.__readFirstHeader()
850 806 self.nReadBlocks = 0
851 807 return 1
852 808
853 809 def __waitNewBlock(self):
854 810 """
855 811 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
856 812
857 813 Si el modo de lectura es OffLine siempre retorn 0
858 814 """
859 815 if not self.online:
860 816 return 0
861 817
862 818 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
863 819 return 0
864 820
865 821 currentPointer = self.fp.tell()
866 822
867 823 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
868 824
869 825 for nTries in range( self.nTries ):
870 826
871 827 self.fp.close()
872 828 self.fp = open( self.filename, 'rb' )
873 829 self.fp.seek( currentPointer )
874 830
875 831 self.fileSize = os.path.getsize( self.filename )
876 832 currentSize = self.fileSize - currentPointer
877 833
878 834 if ( currentSize >= neededSize ):
879 835 self.basicHeaderObj.read(self.fp)
880 836 return 1
881 837
882 838 if self.fileSize == self.fileSizeByHeader:
883 839 # self.flagEoF = True
884 840 return 0
885 841
886 842 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
887 843 sleep( self.delay )
888 844
889 845
890 846 return 0
891 847
892 848 def waitDataBlock(self,pointer_location):
893 849
894 850 currentPointer = pointer_location
895 851
896 852 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
897 853
898 854 for nTries in range( self.nTries ):
899 855 self.fp.close()
900 856 self.fp = open( self.filename, 'rb' )
901 857 self.fp.seek( currentPointer )
902 858
903 859 self.fileSize = os.path.getsize( self.filename )
904 860 currentSize = self.fileSize - currentPointer
905 861
906 862 if ( currentSize >= neededSize ):
907 863 return 1
908 864
909 865 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
910 866 sleep( self.delay )
911 867
912 868 return 0
913 869
914 870 def __jumpToLastBlock(self):
915 871
916 872 if not(self.__isFirstTimeOnline):
917 873 return
918 874
919 875 csize = self.fileSize - self.fp.tell()
920 876 blocksize = self.processingHeaderObj.blockSize
921 877
922 878 #salta el primer bloque de datos
923 879 if csize > self.processingHeaderObj.blockSize:
924 880 self.fp.seek(self.fp.tell() + blocksize)
925 881 else:
926 882 return
927 883
928 884 csize = self.fileSize - self.fp.tell()
929 885 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
930 886 while True:
931 887
932 888 if self.fp.tell()<self.fileSize:
933 889 self.fp.seek(self.fp.tell() + neededsize)
934 890 else:
935 891 self.fp.seek(self.fp.tell() - neededsize)
936 892 break
937 893
938 894 # csize = self.fileSize - self.fp.tell()
939 895 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
940 896 # factor = int(csize/neededsize)
941 897 # if factor > 0:
942 898 # self.fp.seek(self.fp.tell() + factor*neededsize)
943 899
944 900 self.flagIsNewFile = 0
945 901 self.__isFirstTimeOnline = 0
946 902
947 903 def __setNewBlock(self):
948 904
949 905 if self.fp == None:
950 906 return 0
951 907
952 908 if self.online:
953 909 self.__jumpToLastBlock()
954 910
955 911 if self.flagIsNewFile:
956 912 return 1
957 913
958 914 self.lastUTTime = self.basicHeaderObj.utc
959 915 currentSize = self.fileSize - self.fp.tell()
960 916 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
961 917
962 918 if (currentSize >= neededSize):
963 919 self.basicHeaderObj.read(self.fp)
964 920 return 1
965 921
966 922 if self.__waitNewBlock():
967 923 return 1
968 924
969 925 if not(self.setNextFile()):
970 926 return 0
971 927
972 928 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
973 929
974 930 self.flagDiscontinuousBlock = 0
975 931
976 932 if deltaTime > self.maxTimeStep:
977 933 self.flagDiscontinuousBlock = 1
978 934
979 935 return 1
980 936
981 937 def readNextBlock(self):
982 938
983 939 if not(self.__setNewBlock()):
984 940 return 0
985 941
986 942 if not(self.readBlock()):
987 943 return 0
988 944
989 945 self.getBasicHeader()
990 946
991 947 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
992 948 self.processingHeaderObj.dataBlocksPerFile,
993 949 self.dataOut.datatime.ctime())
994 950 return 1
995 951
996 952 def __readFirstHeader(self):
997 953
998 954 self.basicHeaderObj.read(self.fp)
999 955 self.systemHeaderObj.read(self.fp)
1000 956 self.radarControllerHeaderObj.read(self.fp)
1001 957 self.processingHeaderObj.read(self.fp)
1002 958
1003 959 self.firstHeaderSize = self.basicHeaderObj.size
1004 960
1005 961 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1006 962 if datatype == 0:
1007 963 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1008 964 elif datatype == 1:
1009 965 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1010 966 elif datatype == 2:
1011 967 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1012 968 elif datatype == 3:
1013 969 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1014 970 elif datatype == 4:
1015 971 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1016 972 elif datatype == 5:
1017 973 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1018 974 else:
1019 975 raise ValueError, 'Data type was not defined'
1020 976
1021 977 self.dtype = datatype_str
1022 978 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1023 979 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1024 980 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1025 981 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1026 982 self.getBlockDimension()
1027 983
1028 984 def __verifyFile(self, filename, msgFlag=True):
1029 985 msg = None
1030 986 try:
1031 987 fp = open(filename, 'rb')
1032 988 currentPosition = fp.tell()
1033 989 except IOError:
1034 990 traceback.print_exc()
1035 991 if msgFlag:
1036 992 print "[Reading] The file %s can't be opened" % (filename)
1037 993 return False
1038 994
1039 995 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1040 996
1041 997 if neededSize == 0:
1042 998 basicHeaderObj = BasicHeader(LOCALTIME)
1043 999 systemHeaderObj = SystemHeader()
1044 1000 radarControllerHeaderObj = RadarControllerHeader()
1045 1001 processingHeaderObj = ProcessingHeader()
1046 1002
1047 1003 try:
1048 1004 if not( basicHeaderObj.read(fp) ): raise IOError
1049 1005 if not( systemHeaderObj.read(fp) ): raise IOError
1050 1006 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1051 1007 if not( processingHeaderObj.read(fp) ): raise IOError
1052 1008 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1053 1009
1054 1010 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1055 1011
1056 1012 except IOError:
1057 1013 traceback.print_exc()
1058 1014 sys.exit(0)
1059 1015
1060 1016 if msgFlag:
1061 1017 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1062 1018
1063 1019 fp.close()
1064 1020 return False
1065 1021 else:
1066 1022 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1067 1023
1068 1024 fp.close()
1069 1025 fileSize = os.path.getsize(filename)
1070 1026 currentSize = fileSize - currentPosition
1071 1027 if currentSize < neededSize:
1072 1028 if msgFlag and (msg != None):
1073 1029 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1074 1030 return False
1075 1031
1076 1032 return True
1077 1033
1078 1034 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1079 1035
1080 1036 dateList = []
1081 1037 pathList = []
1082 1038
1083 1039 multi_path = path.split(',')
1084 1040
1085 1041 if not walk:
1086 1042
1087 1043 for single_path in multi_path:
1088 1044
1089 1045 if not os.path.isdir(single_path):
1090 1046 continue
1091 1047
1092 1048 fileList = glob.glob1(single_path, "*"+ext)
1093 1049
1094 1050 for thisFile in fileList:
1095 1051
1096 1052 if not os.path.isfile(os.path.join(single_path, thisFile)):
1097 1053 continue
1098 1054
1099 1055 if not isRadarFile(thisFile):
1100 1056 continue
1101 1057
1102 1058 if not isFileInDateRange(thisFile, startDate, endDate):
1103 1059 continue
1104 1060
1105 1061 thisDate = getDateFromRadarFile(thisFile)
1106 1062
1107 1063 if thisDate in dateList:
1108 1064 continue
1109 1065
1110 1066 dateList.append(thisDate)
1111 1067 pathList.append(single_path)
1112 1068
1113 1069 else:
1114 1070 for single_path in multi_path:
1115 1071
1116 1072 if not os.path.isdir(single_path):
1117 1073 continue
1118 1074
1119 1075 dirList = []
1120 1076
1121 1077 for thisPath in os.listdir(single_path):
1122 1078
1123 1079 if not os.path.isdir(os.path.join(single_path,thisPath)):
1124 1080 continue
1125 1081
1126 1082 if not isRadarFolder(thisPath):
1127 1083 continue
1128 1084
1129 1085 if not isFolderInDateRange(thisPath, startDate, endDate):
1130 1086 continue
1131 1087
1132 1088 dirList.append(thisPath)
1133 1089
1134 1090 if not dirList:
1135 1091 continue
1136 1092
1137 1093 for thisDir in dirList:
1138 1094
1139 1095 datapath = os.path.join(single_path, thisDir, expLabel)
1140 1096 fileList = glob.glob1(datapath, "*"+ext)
1141 1097
1142 1098 if len(fileList) < 1:
1143 1099 continue
1144 1100
1145 1101 thisDate = getDateFromRadarFolder(thisDir)
1146 1102
1147 1103 pathList.append(datapath)
1148 1104 dateList.append(thisDate)
1149 1105
1150 1106 dateList.sort()
1151 1107
1152 1108 if include_path:
1153 1109 return dateList, pathList
1154 1110
1155 1111 return dateList
1156 1112
1157 1113 def setup(self,
1158 1114 path=None,
1159 1115 startDate=None,
1160 1116 endDate=None,
1161 1117 startTime=datetime.time(0,0,0),
1162 1118 endTime=datetime.time(23,59,59),
1163 1119 set=None,
1164 1120 expLabel = "",
1165 1121 ext = None,
1166 1122 online = False,
1167 1123 delay = 60,
1168 1124 walk = True,
1169 1125 getblock = False,
1170 1126 nTxs = 1):
1171 1127
1172 1128 if path == None:
1173 1129 raise ValueError, "[Reading] The path is not valid"
1174 1130
1175 1131 if ext == None:
1176 1132 ext = self.ext
1177 1133
1178 1134 if online:
1179 1135 print "[Reading] Searching files in online mode..."
1180 1136
1181 1137 for nTries in range( self.nTries ):
1182 1138 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1183 1139
1184 1140 if fullpath:
1185 1141 break
1186 1142
1187 1143 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1188 1144 sleep( self.delay )
1189 1145
1190 1146 if not(fullpath):
1191 1147 print "[Reading] There 'isn't any valid file in %s" % path
1192 1148 return None
1193 1149
1194 1150 self.year = year
1195 1151 self.doy = doy
1196 1152 self.set = set - 1
1197 1153 self.path = path
1198 1154 self.foldercounter = foldercounter
1199 1155 last_set = None
1200 1156
1201 1157 else:
1202 1158 print "[Reading] Searching files in offline mode ..."
1203 1159 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1204 1160 startTime=startTime, endTime=endTime,
1205 1161 set=set, expLabel=expLabel, ext=ext,
1206 1162 walk=walk)
1207 1163
1208 1164 if not(pathList):
1209 print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
1210 datetime.datetime.combine(startDate,startTime).ctime(),
1211 datetime.datetime.combine(endDate,endTime).ctime())
1212
1165 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1166 # datetime.datetime.combine(startDate,startTime).ctime(),
1167 # datetime.datetime.combine(endDate,endTime).ctime())
1168 #
1213 1169 sys.exit(-1)
1214 1170
1215 1171
1216 1172 self.fileIndex = -1
1217 1173 self.pathList = pathList
1218 1174 self.filenameList = filenameList
1219 1175 file_name = os.path.basename(filenameList[-1])
1220 1176 basename, ext = os.path.splitext(file_name)
1221 1177 last_set = int(basename[-3:])
1222 1178
1223 1179 self.online = online
1224 1180 self.delay = delay
1225 1181 ext = ext.lower()
1226 1182 self.ext = ext
1227 1183 self.getByBlock = getblock
1228 1184 self.nTxs = int(nTxs)
1229 1185
1230 1186 if not(self.setNextFile()):
1231 1187 if (startDate!=None) and (endDate!=None):
1232 1188 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1233 1189 elif startDate != None:
1234 1190 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1235 1191 else:
1236 1192 print "[Reading] No files"
1237 1193
1238 1194 sys.exit(-1)
1239 1195
1240 1196 # self.getBasicHeader()
1241 1197
1242 1198 if last_set != None:
1243 1199 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1244 1200 return
1245 1201
1246 1202 def getBasicHeader(self):
1247 1203
1248 1204 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1249 1205
1250 1206 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1251 1207
1252 1208 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1253 1209
1254 1210 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1255 1211
1256 1212 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1257 1213
1258 1214 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1259 1215
1260 1216 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1261 1217
1262 1218 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1263 1219
1264 1220
1265 1221 def getFirstHeader(self):
1266 1222
1267 1223 raise ValueError, "This method has not been implemented"
1268 1224
1269 1225 def getData(self):
1270 1226
1271 1227 raise ValueError, "This method has not been implemented"
1272 1228
1273 1229 def hasNotDataInBuffer(self):
1274 1230
1275 1231 raise ValueError, "This method has not been implemented"
1276 1232
1277 1233 def readBlock(self):
1278 1234
1279 1235 raise ValueError, "This method has not been implemented"
1280 1236
1281 1237 def isEndProcess(self):
1282 1238
1283 1239 return self.flagNoMoreFiles
1284 1240
1285 1241 def printReadBlocks(self):
1286 1242
1287 1243 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1288 1244
1289 1245 def printTotalBlocks(self):
1290 1246
1291 1247 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1292 1248
1293 1249 def printNumberOfBlock(self):
1294 1250
1295 1251 if self.flagIsNewBlock:
1296 1252 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1297 1253 self.processingHeaderObj.dataBlocksPerFile,
1298 1254 self.dataOut.datatime.ctime())
1299 1255
1300 1256 def printInfo(self):
1301 1257
1302 1258 if self.__printInfo == False:
1303 1259 return
1304 1260
1305 1261 self.basicHeaderObj.printInfo()
1306 1262 self.systemHeaderObj.printInfo()
1307 1263 self.radarControllerHeaderObj.printInfo()
1308 1264 self.processingHeaderObj.printInfo()
1309 1265
1310 1266 self.__printInfo = False
1311 1267
1312 1268
1313 1269 def run(self, **kwargs):
1314 1270
1315 1271 if not(self.isConfig):
1316 1272
1317 1273 # self.dataOut = dataOut
1318 1274 self.setup(**kwargs)
1319 1275 self.isConfig = True
1320 1276
1321 1277 self.getData()
1322 1278
1323 1279 class JRODataWriter(JRODataIO):
1324 1280
1325 1281 """
1326 1282 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1327 1283 de los datos siempre se realiza por bloques.
1328 1284 """
1329 1285
1330 1286 blockIndex = 0
1331 1287
1332 1288 path = None
1333 1289
1334 1290 setFile = None
1335 1291
1336 1292 profilesPerBlock = None
1337 1293
1338 1294 blocksPerFile = None
1339 1295
1340 1296 nWriteBlocks = 0
1341 1297
1342 1298 fileDate = None
1343 1299
1344 1300 def __init__(self, dataOut=None):
1345 1301 raise ValueError, "Not implemented"
1346 1302
1347 1303
1348 1304 def hasAllDataInBuffer(self):
1349 1305 raise ValueError, "Not implemented"
1350 1306
1351 1307
1352 1308 def setBlockDimension(self):
1353 1309 raise ValueError, "Not implemented"
1354 1310
1355 1311
1356 1312 def writeBlock(self):
1357 1313 raise ValueError, "No implemented"
1358 1314
1359 1315
1360 1316 def putData(self):
1361 1317 raise ValueError, "No implemented"
1362 1318
1363 1319
1364 1320 def getProcessFlags(self):
1365 1321
1366 1322 processFlags = 0
1367 1323
1368 1324 dtype_index = get_dtype_index(self.dtype)
1369 1325 procflag_dtype = get_procflag_dtype(dtype_index)
1370 1326
1371 1327 processFlags += procflag_dtype
1372 1328
1373 1329 if self.dataOut.flagDecodeData:
1374 1330 processFlags += PROCFLAG.DECODE_DATA
1375 1331
1376 1332 if self.dataOut.flagDeflipData:
1377 1333 processFlags += PROCFLAG.DEFLIP_DATA
1378 1334
1379 1335 if self.dataOut.code is not None:
1380 1336 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1381 1337
1382 1338 if self.dataOut.nCohInt > 1:
1383 1339 processFlags += PROCFLAG.COHERENT_INTEGRATION
1384 1340
1385 1341 if self.dataOut.type == "Spectra":
1386 1342 if self.dataOut.nIncohInt > 1:
1387 1343 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1388 1344
1389 1345 if self.dataOut.data_dc is not None:
1390 1346 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1391 1347
1392 1348 if self.dataOut.flagShiftFFT:
1393 1349 processFlags += PROCFLAG.SHIFT_FFT_DATA
1394 1350
1395 1351 return processFlags
1396 1352
1397 1353 def setBasicHeader(self):
1398 1354
1399 1355 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1400 1356 self.basicHeaderObj.version = self.versionFile
1401 1357 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1402 1358
1403 1359 utc = numpy.floor(self.dataOut.utctime)
1404 1360 milisecond = (self.dataOut.utctime - utc)* 1000.0
1405 1361
1406 1362 self.basicHeaderObj.utc = utc
1407 1363 self.basicHeaderObj.miliSecond = milisecond
1408 1364 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1409 1365 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1410 1366 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1411 1367
1412 1368 def setFirstHeader(self):
1413 1369 """
1414 1370 Obtiene una copia del First Header
1415 1371
1416 1372 Affected:
1417 1373
1418 1374 self.basicHeaderObj
1419 1375 self.systemHeaderObj
1420 1376 self.radarControllerHeaderObj
1421 1377 self.processingHeaderObj self.
1422 1378
1423 1379 Return:
1424 1380 None
1425 1381 """
1426 1382
1427 1383 raise ValueError, "No implemented"
1428 1384
1429 1385 def __writeFirstHeader(self):
1430 1386 """
1431 1387 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1432 1388
1433 1389 Affected:
1434 1390 __dataType
1435 1391
1436 1392 Return:
1437 1393 None
1438 1394 """
1439 1395
1440 1396 # CALCULAR PARAMETROS
1441 1397
1442 1398 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1443 1399 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1444 1400
1445 1401 self.basicHeaderObj.write(self.fp)
1446 1402 self.systemHeaderObj.write(self.fp)
1447 1403 self.radarControllerHeaderObj.write(self.fp)
1448 1404 self.processingHeaderObj.write(self.fp)
1449 1405
1450 1406 def __setNewBlock(self):
1451 1407 """
1452 1408 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1453 1409
1454 1410 Return:
1455 1411 0 : si no pudo escribir nada
1456 1412 1 : Si escribio el Basic el First Header
1457 1413 """
1458 1414 if self.fp == None:
1459 1415 self.setNextFile()
1460 1416
1461 1417 if self.flagIsNewFile:
1462 1418 return 1
1463 1419
1464 1420 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1465 1421 self.basicHeaderObj.write(self.fp)
1466 1422 return 1
1467 1423
1468 1424 if not( self.setNextFile() ):
1469 1425 return 0
1470 1426
1471 1427 return 1
1472 1428
1473 1429
1474 1430 def writeNextBlock(self):
1475 1431 """
1476 1432 Selecciona el bloque siguiente de datos y los escribe en un file
1477 1433
1478 1434 Return:
1479 1435 0 : Si no hizo pudo escribir el bloque de datos
1480 1436 1 : Si no pudo escribir el bloque de datos
1481 1437 """
1482 1438 if not( self.__setNewBlock() ):
1483 1439 return 0
1484 1440
1485 1441 self.writeBlock()
1486 1442
1487 1443 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1488 1444 self.processingHeaderObj.dataBlocksPerFile)
1489 1445
1490 1446 return 1
1491 1447
1492 1448 def setNextFile(self):
1493 1449 """
1494 1450 Determina el siguiente file que sera escrito
1495 1451
1496 1452 Affected:
1497 1453 self.filename
1498 1454 self.subfolder
1499 1455 self.fp
1500 1456 self.setFile
1501 1457 self.flagIsNewFile
1502 1458
1503 1459 Return:
1504 1460 0 : Si el archivo no puede ser escrito
1505 1461 1 : Si el archivo esta listo para ser escrito
1506 1462 """
1507 1463 ext = self.ext
1508 1464 path = self.path
1509 1465
1510 1466 if self.fp != None:
1511 1467 self.fp.close()
1512 1468
1513 1469 timeTuple = time.localtime( self.dataOut.utctime)
1514 1470 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1515 1471
1516 1472 fullpath = os.path.join( path, subfolder )
1517 1473 setFile = self.setFile
1518 1474
1519 1475 if not( os.path.exists(fullpath) ):
1520 1476 os.mkdir(fullpath)
1521 1477 setFile = -1 #inicializo mi contador de seteo
1522 1478 else:
1523 1479 filesList = os.listdir( fullpath )
1524 1480 if len( filesList ) > 0:
1525 1481 filesList = sorted( filesList, key=str.lower )
1526 1482 filen = filesList[-1]
1527 1483 # el filename debera tener el siguiente formato
1528 1484 # 0 1234 567 89A BCDE (hex)
1529 1485 # x YYYY DDD SSS .ext
1530 1486 if isNumber( filen[8:11] ):
1531 1487 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1532 1488 else:
1533 1489 setFile = -1
1534 1490 else:
1535 1491 setFile = -1 #inicializo mi contador de seteo
1536 1492
1537 1493 setFile += 1
1538 1494
1539 1495 #If this is a new day it resets some values
1540 1496 if self.dataOut.datatime.date() > self.fileDate:
1541 1497 setFile = 0
1542 1498 self.nTotalBlocks = 0
1543 1499
1544 1500 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1545 1501
1546 1502 filename = os.path.join( path, subfolder, filen )
1547 1503
1548 1504 fp = open( filename,'wb' )
1549 1505
1550 1506 self.blockIndex = 0
1551 1507
1552 1508 #guardando atributos
1553 1509 self.filename = filename
1554 1510 self.subfolder = subfolder
1555 1511 self.fp = fp
1556 1512 self.setFile = setFile
1557 1513 self.flagIsNewFile = 1
1558 1514 self.fileDate = self.dataOut.datatime.date()
1559 1515
1560 1516 self.setFirstHeader()
1561 1517
1562 1518 print '[Writing] Opening file: %s'%self.filename
1563 1519
1564 1520 self.__writeFirstHeader()
1565 1521
1566 1522 return 1
1567 1523
1568 1524 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1569 1525 """
1570 1526 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1571 1527
1572 1528 Inputs:
1573 1529 path : directory where data will be saved
1574 1530 profilesPerBlock : number of profiles per block
1575 1531 set : initial file set
1576 1532 datatype : An integer number that defines data type:
1577 1533 0 : int8 (1 byte)
1578 1534 1 : int16 (2 bytes)
1579 1535 2 : int32 (4 bytes)
1580 1536 3 : int64 (8 bytes)
1581 1537 4 : float32 (4 bytes)
1582 1538 5 : double64 (8 bytes)
1583 1539
1584 1540 Return:
1585 1541 0 : Si no realizo un buen seteo
1586 1542 1 : Si realizo un buen seteo
1587 1543 """
1588 1544
1589 1545 if ext == None:
1590 1546 ext = self.ext
1591 1547
1592 1548 self.ext = ext.lower()
1593 1549
1594 1550 self.path = path
1595 1551
1596 1552 if set is None:
1597 1553 self.setFile = -1
1598 1554 else:
1599 1555 self.setFile = set - 1
1600 1556
1601 1557 self.blocksPerFile = blocksPerFile
1602 1558
1603 1559 self.profilesPerBlock = profilesPerBlock
1604 1560
1605 1561 self.dataOut = dataOut
1606 1562 self.fileDate = self.dataOut.datatime.date()
1607 1563 #By default
1608 1564 self.dtype = self.dataOut.dtype
1609 1565
1610 1566 if datatype is not None:
1611 1567 self.dtype = get_numpy_dtype(datatype)
1612 1568
1613 1569 if not(self.setNextFile()):
1614 1570 print "[Writing] There isn't a next file"
1615 1571 return 0
1616 1572
1617 1573 self.setBlockDimension()
1618 1574
1619 1575 return 1
1620 1576
1621 1577 def run(self, dataOut, **kwargs):
1622 1578
1623 1579 if not(self.isConfig):
1624 1580
1625 1581 self.setup(dataOut, **kwargs)
1626 1582 self.isConfig = True
1627 1583
1628 1584 self.putData()
1629 1585
General Comments 0
You need to be logged in to leave comments. Login now