##// END OF EJS Templates
schain-jc
RSS
Alexander Valdez -
r563:d1e598150683
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1344 +1,1349
1 1 '''
2 2
3 3 '''
4 4 import os
5 5 import sys
6 6 import glob
7 7 import time
8 8 import numpy
9 9 import fnmatch
10 10 import time, datetime
11 11 #import h5py
12 12 import traceback
13 13
14 14 #try:
15 15 # import pyfits
16 16 #except:
17 17 # print "pyfits module has not been imported, it should be installed to save files in fits format"
18 18
19 19 #from jrodata import *
20 20 #from jroheaderIO import *
21 21 #from jroprocessing import *
22 22
23 23 #import re
24 24 #from xml.etree.ElementTree import Element, SubElement, ElementTree
25 25
26 26
27 27 LOCALTIME = True #-18000
28 28
29 29 from model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
30 30
31 31 def isNumber(str):
32 32 """
33 33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
34 34
35 35 Excepciones:
36 36 Si un determinado string no puede ser convertido a numero
37 37 Input:
38 38 str, string al cual se le analiza para determinar si convertible a un numero o no
39 39
40 40 Return:
41 41 True : si el string es uno numerico
42 42 False : no es un string numerico
43 43 """
44 44 try:
45 45 float( str )
46 46 return True
47 47 except:
48 48 return False
49 49
50 50 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
51 51 """
52 52 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
53 53
54 54 Inputs:
55 55 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
56 56
57 57 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
58 58 segundos contados desde 01/01/1970.
59 59 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
60 60 segundos contados desde 01/01/1970.
61 61
62 62 Return:
63 63 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
64 64 fecha especificado, de lo contrario retorna False.
65 65
66 66 Excepciones:
67 67 Si el archivo no existe o no puede ser abierto
68 68 Si la cabecera no puede ser leida.
69 69
70 70 """
71 71 basicHeaderObj = BasicHeader(LOCALTIME)
72 72
73 73 try:
74 74 fp = open(filename,'rb')
75 75 except IOError:
76 76 traceback.print_exc()
77 77 raise IOError, "The file %s can't be opened" %(filename)
78 78
79 79 sts = basicHeaderObj.read(fp)
80 80 fp.close()
81 81
82 82 if not(sts):
83 83 print "Skipping the file %s because it has not a valid header" %(filename)
84 84 return 0
85 85
86 86 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
87 87 return 0
88 88
89 89 return 1
90 90
91 91 def isFileinThisTime(filename, startTime, endTime):
92 92 """
93 93 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
94 94
95 95 Inputs:
96 96 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
97 97
98 98 startTime : tiempo inicial del rango seleccionado en formato datetime.time
99 99
100 100 endTime : tiempo final del rango seleccionado en formato datetime.time
101 101
102 102 Return:
103 103 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
104 104 fecha especificado, de lo contrario retorna False.
105 105
106 106 Excepciones:
107 107 Si el archivo no existe o no puede ser abierto
108 108 Si la cabecera no puede ser leida.
109 109
110 110 """
111 111
112 112
113 113 try:
114 114 fp = open(filename,'rb')
115 115 except IOError:
116 116 traceback.print_exc()
117 117 raise IOError, "The file %s can't be opened" %(filename)
118 118
119 119 basicHeaderObj = BasicHeader(LOCALTIME)
120 120 sts = basicHeaderObj.read(fp)
121 121 fp.close()
122 122
123 123 thisDatetime = basicHeaderObj.datatime
124 124 thisTime = thisDatetime.time()
125 125
126 126 if not(sts):
127 127 print "Skipping the file %s because it has not a valid header" %(filename)
128 128 return None
129 129
130 130 if not ((startTime <= thisTime) and (endTime > thisTime)):
131 131 return None
132 132
133 133 return thisDatetime
134 134
135 135 def getFileFromSet(path, ext, set):
136 136 validFilelist = []
137 137 fileList = os.listdir(path)
138 138
139 139 # 0 1234 567 89A BCDE
140 140 # H YYYY DDD SSS .ext
141 141
142 142 for thisFile in fileList:
143 143 try:
144 144 year = int(thisFile[1:5])
145 145 doy = int(thisFile[5:8])
146 146 except:
147 147 continue
148 148
149 149 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
150 150 continue
151 151
152 152 validFilelist.append(thisFile)
153 153
154 154 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
155 155
156 156 if len(myfile)!= 0:
157 157 return myfile[0]
158 158 else:
159 159 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
160 160 print 'the filename %s does not exist'%filename
161 161 print '...going to the last file: '
162 162
163 163 if validFilelist:
164 164 validFilelist = sorted( validFilelist, key=str.lower )
165 165 return validFilelist[-1]
166 166
167 167 return None
168 168
169 169 def getlastFileFromPath(path, ext):
170 170 """
171 171 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
172 172 al final de la depuracion devuelve el ultimo file de la lista que quedo.
173 173
174 174 Input:
175 175 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
176 176 ext : extension de los files contenidos en una carpeta
177 177
178 178 Return:
179 179 El ultimo file de una determinada carpeta, no se considera el path.
180 180 """
181 181 validFilelist = []
182 182 fileList = os.listdir(path)
183 183
184 184 # 0 1234 567 89A BCDE
185 185 # H YYYY DDD SSS .ext
186 186
187 187 for thisFile in fileList:
188 188
189 189 year = thisFile[1:5]
190 190 if not isNumber(year):
191 191 continue
192 192
193 193 doy = thisFile[5:8]
194 194 if not isNumber(doy):
195 195 continue
196 196
197 197 year = int(year)
198 198 doy = int(doy)
199 199
200 200 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
201 201 continue
202 202
203 203 validFilelist.append(thisFile)
204 204
205 205 if validFilelist:
206 206 validFilelist = sorted( validFilelist, key=str.lower )
207 207 return validFilelist[-1]
208 208
209 209 return None
210 210
211 211 def checkForRealPath(path, foldercounter, year, doy, set, ext):
212 212 """
213 213 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
214 214 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
215 215 el path exacto de un determinado file.
216 216
217 217 Example :
218 218 nombre correcto del file es .../.../D2009307/P2009307367.ext
219 219
220 220 Entonces la funcion prueba con las siguientes combinaciones
221 221 .../.../y2009307367.ext
222 222 .../.../Y2009307367.ext
223 223 .../.../x2009307/y2009307367.ext
224 224 .../.../x2009307/Y2009307367.ext
225 225 .../.../X2009307/y2009307367.ext
226 226 .../.../X2009307/Y2009307367.ext
227 227 siendo para este caso, la ultima combinacion de letras, identica al file buscado
228 228
229 229 Return:
230 230 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
231 231 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
232 232 para el filename
233 233 """
234 234 fullfilename = None
235 235 find_flag = False
236 236 filename = None
237 237
238 238 prefixDirList = [None,'d','D']
239 239 if ext.lower() == ".r": #voltage
240 240 prefixFileList = ['d','D']
241 241 elif ext.lower() == ".pdata": #spectra
242 242 prefixFileList = ['p','P']
243 243 else:
244 244 return None, filename
245 245
246 246 #barrido por las combinaciones posibles
247 247 for prefixDir in prefixDirList:
248 248 thispath = path
249 249 if prefixDir != None:
250 250 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
251 251 if foldercounter == 0:
252 252 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
253 253 else:
254 254 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
255 255 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
256 256 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
257 257 fullfilename = os.path.join( thispath, filename ) #formo el path completo
258 258
259 259 if os.path.exists( fullfilename ): #verifico que exista
260 260 find_flag = True
261 261 break
262 262 if find_flag:
263 263 break
264 264
265 265 if not(find_flag):
266 266 return None, filename
267 267
268 268 return fullfilename, filename
269 269
270 270 def isDoyFolder(folder):
271 271 try:
272 272 year = int(folder[1:5])
273 273 except:
274 274 return 0
275 275
276 276 try:
277 277 doy = int(folder[5:8])
278 278 except:
279 279 return 0
280 280
281 281 return 1
282 282
283 283 class JRODataIO:
284 284
285 285 c = 3E8
286 286
287 287 isConfig = False
288 288
289 289 basicHeaderObj = None
290 290
291 291 systemHeaderObj = None
292 292
293 293 radarControllerHeaderObj = None
294 294
295 295 processingHeaderObj = None
296 296
297 297 online = 0
298 298
299 299 dtype = None
300 300
301 301 pathList = []
302 302
303 303 filenameList = []
304 304
305 305 filename = None
306 306
307 307 ext = None
308 308
309 309 flagIsNewFile = 1
310 310
311 311 flagTimeBlock = 0
312 312
313 313 flagIsNewBlock = 0
314 314
315 315 fp = None
316 316
317 317 firstHeaderSize = 0
318 318
319 319 basicHeaderSize = 24
320 320
321 321 versionFile = 1103
322 322
323 323 fileSize = None
324 324
325 325 # ippSeconds = None
326 326
327 327 fileSizeByHeader = None
328 328
329 329 fileIndex = None
330 330
331 331 profileIndex = None
332 332
333 333 blockIndex = None
334 334
335 335 nTotalBlocks = None
336 336
337 337 maxTimeStep = 30
338 338
339 339 lastUTTime = None
340 340
341 341 datablock = None
342 342
343 343 dataOut = None
344 344
345 345 blocksize = None
346 346
347 347 getByBlock = False
348 348
349 349 def __init__(self):
350 350
351 351 raise ValueError, "Not implemented"
352 352
353 353 def run(self):
354 354
355 355 raise ValueError, "Not implemented"
356 356
357 357 class JRODataReader(JRODataIO):
358 358
359 359 nReadBlocks = 0
360 360
361 361 delay = 10 #number of seconds waiting a new file
362 362
363 363 nTries = 3 #quantity tries
364 364
365 365 nFiles = 3 #number of files for searching
366 366
367 367 path = None
368 368
369 369 foldercounter = 0
370 370
371 371 flagNoMoreFiles = 0
372 372
373 373 datetimeList = []
374 374
375 375 __isFirstTimeOnline = 1
376 376
377 377 __printInfo = True
378 378
379 379 profileIndex = None
380 380
381 381 nTxs = 1
382 382
383 383 txIndex = None
384 384
385 385 def __init__(self):
386 386
387 387 """
388 388
389 389 """
390 390
391 391 raise ValueError, "This method has not been implemented"
392 392
393 393
394 394 def createObjByDefault(self):
395 395 """
396 396
397 397 """
398 398 raise ValueError, "This method has not been implemented"
399 399
400 400 def getBlockDimension(self):
401 401
402 402 raise ValueError, "No implemented"
403 403
404 404 def __searchFilesOffLine(self,
405 405 path,
406 406 startDate,
407 407 endDate,
408 408 startTime=datetime.time(0,0,0),
409 409 endTime=datetime.time(23,59,59),
410 410 set=None,
411 411 expLabel='',
412 412 ext='.r',
413 413 walk=True):
414 414
415 415 pathList = []
416 416
417 417 if not walk:
418 418 #pathList.append(path)
419 419 multi_path = path.split(',')
420 420 for single_path in multi_path:
421 421 pathList.append(single_path)
422 422
423 423 else:
424 424 #dirList = []
425 425 multi_path = path.split(',')
426 426 for single_path in multi_path:
427 427 dirList = []
428 428 for thisPath in os.listdir(single_path):
429 429 if not os.path.isdir(os.path.join(single_path,thisPath)):
430 430 continue
431 431 if not isDoyFolder(thisPath):
432 432 continue
433 433
434 434 dirList.append(thisPath)
435 435
436 436 if not(dirList):
437 437 return None, None
438 438
439 439 thisDate = startDate
440 440
441 441 while(thisDate <= endDate):
442 442 year = thisDate.timetuple().tm_year
443 443 doy = thisDate.timetuple().tm_yday
444 444
445 445 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
446 446 if len(matchlist) == 0:
447 447 thisDate += datetime.timedelta(1)
448 448 continue
449 449 for match in matchlist:
450 450 pathList.append(os.path.join(single_path,match,expLabel))
451 451
452 452 thisDate += datetime.timedelta(1)
453 453
454 454 if pathList == []:
455 455 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
456 456 return None, None
457 457
458 458 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
459 459
460 460 filenameList = []
461 461 datetimeList = []
462 462 pathDict = {}
463 463 filenameList_to_sort = []
464 464
465 465 for i in range(len(pathList)):
466 466
467 467 thisPath = pathList[i]
468 468
469 469 fileList = glob.glob1(thisPath, "*%s" %ext)
470 470 fileList.sort()
471 471 pathDict.setdefault(fileList[0])
472 472 pathDict[fileList[0]] = i
473 473 filenameList_to_sort.append(fileList[0])
474 474
475 475 filenameList_to_sort.sort()
476 476
477 477 for file in filenameList_to_sort:
478 478 thisPath = pathList[pathDict[file]]
479 479
480 480 fileList = glob.glob1(thisPath, "*%s" %ext)
481 481 fileList.sort()
482 482
483 483 for file in fileList:
484 484
485 485 filename = os.path.join(thisPath,file)
486 486 thisDatetime = isFileinThisTime(filename, startTime, endTime)
487 487
488 488 if not(thisDatetime):
489 489 continue
490 490
491 491 filenameList.append(filename)
492 492 datetimeList.append(thisDatetime)
493 493
494 494 if not(filenameList):
495 495 print "Any file was found for the time range %s - %s" %(startTime, endTime)
496 496 return None, None
497 497
498 498 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
499 499 print
500 500
501 501 for i in range(len(filenameList)):
502 502 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
503 503
504 504 self.filenameList = filenameList
505 505 self.datetimeList = datetimeList
506 506
507 507 return pathList, filenameList
508 508
509 509 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
510 510
511 511 """
512 512 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
513 513 devuelve el archivo encontrado ademas de otros datos.
514 514
515 515 Input:
516 516 path : carpeta donde estan contenidos los files que contiene data
517 517
518 518 expLabel : Nombre del subexperimento (subfolder)
519 519
520 520 ext : extension de los files
521 521
522 522 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
523 523
524 524 Return:
525 525 directory : eL directorio donde esta el file encontrado
526 526 filename : el ultimo file de una determinada carpeta
527 527 year : el anho
528 528 doy : el numero de dia del anho
529 529 set : el set del archivo
530 530
531 531
532 532 """
533 533 dirList = []
534 534
535 535 if not walk:
536 536 fullpath = path
537 537 foldercounter = 0
538 538 else:
539 539 #Filtra solo los directorios
540 540 for thisPath in os.listdir(path):
541 541 if not os.path.isdir(os.path.join(path,thisPath)):
542 542 continue
543 543 if not isDoyFolder(thisPath):
544 544 continue
545 545
546 546 dirList.append(thisPath)
547 547
548 548 if not(dirList):
549 549 return None, None, None, None, None, None
550 550
551 551 dirList = sorted( dirList, key=str.lower )
552 552
553 553 doypath = dirList[-1]
554 554 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
555 555 fullpath = os.path.join(path, doypath, expLabel)
556 556
557 557
558 558 print "%s folder was found: " %(fullpath )
559 559
560 560 if set == None:
561 561 filename = getlastFileFromPath(fullpath, ext)
562 562 else:
563 563 filename = getFileFromSet(fullpath, ext, set)
564 564
565 565 if not(filename):
566 566 return None, None, None, None, None, None
567 567
568 568 print "%s file was found" %(filename)
569 569
570 570 if not(self.__verifyFile(os.path.join(fullpath, filename))):
571 571 return None, None, None, None, None, None
572 572
573 573 year = int( filename[1:5] )
574 574 doy = int( filename[5:8] )
575 575 set = int( filename[8:11] )
576 576
577 577 return fullpath, foldercounter, filename, year, doy, set
578 578
579 579 def __setNextFileOffline(self):
580 580
581 581 idFile = self.fileIndex
582 582
583 583 while (True):
584 584 idFile += 1
585 585 if not(idFile < len(self.filenameList)):
586 586 self.flagNoMoreFiles = 1
587 587 print "No more Files"
588 588 return 0
589 589
590 590 filename = self.filenameList[idFile]
591 591
592 592 if not(self.__verifyFile(filename)):
593 593 continue
594 594
595 595 fileSize = os.path.getsize(filename)
596 596 fp = open(filename,'rb')
597 597 break
598 598
599 599 self.flagIsNewFile = 1
600 600 self.fileIndex = idFile
601 601 self.filename = filename
602 602 self.fileSize = fileSize
603 603 self.fp = fp
604 604
605 605 print "Setting the file: %s"%self.filename
606 606
607 607 return 1
608 608
609 609 def __setNextFileOnline(self):
610 610 """
611 611 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
612 612 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
613 613 siguientes.
614 614
615 615 Affected:
616 616 self.flagIsNewFile
617 617 self.filename
618 618 self.fileSize
619 619 self.fp
620 620 self.set
621 621 self.flagNoMoreFiles
622 622
623 623 Return:
624 624 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
625 625 1 : si el file fue abierto con exito y esta listo a ser leido
626 626
627 627 Excepciones:
628 628 Si un determinado file no puede ser abierto
629 629 """
630 630 nFiles = 0
631 631 fileOk_flag = False
632 632 firstTime_flag = True
633 633
634 634 self.set += 1
635 635
636 636 if self.set > 999:
637 637 self.set = 0
638 638 self.foldercounter += 1
639 639
640 640 #busca el 1er file disponible
641 641 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
642 642 if fullfilename:
643 643 if self.__verifyFile(fullfilename, False):
644 644 fileOk_flag = True
645 645
646 646 #si no encuentra un file entonces espera y vuelve a buscar
647 647 if not(fileOk_flag):
648 648 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
649 649
650 650 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
651 651 tries = self.nTries
652 652 else:
653 653 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
654 654
655 655 for nTries in range( tries ):
656 656 if firstTime_flag:
657 657 print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
658 658 time.sleep( self.delay )
659 659 else:
660 660 print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
661 661
662 662 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
663 663 if fullfilename:
664 664 if self.__verifyFile(fullfilename):
665 665 fileOk_flag = True
666 666 break
667 667
668 668 if fileOk_flag:
669 669 break
670 670
671 671 firstTime_flag = False
672 672
673 673 print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename
674 674 self.set += 1
675 675
676 676 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
677 677 self.set = 0
678 678 self.doy += 1
679 679 self.foldercounter = 0
680 680
681 681 if fileOk_flag:
682 682 self.fileSize = os.path.getsize( fullfilename )
683 683 self.filename = fullfilename
684 684 self.flagIsNewFile = 1
685 685 if self.fp != None: self.fp.close()
686 686 self.fp = open(fullfilename, 'rb')
687 687 self.flagNoMoreFiles = 0
688 688 print 'Setting the file: %s' % fullfilename
689 689 else:
690 690 self.fileSize = 0
691 691 self.filename = None
692 692 self.flagIsNewFile = 0
693 693 self.fp = None
694 694 self.flagNoMoreFiles = 1
695 695 print 'No more Files'
696 696
697 697 return fileOk_flag
698 698
699 699 def setNextFile(self):
700 700 if self.fp != None:
701 701 self.fp.close()
702 702
703 703 if self.online:
704 704 newFile = self.__setNextFileOnline()
705 705 else:
706 706 newFile = self.__setNextFileOffline()
707 707
708 708 if not(newFile):
709 709 return 0
710 710
711 711 self.__readFirstHeader()
712 712 self.nReadBlocks = 0
713 713 return 1
714 714
715 715 def __waitNewBlock(self):
716 716 """
717 717 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
718 718
719 719 Si el modo de lectura es OffLine siempre retorn 0
720 720 """
721 721 if not self.online:
722 722 return 0
723 723
724 724 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
725 725 return 0
726 726
727 727 currentPointer = self.fp.tell()
728 728
729 729 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
730 730
731 731 for nTries in range( self.nTries ):
732 732
733 733 self.fp.close()
734 734 self.fp = open( self.filename, 'rb' )
735 735 self.fp.seek( currentPointer )
736 736
737 737 self.fileSize = os.path.getsize( self.filename )
738 738 currentSize = self.fileSize - currentPointer
739 739
740 740 if ( currentSize >= neededSize ):
741 741 self.basicHeaderObj.read(self.fp)
742 742 return 1
743 743
744 744 if self.fileSize == self.fileSizeByHeader:
745 745 # self.flagEoF = True
746 746 return 0
747 747
748 748 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
749 749 time.sleep( self.delay )
750 750
751 751
752 752 return 0
753 753
754 754 def waitDataBlock(self,pointer_location):
755 755
756 756 currentPointer = pointer_location
757 757
758 758 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
759 759
760 760 for nTries in range( self.nTries ):
761 761 self.fp.close()
762 762 self.fp = open( self.filename, 'rb' )
763 763 self.fp.seek( currentPointer )
764 764
765 765 self.fileSize = os.path.getsize( self.filename )
766 766 currentSize = self.fileSize - currentPointer
767 767
768 768 if ( currentSize >= neededSize ):
769 769 return 1
770 770
771 771 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
772 772 time.sleep( self.delay )
773 773
774 774 return 0
775 775
776 776 def __jumpToLastBlock(self):
777 777
778 778 if not(self.__isFirstTimeOnline):
779 779 return
780 780
781 781 csize = self.fileSize - self.fp.tell()
782 782 blocksize = self.processingHeaderObj.blockSize
783 783
784 784 #salta el primer bloque de datos
785 785 if csize > self.processingHeaderObj.blockSize:
786 786 self.fp.seek(self.fp.tell() + blocksize)
787 787 else:
788 788 return
789 789
790 790 csize = self.fileSize - self.fp.tell()
791 791 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
792 792 while True:
793 793
794 794 if self.fp.tell()<self.fileSize:
795 795 self.fp.seek(self.fp.tell() + neededsize)
796 796 else:
797 797 self.fp.seek(self.fp.tell() - neededsize)
798 798 break
799 799
800 800 # csize = self.fileSize - self.fp.tell()
801 801 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
802 802 # factor = int(csize/neededsize)
803 803 # if factor > 0:
804 804 # self.fp.seek(self.fp.tell() + factor*neededsize)
805 805
806 806 self.flagIsNewFile = 0
807 807 self.__isFirstTimeOnline = 0
808 808
809 809 def __setNewBlock(self):
810 810
811 811 if self.fp == None:
812 812 return 0
813 813
814 814 if self.online:
815 815 self.__jumpToLastBlock()
816 816
817 817 if self.flagIsNewFile:
818 818 return 1
819 819
820 820 self.lastUTTime = self.basicHeaderObj.utc
821 821 currentSize = self.fileSize - self.fp.tell()
822 822 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
823 823
824 824 if (currentSize >= neededSize):
825 825 self.basicHeaderObj.read(self.fp)
826 826 return 1
827 827
828 828 if self.__waitNewBlock():
829 829 return 1
830 830
831 831 if not(self.setNextFile()):
832 832 return 0
833 833
834 834 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
835 835
836 836 self.flagTimeBlock = 0
837 837
838 838 if deltaTime > self.maxTimeStep:
839 839 self.flagTimeBlock = 1
840 840
841 841 return 1
842 842
843 843 def readNextBlock(self):
844 844 if not(self.__setNewBlock()):
845 845 return 0
846 846
847 847 if not(self.readBlock()):
848 848 return 0
849 849
850 850 return 1
851 851
852 852 def __readFirstHeader(self):
853 853
854 854 self.basicHeaderObj.read(self.fp)
855 855 self.systemHeaderObj.read(self.fp)
856 856 self.radarControllerHeaderObj.read(self.fp)
857 857 self.processingHeaderObj.read(self.fp)
858 858
859 859 self.firstHeaderSize = self.basicHeaderObj.size
860 860
861 861 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
862 862 if datatype == 0:
863 863 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
864 864 elif datatype == 1:
865 865 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
866 866 elif datatype == 2:
867 867 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
868 868 elif datatype == 3:
869 869 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
870 870 elif datatype == 4:
871 871 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
872 872 elif datatype == 5:
873 873 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
874 874 else:
875 875 raise ValueError, 'Data type was not defined'
876 876
877 877 self.dtype = datatype_str
878 878 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
879 879 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
880 880 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
881 881 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
882 882 self.getBlockDimension()
883 883
884 884 def __verifyFile(self, filename, msgFlag=True):
885 885 msg = None
886 886 try:
887 887 fp = open(filename, 'rb')
888 888 currentPosition = fp.tell()
889 889 except IOError:
890 890 traceback.print_exc()
891 891 if msgFlag:
892 892 print "The file %s can't be opened" % (filename)
893 893 return False
894 894
895 895 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
896 896
897 897 if neededSize == 0:
898 898 basicHeaderObj = BasicHeader(LOCALTIME)
899 899 systemHeaderObj = SystemHeader()
900 900 radarControllerHeaderObj = RadarControllerHeader()
901 901 processingHeaderObj = ProcessingHeader()
902 902
903 903 try:
904 904 if not( basicHeaderObj.read(fp) ): raise IOError
905 905 if not( systemHeaderObj.read(fp) ): raise IOError
906 906 if not( radarControllerHeaderObj.read(fp) ): raise IOError
907 907 if not( processingHeaderObj.read(fp) ): raise IOError
908 908 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
909 909
910 910 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
911 911
912 912 except IOError:
913 913 traceback.print_exc()
914 914 if msgFlag:
915 915 print "\tThe file %s is empty or it hasn't enough data" % filename
916 916
917 917 fp.close()
918 918 return False
919 919 else:
920 920 msg = "\tSkipping the file %s due to it hasn't enough data" %filename
921 921
922 922 fp.close()
923 923 fileSize = os.path.getsize(filename)
924 924 currentSize = fileSize - currentPosition
925 925 if currentSize < neededSize:
926 926 if msgFlag and (msg != None):
927 927 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
928 928 return False
929 929
930 930 return True
931 931
932 932 def setup(self,
933 933 path=None,
934 934 startDate=None,
935 935 endDate=None,
936 936 startTime=datetime.time(0,0,0),
937 937 endTime=datetime.time(23,59,59),
938 938 set=None,
939 939 expLabel = "",
940 940 ext = None,
941 941 online = False,
942 942 delay = 60,
943 943 walk = True,
944 944 getblock = False,
945 945 nTxs = 1):
946 946
947 947 if path == None:
948 948 raise ValueError, "The path is not valid"
949 949
950 950 if ext == None:
951 951 ext = self.ext
952 952
953 953 if online:
954 954 print "Searching files in online mode..."
955 955
956 956 for nTries in range( self.nTries ):
957 957 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
958 958
959 959 if fullpath:
960 960 break
961 961
962 962 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
963 963 time.sleep( self.delay )
964 964
965 965 if not(fullpath):
966 966 print "There 'isn't valied files in %s" % path
967 967 return None
968 968
969 969 self.year = year
970 970 self.doy = doy
971 971 self.set = set - 1
972 972 self.path = path
973 973 self.foldercounter = foldercounter
974 974 last_set = None
975 975
976 976 else:
977 977 print "Searching files in offline mode ..."
978 978 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
979 979 startTime=startTime, endTime=endTime,
980 980 set=set, expLabel=expLabel, ext=ext,
981 981 walk=walk)
982 982
983 983 if not(pathList):
984 984 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
985 985 datetime.datetime.combine(startDate,startTime).ctime(),
986 986 datetime.datetime.combine(endDate,endTime).ctime())
987 987
988 988 sys.exit(-1)
989 989
990 990
991 991 self.fileIndex = -1
992 992 self.pathList = pathList
993 993 self.filenameList = filenameList
994 994 file_name = os.path.basename(filenameList[-1])
995 995 basename, ext = os.path.splitext(file_name)
996 996 last_set = int(basename[-3:])
997 997
998 998 self.online = online
999 999 self.delay = delay
1000 1000 ext = ext.lower()
1001 1001 self.ext = ext
1002 1002 self.getByBlock = getblock
1003 1003 self.nTxs = int(nTxs)
1004 1004
1005 1005 if not(self.setNextFile()):
1006 1006 if (startDate!=None) and (endDate!=None):
1007 1007 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1008 1008 elif startDate != None:
1009 1009 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1010 1010 else:
1011 1011 print "No files"
1012 1012
1013 1013 sys.exit(-1)
1014 1014
1015 1015 # self.updateDataHeader()
1016 1016 if last_set != None:
1017 1017 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1018 1018 return
1019 1019
1020 1020 def getBasicHeader(self):
1021 1021
1022 1022 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1023 1023
1024 1024 self.dataOut.flagTimeBlock = self.flagTimeBlock
1025 1025
1026 1026 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1027 1027
1028 1028 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1029 1029
1030 1030 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1031 1031
1032 1032 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1033 1033
1034 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1035
1036 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1037
1038
1034 1039 def getFirstHeader(self):
1035 1040
1036 1041 raise ValueError, "This method has not been implemented"
1037 1042
1038 1043 def getData(self):
1039 1044
1040 1045 raise ValueError, "This method has not been implemented"
1041 1046
1042 1047 def hasNotDataInBuffer(self):
1043 1048
1044 1049 raise ValueError, "This method has not been implemented"
1045 1050
1046 1051 def readBlock(self):
1047 1052
1048 1053 raise ValueError, "This method has not been implemented"
1049 1054
1050 1055 def isEndProcess(self):
1051 1056
1052 1057 return self.flagNoMoreFiles
1053 1058
1054 1059 def printReadBlocks(self):
1055 1060
1056 1061 print "Number of read blocks per file %04d" %self.nReadBlocks
1057 1062
1058 1063 def printTotalBlocks(self):
1059 1064
1060 1065 print "Number of read blocks %04d" %self.nTotalBlocks
1061 1066
1062 1067 def printNumberOfBlock(self):
1063 1068
1064 1069 if self.flagIsNewBlock:
1065 1070 print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
1066 1071 self.dataOut.blocknow = self.basicHeaderObj.dataBlock
1067 1072
1068 1073 def printInfo(self):
1069 1074
1070 1075 if self.__printInfo == False:
1071 1076 return
1072 1077
1073 1078 self.basicHeaderObj.printInfo()
1074 1079 self.systemHeaderObj.printInfo()
1075 1080 self.radarControllerHeaderObj.printInfo()
1076 1081 self.processingHeaderObj.printInfo()
1077 1082
1078 1083 self.__printInfo = False
1079 1084
1080 1085
1081 1086 def run(self, **kwargs):
1082 1087
1083 1088 if not(self.isConfig):
1084 1089
1085 1090 # self.dataOut = dataOut
1086 1091 self.setup(**kwargs)
1087 1092 self.isConfig = True
1088 1093
1089 1094 self.getData()
1090 1095
1091 1096 class JRODataWriter(JRODataIO):
1092 1097
1093 1098 """
1094 1099 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1095 1100 de los datos siempre se realiza por bloques.
1096 1101 """
1097 1102
1098 1103 blockIndex = 0
1099 1104
1100 1105 path = None
1101 1106
1102 1107 setFile = None
1103 1108
1104 1109 profilesPerBlock = None
1105 1110
1106 1111 blocksPerFile = None
1107 1112
1108 1113 nWriteBlocks = 0
1109 1114
1110 1115 def __init__(self, dataOut=None):
1111 1116 raise ValueError, "Not implemented"
1112 1117
1113 1118
1114 1119 def hasAllDataInBuffer(self):
1115 1120 raise ValueError, "Not implemented"
1116 1121
1117 1122
1118 1123 def setBlockDimension(self):
1119 1124 raise ValueError, "Not implemented"
1120 1125
1121 1126
1122 1127 def writeBlock(self):
1123 1128 raise ValueError, "No implemented"
1124 1129
1125 1130
1126 1131 def putData(self):
1127 1132 raise ValueError, "No implemented"
1128 1133
1129 1134
1130 1135 def setBasicHeader(self):
1131 1136
1132 1137 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1133 1138 self.basicHeaderObj.version = self.versionFile
1134 1139 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1135 1140
1136 1141 utc = numpy.floor(self.dataOut.utctime)
1137 1142 milisecond = (self.dataOut.utctime - utc)* 1000.0
1138 1143
1139 1144 self.basicHeaderObj.utc = utc
1140 1145 self.basicHeaderObj.miliSecond = milisecond
1141 1146 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1142 1147 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1143 1148 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1144 1149
1145 1150 def setFirstHeader(self):
1146 1151 """
1147 1152 Obtiene una copia del First Header
1148 1153
1149 1154 Affected:
1150 1155
1151 1156 self.basicHeaderObj
1152 1157 self.systemHeaderObj
1153 1158 self.radarControllerHeaderObj
1154 1159 self.processingHeaderObj self.
1155 1160
1156 1161 Return:
1157 1162 None
1158 1163 """
1159 1164
1160 1165 raise ValueError, "No implemented"
1161 1166
1162 1167 def __writeFirstHeader(self):
1163 1168 """
1164 1169 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1165 1170
1166 1171 Affected:
1167 1172 __dataType
1168 1173
1169 1174 Return:
1170 1175 None
1171 1176 """
1172 1177
1173 1178 # CALCULAR PARAMETROS
1174 1179
1175 1180 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1176 1181 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1177 1182
1178 1183 self.basicHeaderObj.write(self.fp)
1179 1184 self.systemHeaderObj.write(self.fp)
1180 1185 self.radarControllerHeaderObj.write(self.fp)
1181 1186 self.processingHeaderObj.write(self.fp)
1182 1187
1183 1188 self.dtype = self.dataOut.dtype
1184 1189
1185 1190 def __setNewBlock(self):
1186 1191 """
1187 1192 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1188 1193
1189 1194 Return:
1190 1195 0 : si no pudo escribir nada
1191 1196 1 : Si escribio el Basic el First Header
1192 1197 """
1193 1198 if self.fp == None:
1194 1199 self.setNextFile()
1195 1200
1196 1201 if self.flagIsNewFile:
1197 1202 return 1
1198 1203
1199 1204 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1200 1205 self.basicHeaderObj.write(self.fp)
1201 1206 return 1
1202 1207
1203 1208 if not( self.setNextFile() ):
1204 1209 return 0
1205 1210
1206 1211 return 1
1207 1212
1208 1213
1209 1214 def writeNextBlock(self):
1210 1215 """
1211 1216 Selecciona el bloque siguiente de datos y los escribe en un file
1212 1217
1213 1218 Return:
1214 1219 0 : Si no hizo pudo escribir el bloque de datos
1215 1220 1 : Si no pudo escribir el bloque de datos
1216 1221 """
1217 1222 if not( self.__setNewBlock() ):
1218 1223 return 0
1219 1224
1220 1225 self.writeBlock()
1221 1226
1222 1227 return 1
1223 1228
1224 1229 def setNextFile(self):
1225 1230 """
1226 1231 Determina el siguiente file que sera escrito
1227 1232
1228 1233 Affected:
1229 1234 self.filename
1230 1235 self.subfolder
1231 1236 self.fp
1232 1237 self.setFile
1233 1238 self.flagIsNewFile
1234 1239
1235 1240 Return:
1236 1241 0 : Si el archivo no puede ser escrito
1237 1242 1 : Si el archivo esta listo para ser escrito
1238 1243 """
1239 1244 ext = self.ext
1240 1245 path = self.path
1241 1246
1242 1247 if self.fp != None:
1243 1248 self.fp.close()
1244 1249
1245 1250 timeTuple = time.localtime( self.dataOut.utctime)
1246 1251 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1247 1252
1248 1253 fullpath = os.path.join( path, subfolder )
1249 1254 if not( os.path.exists(fullpath) ):
1250 1255 os.mkdir(fullpath)
1251 1256 self.setFile = -1 #inicializo mi contador de seteo
1252 1257 else:
1253 1258 filesList = os.listdir( fullpath )
1254 1259 if len( filesList ) > 0:
1255 1260 filesList = sorted( filesList, key=str.lower )
1256 1261 filen = filesList[-1]
1257 1262 # el filename debera tener el siguiente formato
1258 1263 # 0 1234 567 89A BCDE (hex)
1259 1264 # x YYYY DDD SSS .ext
1260 1265 if isNumber( filen[8:11] ):
1261 1266 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1262 1267 else:
1263 1268 self.setFile = -1
1264 1269 else:
1265 1270 self.setFile = -1 #inicializo mi contador de seteo
1266 1271
1267 1272 setFile = self.setFile
1268 1273 setFile += 1
1269 1274
1270 1275 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
1271 1276 timeTuple.tm_year,
1272 1277 timeTuple.tm_yday,
1273 1278 setFile,
1274 1279 ext )
1275 1280
1276 1281 filename = os.path.join( path, subfolder, file )
1277 1282
1278 1283 fp = open( filename,'wb' )
1279 1284
1280 1285 self.blockIndex = 0
1281 1286
1282 1287 #guardando atributos
1283 1288 self.filename = filename
1284 1289 self.subfolder = subfolder
1285 1290 self.fp = fp
1286 1291 self.setFile = setFile
1287 1292 self.flagIsNewFile = 1
1288 1293
1289 1294 self.setFirstHeader()
1290 1295
1291 1296 print 'Writing the file: %s'%self.filename
1292 1297
1293 1298 self.__writeFirstHeader()
1294 1299
1295 1300 return 1
1296 1301
1297 1302 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
1298 1303 """
1299 1304 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1300 1305
1301 1306 Inputs:
1302 1307 path : el path destino en el cual se escribiran los files a crear
1303 1308 format : formato en el cual sera salvado un file
1304 1309 set : el setebo del file
1305 1310
1306 1311 Return:
1307 1312 0 : Si no realizo un buen seteo
1308 1313 1 : Si realizo un buen seteo
1309 1314 """
1310 1315
1311 1316 if ext == None:
1312 1317 ext = self.ext
1313 1318
1314 1319 ext = ext.lower()
1315 1320
1316 1321 self.ext = ext
1317 1322
1318 1323 self.path = path
1319 1324
1320 1325 self.setFile = set - 1
1321 1326
1322 1327 self.blocksPerFile = blocksPerFile
1323 1328
1324 1329 self.profilesPerBlock = profilesPerBlock
1325 1330
1326 1331 self.dataOut = dataOut
1327 1332
1328 1333 if not(self.setNextFile()):
1329 1334 print "There isn't a next file"
1330 1335 return 0
1331 1336
1332 1337 self.setBlockDimension()
1333 1338
1334 1339 return 1
1335 1340
1336 1341 def run(self, dataOut, **kwargs):
1337 1342
1338 1343 if not(self.isConfig):
1339 1344
1340 1345 self.setup(dataOut, **kwargs)
1341 1346 self.isConfig = True
1342 1347
1343 1348 self.putData()
1344 1349
General Comments 0
You need to be logged in to leave comments. Login now