##// END OF EJS Templates
schain-jc
RSS
ONLINE & SET IS READY
Alexander Valdez -
r552:7f199d9437df
parent child
Show More
Show file before | Show file after | Hide comments
@@ -8,6 +8,7 import os,sys
8 8 import time,datetime
9 9 import h5py
10 10 import numpy
11 import fnmatch
11 12 import re
12 13 import stuffr
13 14
@@ -55,13 +56,13 def getFileFromSet(path, ext, set):
55 56 continue
56 57
57 58 validFilelist.append(thisFile)
58 myfile = fnmatch.filter(validFilelist,'*%6.6d*'%(number))
59 myfile = fnmatch.filter(validFilelist,'*%6.6d*'%(set))
59 60 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
60 61
61 62 if len(myfile)!= 0:
62 63 return myfile[0]
63 64 else:
64 filename = '*%6.6d%s'%(number,ext.lower())
65 filename = '*%6.6d%s'%(set,ext.lower())
65 66 print 'the filename %s does not exist'%filename
66 67 print '...going to the last file: '
67 68
@@ -150,6 +151,8 class HFReader(ProcessingUnit):
150 151
151 152 self.datablock = None
152 153
154 self.filename_current=None
155
153 156 self.utc = 0
154 157
155 158 self.ext='.hdf5'
@@ -171,7 +174,7 class HFReader(ProcessingUnit):
171 174
172 175 self.flagNoMoreFiles= False
173 176
174 self.__waitForNewFile = 3
177 self.__waitForNewFile = 1
175 178
176 179
177 180 #--------------------------------------------------
@@ -215,6 +218,7 class HFReader(ProcessingUnit):
215 218 #self.__frequency=3.64e6
216 219 self.__frequency=None
217 220 self.__online = False
221 self.filename_next_set=None
218 222
219 223 #print "Frequency of Operation:", self.__frequency
220 224
@@ -307,6 +311,8 class HFReader(ProcessingUnit):
307 311 #print self.dirnameList
308 312 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
309 313 self.filenameList= dirList
314 #print self.filenameList
315 #print "pase",len(self.filenameList)
310 316
311 317 def __selectDataForTimes(self, online=False):
312 318
@@ -325,14 +331,42 class HFReader(ProcessingUnit):
325 331 self.status=0
326 332 return None
327 333 else:
328 #if self.set== None:
329 self.filenameList=[self.filenameList[-1]]
330 #else:
331 # try:
332 # filename=getFileFromSet(self.path,self.ext,self.set)
333 # self.filenameList=self.path+"/"+filename
334 # except:
335 # self.filenameList=[self.filenameList[-1]]
334 if self.set== None:
335 filename =getlastFileFromPath(self.path,self.ext)
336
337 if self.flag_nextfile==True:
338 self.dirnameList=[filename]
339 fullfilename=self.path+"/"+filename
340 self.filenameList=[self.filenameList[-1]]
341 self.filename_next_set=int(filename[4:10])+1
342
343 self.flag_nextfile=False
344 else:
345 if self.filename_next_set== int(filename[4:10]):
346 self.dirnameList=[filename]
347 fullfilename=self.path+"/"+filename
348 self.filenameList=[self.filenameList[-1]]
349 self.filename_next_set=int(filename[4:10])+1
350
351 else:
352 set=self.filename_next_set
353 filename=getFileFromSet(self.path,self.ext,set=set)
354 self.filename_next_set=int(filename[4:10])+1
355 if filename==None:
356 filename =getlastFileFromPath(self.path,self.ext)
357 self.filename_next_set=int(filename[4:10])
358 self.dirnameList=[filename]
359 fullfilename=self.path+"/"+filename
360 self.filenameList=[self.filenameList[-1]]
361
362 else:
363 try:
364 filename=getFileFromSet(self.path,self.ext,self.set)
365 self.dirnameList=[filename]
366 fullfilename=self.path+"/"+filename
367 self.filenameList=[fullfilename]
368 except:
369 self.filenameList=[self.filenameList[-1]]
336 370
337 371
338 372 def __searchFilesOffline(self,
@@ -372,26 +406,27 class HFReader(ProcessingUnit):
372 406 endDate = datetime.datetime.utcnow().date()
373 407
374 408 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
375
409
376 410 self.__checkPath()
377 411
378 # fullpath=path
379 #
380 # print "%s folder was found: " %(fullpath )
381 #
382 # if set == None:
383 # filename =getlastFileFromPath(fullpath,ext)
384 # startDate= datetime.datetime.utcnow().date
385 # endDate= datetime.datetime.utcnow().date()
412 fullpath=path
413 print "%s folder was found: " %(fullpath )
414
415 if set == None:
416 self.set=None
417 filename =getlastFileFromPath(fullpath,ext)
418 startDate= datetime.datetime.utcnow().date
419 endDate= datetime.datetime.utcnow().date()
386 420 #
387 # else:
388 # filename= getFileFromSet(fullpath,ext,set)
389 # startDate=None
390 # endDate=None
421 else:
422 filename= getFileFromSet(fullpath,ext,set)
423 startDate=None
424 endDate=None
391 425 #
392 # if not (filename):
393 # return None,None,None,None,None
394 # print "%s file was found" %(filename)
426 if not (filename):
427 return None,None,None,None,None
428 #print "%s file was found" %(filename)
429
395 430 #
396 431 # dir_hf_filename= self.path+"/"+filename
397 432 # fp= h5py.File(dir_hf_filename,'r')
@@ -405,10 +440,15 class HFReader(ProcessingUnit):
405 440 # set= int(filename[4:10])
406 441 # self.set=set-1
407 442 #self.dirnameList=[filename]
408 self.__findDataForDates(online=True)
409 self.dirnameList=[self.dirnameList[-1]]
443 filenameList= fullpath+"/"+filename
444 self.dirnameList=[filename]
445 self.filenameList=[filenameList]
446 self.flag_nextfile=True
447
448 #self.__findDataForDates(online=True)
449 #self.dirnameList=[self.dirnameList[-1]]
410 450 #print self.dirnameList
411 self.__selectDataForTimes(online=True)
451 #self.__selectDataForTimes(online=True)
412 452 #return fullpath,filename,year,month,dom,set
413 453 return
414 454
@@ -454,20 +494,45 class HFReader(ProcessingUnit):
454 494
455 495 def __setNextFileOnline(self):
456 496 """
457 """
458 filename = self.filenameList[0]
497 """
498 if self.set==None:
499 pass
500 else:
501 self.set +=1
502
503 filename = self.filenameList[0]#fullfilename
459 504 if self.filename_online != None:
460 505 self.__selectDataForTimes(online=True)
461 506 filename = self.filenameList[0]
462 507 while self.filename_online == filename:
463 508 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
464 509 time.sleep(self.__waitForNewFile)
465 self.__findDataForDates(True)
510 #self.__findDataForDates(online=True)
466 511 self.__selectDataForTimes(online=True)
467 512 filename = self.filenameList[0]
468 print filename
513 sizeoffile=os.path.getsize(filename)
514
515 #print filename
516 sizeoffile=os.path.getsize(filename)
517 if sizeoffile<1650240:
518 print "%s is not the rigth size"%filename
519 delay=2
520 print 'waiting %d seconds for delay...'%(delay)
521 time.sleep(delay)
522 sizeoffile=os.path.getsize(filename)
523 if sizeoffile<1650240:
524 delay
525 print 'waiting %d more seconds for delay...'%(delay)
526 time.sleep(delay)
527
528
469 529
470 hfFilePointer=h5py.File(filename,'r')
530 try:
531 hfFilePointer=h5py.File(filename,'r')
532
533 except:
534 print "Error reading file %s"%filename
535
471 536 self.filename_online=filename
472 537 epoc=hfFilePointer['t'].value
473 538
@@ -513,15 +578,20 class HFReader(ProcessingUnit):
513 578 self.timezone= timezone
514 579 self.online= online
515 580 self.all=all
516
517
581 #if set==None:
582
583 #print set
518 584 if not(online):
519 585 print "Searching files in offline mode..."
520 586
521 587 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
522 588 else:
523 589 print "Searching files in online mode..."
524 self.__searchFilesOnline(path, walk)
590 self.__searchFilesOnline(path, walk,ext,set=set)
591 if set==None:
592 pass
593 else:
594 self.set=set-1
525 595
526 596 # for nTries in range(self.nTries):
527 597 #
@@ -542,6 +612,7 class HFReader(ProcessingUnit):
542 612
543 613 self.__getExpParameters()
544 614
615
545 616 self.fileIndex = -1
546 617
547 618 self.__setNextFile(online)
General Comments 0
You need to be logged in to leave comments. Login now