##// END OF EJS Templates
schain-jc
RSS
ONLINE & SET IS READY
Alexander Valdez -
r552:7f199d9437df
parent child
Show More
Show file before | Show file after | Hide comments
@@ -8,6 +8,7 import os,sys
8 import time,datetime
8 import time,datetime
9 import h5py
9 import h5py
10 import numpy
10 import numpy
11 import fnmatch
11 import re
12 import re
12 import stuffr
13 import stuffr
13
14
@@ -55,13 +56,13 def getFileFromSet(path, ext, set):
55 continue
56 continue
56
57
57 validFilelist.append(thisFile)
58 validFilelist.append(thisFile)
58 myfile = fnmatch.filter(validFilelist,'*%6.6d*'%(number))
59 myfile = fnmatch.filter(validFilelist,'*%6.6d*'%(set))
59 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
60 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
60
61
61 if len(myfile)!= 0:
62 if len(myfile)!= 0:
62 return myfile[0]
63 return myfile[0]
63 else:
64 else:
64 filename = '*%6.6d%s'%(number,ext.lower())
65 filename = '*%6.6d%s'%(set,ext.lower())
65 print 'the filename %s does not exist'%filename
66 print 'the filename %s does not exist'%filename
66 print '...going to the last file: '
67 print '...going to the last file: '
67
68
@@ -150,6 +151,8 class HFReader(ProcessingUnit):
150
151
151 self.datablock = None
152 self.datablock = None
152
153
154 self.filename_current=None
155
153 self.utc = 0
156 self.utc = 0
154
157
155 self.ext='.hdf5'
158 self.ext='.hdf5'
@@ -171,7 +174,7 class HFReader(ProcessingUnit):
171
174
172 self.flagNoMoreFiles= False
175 self.flagNoMoreFiles= False
173
176
174 self.__waitForNewFile = 3
177 self.__waitForNewFile = 1
175
178
176
179
177 #--------------------------------------------------
180 #--------------------------------------------------
@@ -215,6 +218,7 class HFReader(ProcessingUnit):
215 #self.__frequency=3.64e6
218 #self.__frequency=3.64e6
216 self.__frequency=None
219 self.__frequency=None
217 self.__online = False
220 self.__online = False
221 self.filename_next_set=None
218
222
219 #print "Frequency of Operation:", self.__frequency
223 #print "Frequency of Operation:", self.__frequency
220
224
@@ -307,6 +311,8 class HFReader(ProcessingUnit):
307 #print self.dirnameList
311 #print self.dirnameList
308 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
312 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
309 self.filenameList= dirList
313 self.filenameList= dirList
314 #print self.filenameList
315 #print "pase",len(self.filenameList)
310
316
311 def __selectDataForTimes(self, online=False):
317 def __selectDataForTimes(self, online=False):
312
318
@@ -325,14 +331,42 class HFReader(ProcessingUnit):
325 self.status=0
331 self.status=0
326 return None
332 return None
327 else:
333 else:
328 #if self.set== None:
334 if self.set== None:
329 self.filenameList=[self.filenameList[-1]]
335 filename =getlastFileFromPath(self.path,self.ext)
330 #else:
336
331 # try:
337 if self.flag_nextfile==True:
332 # filename=getFileFromSet(self.path,self.ext,self.set)
338 self.dirnameList=[filename]
333 # self.filenameList=self.path+"/"+filename
339 fullfilename=self.path+"/"+filename
334 # except:
340 self.filenameList=[self.filenameList[-1]]
335 # self.filenameList=[self.filenameList[-1]]
341 self.filename_next_set=int(filename[4:10])+1
342
343 self.flag_nextfile=False
344 else:
345 if self.filename_next_set== int(filename[4:10]):
346 self.dirnameList=[filename]
347 fullfilename=self.path+"/"+filename
348 self.filenameList=[self.filenameList[-1]]
349 self.filename_next_set=int(filename[4:10])+1
350
351 else:
352 set=self.filename_next_set
353 filename=getFileFromSet(self.path,self.ext,set=set)
354 self.filename_next_set=int(filename[4:10])+1
355 if filename==None:
356 filename =getlastFileFromPath(self.path,self.ext)
357 self.filename_next_set=int(filename[4:10])
358 self.dirnameList=[filename]
359 fullfilename=self.path+"/"+filename
360 self.filenameList=[self.filenameList[-1]]
361
362 else:
363 try:
364 filename=getFileFromSet(self.path,self.ext,self.set)
365 self.dirnameList=[filename]
366 fullfilename=self.path+"/"+filename
367 self.filenameList=[fullfilename]
368 except:
369 self.filenameList=[self.filenameList[-1]]
336
370
337
371
338 def __searchFilesOffline(self,
372 def __searchFilesOffline(self,
@@ -372,26 +406,27 class HFReader(ProcessingUnit):
372 endDate = datetime.datetime.utcnow().date()
406 endDate = datetime.datetime.utcnow().date()
373
407
374 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
408 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
375
409
376 self.__checkPath()
410 self.__checkPath()
377
411
378 # fullpath=path
412 fullpath=path
379 #
413 print "%s folder was found: " %(fullpath )
380 # print "%s folder was found: " %(fullpath )
414
381 #
415 if set == None:
382 # if set == None:
416 self.set=None
383 # filename =getlastFileFromPath(fullpath,ext)
417 filename =getlastFileFromPath(fullpath,ext)
384 # startDate= datetime.datetime.utcnow().date
418 startDate= datetime.datetime.utcnow().date
385 # endDate= datetime.datetime.utcnow().date()
419 endDate= datetime.datetime.utcnow().date()
386 #
420 #
387 # else:
421 else:
388 # filename= getFileFromSet(fullpath,ext,set)
422 filename= getFileFromSet(fullpath,ext,set)
389 # startDate=None
423 startDate=None
390 # endDate=None
424 endDate=None
391 #
425 #
392 # if not (filename):
426 if not (filename):
393 # return None,None,None,None,None
427 return None,None,None,None,None
394 # print "%s file was found" %(filename)
428 #print "%s file was found" %(filename)
429
395 #
430 #
396 # dir_hf_filename= self.path+"/"+filename
431 # dir_hf_filename= self.path+"/"+filename
397 # fp= h5py.File(dir_hf_filename,'r')
432 # fp= h5py.File(dir_hf_filename,'r')
@@ -405,10 +440,15 class HFReader(ProcessingUnit):
405 # set= int(filename[4:10])
440 # set= int(filename[4:10])
406 # self.set=set-1
441 # self.set=set-1
407 #self.dirnameList=[filename]
442 #self.dirnameList=[filename]
408 self.__findDataForDates(online=True)
443 filenameList= fullpath+"/"+filename
409 self.dirnameList=[self.dirnameList[-1]]
444 self.dirnameList=[filename]
445 self.filenameList=[filenameList]
446 self.flag_nextfile=True
447
448 #self.__findDataForDates(online=True)
449 #self.dirnameList=[self.dirnameList[-1]]
410 #print self.dirnameList
450 #print self.dirnameList
411 self.__selectDataForTimes(online=True)
451 #self.__selectDataForTimes(online=True)
412 #return fullpath,filename,year,month,dom,set
452 #return fullpath,filename,year,month,dom,set
413 return
453 return
414
454
@@ -454,20 +494,45 class HFReader(ProcessingUnit):
454
494
455 def __setNextFileOnline(self):
495 def __setNextFileOnline(self):
456 """
496 """
457 """
497 """
458 filename = self.filenameList[0]
498 if self.set==None:
499 pass
500 else:
501 self.set +=1
502
503 filename = self.filenameList[0]#fullfilename
459 if self.filename_online != None:
504 if self.filename_online != None:
460 self.__selectDataForTimes(online=True)
505 self.__selectDataForTimes(online=True)
461 filename = self.filenameList[0]
506 filename = self.filenameList[0]
462 while self.filename_online == filename:
507 while self.filename_online == filename:
463 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
508 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
464 time.sleep(self.__waitForNewFile)
509 time.sleep(self.__waitForNewFile)
465 self.__findDataForDates(True)
510 #self.__findDataForDates(online=True)
466 self.__selectDataForTimes(online=True)
511 self.__selectDataForTimes(online=True)
467 filename = self.filenameList[0]
512 filename = self.filenameList[0]
468 print filename
513 sizeoffile=os.path.getsize(filename)
514
515 #print filename
516 sizeoffile=os.path.getsize(filename)
517 if sizeoffile<1650240:
518 print "%s is not the rigth size"%filename
519 delay=2
520 print 'waiting %d seconds for delay...'%(delay)
521 time.sleep(delay)
522 sizeoffile=os.path.getsize(filename)
523 if sizeoffile<1650240:
524 delay
525 print 'waiting %d more seconds for delay...'%(delay)
526 time.sleep(delay)
527
528
469
529
470 hfFilePointer=h5py.File(filename,'r')
530 try:
531 hfFilePointer=h5py.File(filename,'r')
532
533 except:
534 print "Error reading file %s"%filename
535
471 self.filename_online=filename
536 self.filename_online=filename
472 epoc=hfFilePointer['t'].value
537 epoc=hfFilePointer['t'].value
473
538
@@ -513,15 +578,20 class HFReader(ProcessingUnit):
513 self.timezone= timezone
578 self.timezone= timezone
514 self.online= online
579 self.online= online
515 self.all=all
580 self.all=all
516
581 #if set==None:
517
582
583 #print set
518 if not(online):
584 if not(online):
519 print "Searching files in offline mode..."
585 print "Searching files in offline mode..."
520
586
521 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
587 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
522 else:
588 else:
523 print "Searching files in online mode..."
589 print "Searching files in online mode..."
524 self.__searchFilesOnline(path, walk)
590 self.__searchFilesOnline(path, walk,ext,set=set)
591 if set==None:
592 pass
593 else:
594 self.set=set-1
525
595
526 # for nTries in range(self.nTries):
596 # for nTries in range(self.nTries):
527 #
597 #
@@ -542,6 +612,7 class HFReader(ProcessingUnit):
542
612
543 self.__getExpParameters()
613 self.__getExpParameters()
544
614
615
545 self.fileIndex = -1
616 self.fileIndex = -1
546
617
547 self.__setNextFile(online)
618 self.__setNextFile(online)
General Comments 0
You need to be logged in to leave comments. Login now