''' Created on Jul 3, 2014 @author: roj-com0419 ''' import os import time,datetime import os import h5py import re import stuffr from model.data.jrodata import * from model.proc.jroproc_base import ProcessingUnit, Operation class HFReader(ProcessingUnit): ''' classdocs ''' path = None startDate= None endDate = None startTime= None endTime = None walk = None isConfig = False dataOut=None ext = ".hdf5" def __init__(self): ''' Constructor ''' ProcessingUnit.__init__(self) self.isConfig =False self.datablock = None self.utc = 0 self.ext='.hdf5' self.flagIsNewFile = 1 #------------------------------------------------- self.fileIndex=None self.profileIndex_offset=None self.filenameList=[] self.hfFilePointer= None self.__filename_online = None self.status=True self.flagNoMoreFiles= False #-------------------------------------------------- self.dataOut = self.createObjByDefault() def createObjByDefault(self): dataObj = Voltage() return dataObj def setObjProperties(self): pass def getBlockDimension(self): """ Obtiene la cantidad de puntos a leer por cada bloque de datos Affected: self.blocksize Return: None """ pts2read =self.nChannels*self.nHeights*self.nProfiles self.blocksize = pts2read def __readHeader(self): self.nProfiles = 100 self.nHeights = 1000 self.nChannels = 2 self.__firstHeigth=0 self.__nSamples=1000 self.__deltaHeigth=1.5 self.__sample_rate=10e5 self.__frequency=2.72e6 self.__online = False def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''): self.path = path self.startDate = startDate self.endDate = endDate self.startTime = startTime self.endTime = endTime self.walk = walk def __checkPath(self): if os.path.exists(self.path): self.status=1 else: self.status=0 print 'Path %s does not exits'%self.path return def __selDates(self, hf_dirname_format): try: dir_hf_filename= self.path+"/"+hf_dirname_format fp= h5py.File(dir_hf_filename,'r') hipoc=fp['t'].value date_time=stuffr.unix2datestr(hipoc) year =int(date_time[0:4]) month=int(date_time[5:7]) dom =int(date_time[8:10]) thisDate= datetime.date(year,month,dom) if (thisDate>=self.startDate and thisDate <= self.endDate): return hf_dirname_format except: return None def __findDataForDates(self,online=False): if not(self.status): return None pat = '\d+.\d+' dirnameList = [re.search(pat,x) for x in os.listdir(self.path)] dirnameList = filter(lambda x:x!=None,dirnameList) dirnameList = [x.string for x in dirnameList] if not(online): dirnameList = [self.__selDates(x) for x in dirnameList] dirnameList = filter(lambda x:x!=None,dirnameList) if len(dirnameList)>0: self.status = 1 self.dirnameList = dirnameList self.dirnameList.sort() else: self.status = 0 return None def __getTimeFromData(self): startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime) endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime) print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader) print '........................................' filter_filenameList=[] self.filenameList.sort() for i in range(len(self.filenameList)-1): filename=self.filenameList[i] dir_hf_filename= filename fp= h5py.File(dir_hf_filename,'r') hipoc=fp['t'].value date_time=stuffr.unix2datestr(hipoc) year =int(date_time[0:4]) month=int(date_time[5:7]) dom =int(date_time[8:10]) hour =int(date_time[11:13]) min =int(date_time[14:16]) sec =int(date_time[17:19]) this_time=datetime.datetime(year,month,dom,hour,min,sec) if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader): filter_filenameList.append(filename) filter_filenameList.sort() self.filenameList = filter_filenameList return 1 def __getFilenameList(self): dirList = [os.path.join(self.path,x) for x in self.dirnameList] self.filenameList= dirList def __selectDataForTimes(self, online=False): if not(self.status): return none self.__getFilenameList() if not(online): if not(self.all): self.__getTimeFromData() if len(self.filenameList)>0: self.status=1 self.filenameList.sort() else: self.status=0 return None else: self.filenameList=[self.filenameList[-2]] def __searchFilesOffline(self, path, startDate, endDate, ext, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), walk=True): self.__setParameters(path, startDate, endDate, startTime, endTime, walk) self.__checkPath() self.__findDataForDates() self.__selectDataForTimes() for i in range(len(self.filenameList)): print "%s"% (self.filenameList[i]) return def __searchFilesOnline(self, path, walk=True): startDate= datetime.datetime.utcnow().date endDate= datetime.datetime.utcnow().date() self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk) self.__checkPath() self.__findDataForDates(online=True) self.dirnameList= [self.dirnameList[-1]] self.__selectDataForTimes(online=True) return def __setNextFile(self,online=False): """ """ if not(online): newFile = self.__setNextFileOffline() else: newFile = self.__setNextFileOnline() if not(newFile): return 0 return 1 def __setNextFileOffline(self): """ """ idFile= self.fileIndex while(True): idFile += 1 if not (idFile < len(self.filenameList)): self.flagNoMoreFiles = 1 print "No more Files" return 0 filename = self.filenameList[idFile] hfFilePointer =h5py.File(filename,'r') epoc=hfFilePointer['t'].value #this_time=datetime.datetime(year,month,dom,hour,min,sec) break self.flagIsNewFile = 1 self.fileIndex = idFile self.filename = filename self.hfFilePointer = hfFilePointer self.__t0=epoc print "Setting the file: %s"%self.filename return 1 def __setNextFileOnline(self): """ """ filename = self.filenameList[0] if self.__filename_online != None: self.__selecDataForTimes(online=True) filename = self.filenameList[0] while self.__filename_online == filename: print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile) time.sleep(self.__waitForNewFile) self.__selectDataForTimes(online=True) filename = self.filenameList[0] self.__filename_online=filename self.hfFilePointer=h5py.File(filename,'r') self.flagIsNewFile = 1 self.filename = filename print "Setting the file: %s"%self.filename return 1 def __getExpParameters(self): if not(self.status): return None fp=self.hfFilePointer def setup(self, path = None, startDate = None, endDate = None, startTime = datetime.time(0,0,0), endTime = datetime.time(23,59,59), set = None, expLabel = "", ext = None, all=0, timezone='ut', online = False, delay = 60, walk = True): ''' In this method we should set all initial parameters. ''' self.timezone= timezone self.online= online self.all=all if ext==None: ext = self.ext if not(online): self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk) else: self.__searchFilesOnline(path,walk) if not(self.filenameList): print "There is no files into the folder: %s"%(path) sys.exit(-1) self.__getExpParameters() self.fileIndex = -1 self.__setNextFile(online) self.__readMetadata() self.__setLocalVariables() self.__setHeaderDO() #self.profileIndex_offset= 0 #self.profileIndex = self.profileIndex_offset self.isConfig = True def __readMetadata(self): self.__readHeader() def __setLocalVariables(self): self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex) # self.profileIndex = 9999 def __setHeaderDO(self): self.dataOut.radarControllerHeaderObj = RadarControllerHeader() self.dataOut.systemHeaderObj = SystemHeader() self.dataOut.type = "Voltage" self.dataOut.data = None self.dataOut.dtype = numpy.dtype([('real','= self.nProfiles: return 1 return 0 def readNextBlock(self): if not(self.__setNewBlock()): return 0 if not(self.readBlock()): return 0 return 1 def __setNewBlock(self): if self.hfFilePointer==None: return 0 if self.flagIsNewFile: return 1 if self.profileIndex < self.nProfiles: return 1 self.__setNextFile(self.online) return 1 def readBlock(self): fp=self.hfFilePointer #Puntero que apunta al archivo hdf5 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas) ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas) ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles) ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles) self.datablock = numpy.array([ch0,ch1]) self.flagIsNewFile=0 self.profileIndex=0 return 1 def getData(self): if self.flagNoMoreFiles: self.dataOut.flagNoData = True print 'Process finished' return 0 if self.__hasNotDataInBuffer(): if not(self.readNextBlock()): self.dataOut.flagNodata=True return 0 ############################## ############################## self.dataOut.data = self.datablock[:,:,self.profileIndex] self.dataOut.utctime= self.__t0 + self.dataOut.ippSeconds*self.profileIndex self.dataOut.profileIndex= self.profileIndex self.dataOut.flagNoData=False self.profileIndex +=1 return self.dataOut.data def run(self, **kwargs): ''' This method will be called many times so here you should put all your code ''' if not self.isConfig: self.setup(**kwargs) self.isConfig = True self.getData()