diff --git a/schainpy/model/io/jroIO_hf.py b/schainpy/model/io/jroIO_hf.py new file mode 100644 index 0000000..fc992a0 --- /dev/null +++ b/schainpy/model/io/jroIO_hf.py @@ -0,0 +1,500 @@ +''' +Created on Jul 3, 2014 + +@author: roj-com0419 +''' + +import os +import time,datetime +import os +import h5py +import re +import stuffr + +from model.data.jrodata import * +from model.proc.jroproc_base import ProcessingUnit, Operation + +class HFReader(ProcessingUnit): + ''' + classdocs + ''' + path = None + startDate= None + endDate = None + startTime= None + endTime = None + walk = None + isConfig = False + dataOut=None + ext = ".hdf5" + + def __init__(self): + ''' + Constructor + ''' + ProcessingUnit.__init__(self) + + self.isConfig =False + + self.datablock = None + + self.utc = 0 + + self.ext='.hdf5' + + self.flagIsNewFile = 1 + + #------------------------------------------------- + self.fileIndex=None + + self.profileIndex_offset=None + + self.filenameList=[] + + self.hfFilePointer= None + + self.__filename_online = None + + self.status=True + + self.flagNoMoreFiles= False + + #-------------------------------------------------- + + self.dataOut = self.createObjByDefault() + + + def createObjByDefault(self): + + dataObj = Voltage() + + return dataObj + + def setObjProperties(self): + + pass + + def getBlockDimension(self): + """ + Obtiene la cantidad de puntos a leer por cada bloque de datos + + Affected: + self.blocksize + + Return: + None + """ + pts2read =self.nChannels*self.nHeights*self.nProfiles + self.blocksize = pts2read + + def __readHeader(self): + + self.nProfiles = 100 + self.nHeights = 1000 + self.nChannels = 2 + self.__firstHeigth=0 + self.__nSamples=1000 + self.__deltaHeigth=1.5 + self.__sample_rate=10e5 + self.__frequency=2.72e6 + self.__online = False + + + def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''): + self.path = path + self.startDate = startDate + self.endDate = endDate + self.startTime = startTime + self.endTime = endTime + self.walk = walk + + def __checkPath(self): + if os.path.exists(self.path): + self.status=1 + else: + self.status=0 + print 'Path %s does not exits'%self.path + return + + def __selDates(self, hf_dirname_format): + try: + dir_hf_filename= self.path+"/"+hf_dirname_format + fp= h5py.File(dir_hf_filename,'r') + hipoc=fp['t'].value + date_time=stuffr.unix2datestr(hipoc) + year =int(date_time[0:4]) + month=int(date_time[5:7]) + dom =int(date_time[8:10]) + thisDate= datetime.date(year,month,dom) + if (thisDate>=self.startDate and thisDate <= self.endDate): + return hf_dirname_format + except: + return None + + def __findDataForDates(self,online=False): + if not(self.status): + return None + + pat = '\d+.\d+' + dirnameList = [re.search(pat,x) for x in os.listdir(self.path)] + dirnameList = filter(lambda x:x!=None,dirnameList) + dirnameList = [x.string for x in dirnameList] + if not(online): + dirnameList = [self.__selDates(x) for x in dirnameList] + dirnameList = filter(lambda x:x!=None,dirnameList) + if len(dirnameList)>0: + self.status = 1 + self.dirnameList = dirnameList + self.dirnameList.sort() + else: + self.status = 0 + return None + + def __getTimeFromData(self): + startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime) + endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime) + print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader) + print '........................................' + filter_filenameList=[] + self.filenameList.sort() + for i in range(len(self.filenameList)-1): + filename=self.filenameList[i] + dir_hf_filename= filename + fp= h5py.File(dir_hf_filename,'r') + hipoc=fp['t'].value + date_time=stuffr.unix2datestr(hipoc) + year =int(date_time[0:4]) + month=int(date_time[5:7]) + dom =int(date_time[8:10]) + hour =int(date_time[11:13]) + min =int(date_time[14:16]) + sec =int(date_time[17:19]) + this_time=datetime.datetime(year,month,dom,hour,min,sec) + if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader): + filter_filenameList.append(filename) + filter_filenameList.sort() + self.filenameList = filter_filenameList + return 1 + + def __getFilenameList(self): + dirList = [os.path.join(self.path,x) for x in self.dirnameList] + self.filenameList= dirList + + def __selectDataForTimes(self, online=False): + + if not(self.status): + return none + self.__getFilenameList() + if not(online): + if not(self.all): + self.__getTimeFromData() + if len(self.filenameList)>0: + self.status=1 + self.filenameList.sort() + else: + self.status=0 + return None + else: + self.filenameList=[self.filenameList[-2]] + + def __searchFilesOffline(self, + path, + startDate, + endDate, + ext, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + walk=True): + + self.__setParameters(path, startDate, endDate, startTime, endTime, walk) + + self.__checkPath() + + self.__findDataForDates() + + self.__selectDataForTimes() + + for i in range(len(self.filenameList)): + print "%s"% (self.filenameList[i]) + + return + + def __searchFilesOnline(self, + path, + walk=True): + startDate= datetime.datetime.utcnow().date + endDate= datetime.datetime.utcnow().date() + self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk) + self.__checkPath() + self.__findDataForDates(online=True) + self.dirnameList= [self.dirnameList[-1]] + self.__selectDataForTimes(online=True) + return + + def __setNextFile(self,online=False): + """ + """ + if not(online): + newFile = self.__setNextFileOffline() + else: + newFile = self.__setNextFileOnline() + + if not(newFile): + return 0 + return 1 + + def __setNextFileOffline(self): + """ + """ + idFile= self.fileIndex + while(True): + idFile += 1 + if not (idFile < len(self.filenameList)): + self.flagNoMoreFiles = 1 + print "No more Files" + return 0 + filename = self.filenameList[idFile] + hfFilePointer =h5py.File(filename,'r') + + epoc=hfFilePointer['t'].value + #this_time=datetime.datetime(year,month,dom,hour,min,sec) + break + + self.flagIsNewFile = 1 + self.fileIndex = idFile + self.filename = filename + + self.hfFilePointer = hfFilePointer + self.__t0=epoc + + + print "Setting the file: %s"%self.filename + + + + return 1 + + def __setNextFileOnline(self): + """ + """ + filename = self.filenameList[0] + if self.__filename_online != None: + self.__selecDataForTimes(online=True) + filename = self.filenameList[0] + while self.__filename_online == filename: + print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile) + time.sleep(self.__waitForNewFile) + self.__selectDataForTimes(online=True) + filename = self.filenameList[0] + + self.__filename_online=filename + self.hfFilePointer=h5py.File(filename,'r') + self.flagIsNewFile = 1 + self.filename = filename + + print "Setting the file: %s"%self.filename + return 1 + + def __getExpParameters(self): + if not(self.status): + return None + + fp=self.hfFilePointer + + def setup(self, path = None, + startDate = None, + endDate = None, + startTime = datetime.time(0,0,0), + endTime = datetime.time(23,59,59), + set = None, + expLabel = "", + ext = None, + all=0, + timezone='ut', + online = False, + delay = 60, + walk = True): + ''' + In this method we should set all initial parameters. + + ''' + self.timezone= timezone + self.online= online + self.all=all + if ext==None: + ext = self.ext + + if not(online): + self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk) + else: + self.__searchFilesOnline(path,walk) + + if not(self.filenameList): + print "There is no files into the folder: %s"%(path) + sys.exit(-1) + + self.__getExpParameters() + + self.fileIndex = -1 + + self.__setNextFile(online) + + self.__readMetadata() + + self.__setLocalVariables() + + self.__setHeaderDO() + #self.profileIndex_offset= 0 + + #self.profileIndex = self.profileIndex_offset + + self.isConfig = True + + def __readMetadata(self): + self.__readHeader() + + + def __setLocalVariables(self): + + self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex) + # + + + + self.profileIndex = 9999 + + + def __setHeaderDO(self): + + self.dataOut.radarControllerHeaderObj = RadarControllerHeader() + + self.dataOut.systemHeaderObj = SystemHeader() + + self.dataOut.type = "Voltage" + + self.dataOut.data = None + + self.dataOut.dtype = numpy.dtype([('real','= self.nProfiles: + return 1 + + return 0 + + def readNextBlock(self): + if not(self.__setNewBlock()): + return 0 + + if not(self.readBlock()): + return 0 + + return 1 + + def __setNewBlock(self): + + if self.hfFilePointer==None: + return 0 + + if self.flagIsNewFile: + return 1 + + if self.profileIndex < self.nProfiles: + return 1 + + self.__setNextFile(self.online) + + return 1 + + + + def readBlock(self): + fp=self.hfFilePointer #Puntero que apunta al archivo hdf5 + ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas) + ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas) + ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles) + ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles) + self.datablock = numpy.array([ch0,ch1]) + self.flagIsNewFile=0 + + self.profileIndex=0 + + return 1 + + def getData(self): + if self.flagNoMoreFiles: + self.dataOut.flagNoData = True + print 'Process finished' + return 0 + + if self.__hasNotDataInBuffer(): + if not(self.readNextBlock()): + self.dataOut.flagNodata=True + return 0 + + ############################## + ############################## + self.dataOut.data = self.datablock[:,:,self.profileIndex] + self.dataOut.utctime= self.__t0 + self.dataOut.ippSeconds*self.profileIndex + self.dataOut.profileIndex= self.profileIndex + self.dataOut.flagNoData=False + self.profileIndex +=1 + + return self.dataOut.data + + + def run(self, **kwargs): + ''' + This method will be called many times so here you should put all your code + ''' + + if not self.isConfig: + self.setup(**kwargs) + self.isConfig = True + self.getData() + + + + \ No newline at end of file