''' Created on Jul 3, 2014 @author: roj-com0419 ''' import os, sys import time, datetime import h5py import numpy import fnmatch import re from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader from schainpy.model.data.jrodata import Voltage from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation def isNumber(str): """ Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. Excepciones: Si un determinado string no puede ser convertido a numero Input: str, string al cual se le analiza para determinar si convertible a un numero o no Return: True : si el string es uno numerico False : no es un string numerico """ try: float(str) return True except: return False def getFileFromSet(path, ext, set=None): validFilelist = [] fileList = os.listdir(path) if len(fileList) < 1: return None # 0 1234 567 89A BCDE # H YYYY DDD SSS .ext for thisFile in fileList: try: number = int(thisFile[6:16]) # year = int(thisFile[1:5]) # doy = int(thisFile[5:8]) except: continue if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): continue validFilelist.append(thisFile) if len(validFilelist) < 1: return None validFilelist = sorted(validFilelist, key=str.lower) if set == None: return validFilelist[-1] print("set =" , set) for thisFile in validFilelist: if set <= int(thisFile[6:16]): print(thisFile, int(thisFile[6:16])) return thisFile return validFilelist[-1] myfile = fnmatch.filter(validFilelist, '*%10d*' % (set)) # myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set)) if len(myfile) != 0: return myfile[0] else: filename = '*%10.10d%s' % (set, ext.lower()) print('the filename %s does not exist' % filename) print('...going to the last file: ') if validFilelist: validFilelist = sorted(validFilelist, key=str.lower) return validFilelist[-1] return None def getlastFileFromPath(path, ext): """ Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext" al final de la depuracion devuelve el ultimo file de la lista que quedo. Input: fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta ext : extension de los files contenidos en una carpeta Return: El ultimo file de una determinada carpeta, no se considera el path. """ validFilelist = [] fileList = os.listdir(path) # 0 1234 567 89A BCDE # H YYYY DDD SSS .ext for thisFile in fileList: try: number = int(thisFile[6:16]) except: print("There is a file or folder with different format") if not isNumber(number): continue # year = thisFile[1:5] # if not isNumber(year): # continue # doy = thisFile[5:8] # if not isNumber(doy): # continue number = int(number) # year = int(year) # doy = int(doy) if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): continue validFilelist.append(thisFile) if validFilelist: validFilelist = sorted(validFilelist, key=str.lower) return validFilelist[-1] return None class HFReader(ProcessingUnit): ''' classdocs ''' path = None startDate = None endDate = None startTime = None endTime = None walk = None isConfig = False dataOut = None nTries = 3 ext = ".hdf5" def __init__(self, **kwargs): ''' Constructor ''' ProcessingUnit.__init__(self, **kwargs) self.isConfig = False self.datablock = None self.filename_current = None self.utc = 0 self.ext = '.hdf5' self.flagIsNewFile = 1 #------------------------------------------------- self.fileIndex = None self.profileIndex_offset = None self.filenameList = [] self.hfFilePointer = None self.filename_online = None self.status = True self.flagNoMoreFiles = False self.__waitForNewFile = 20 #-------------------------------------------------- self.dataOut = self.createObjByDefault() def createObjByDefault(self): dataObj = Voltage() return dataObj def setObjProperties(self): pass def getBlockDimension(self): """ Obtiene la cantidad de puntos a leer por cada bloque de datos Affected: self.blocksize Return: None """ pts2read = self.nChannels * self.nHeights * self.nProfiles self.blocksize = pts2read def __readHeader(self): self.nProfiles = 100 self.nHeights = 1000 self.nChannels = 2 self.__firstHeigth = 0 self.__nSamples = 1000 self.__deltaHeigth = 1.5 self.__sample_rate = 1e5 # self.__frequency=2.72e6 # self.__frequency=3.64e6 self.__frequency = None self.__online = False self.filename_next_set = None # print "Frequency of Operation:", self.__frequency def __setParameters(self, path='', startDate='', endDate='', startTime='', endTime='', walk=''): self.path = path self.startDate = startDate self.endDate = endDate self.startTime = startTime self.endTime = endTime self.walk = walk def __checkPath(self): if os.path.exists(self.path): self.status = 1 else: self.status = 0 print('Path %s does not exits' % self.path) return return def __selDates(self, hf_dirname_format): try: dir_hf_filename = self.path + "/" + hf_dirname_format fp = h5py.File(dir_hf_filename, 'r') hipoc = fp['t'].value fp.close() date_time = datetime.datetime.utcfromtimestamp(hipoc) year = int(date_time[0:4]) month = int(date_time[5:7]) dom = int(date_time[8:10]) thisDate = datetime.date(year, month, dom) if (thisDate >= self.startDate and thisDate <= self.endDate): return hf_dirname_format except: return None def __findDataForDates(self, online=False): if not(self.status): return None pat = '\d+.\d+' dirnameList = [re.search(pat, x) for x in os.listdir(self.path)] dirnameList = [x for x in dirnameList if x != None] dirnameList = [x.string for x in dirnameList] if not(online): dirnameList = [self.__selDates(x) for x in dirnameList] dirnameList = [x for x in dirnameList if x != None] if len(dirnameList) > 0: self.status = 1 self.dirnameList = dirnameList self.dirnameList.sort() else: self.status = 0 return None def __getTimeFromData(self): startDateTime_Reader = datetime.datetime.combine(self.startDate, self.startTime) endDateTime_Reader = datetime.datetime.combine(self.endDate, self.endTime) print('Filtering Files from %s to %s' % (startDateTime_Reader, endDateTime_Reader)) print('........................................') filter_filenameList = [] self.filenameList.sort() for i in range(len(self.filenameList) - 1): filename = self.filenameList[i] dir_hf_filename = filename fp = h5py.File(dir_hf_filename, 'r') hipoc = fp['t'].value hipoc = hipoc + self.timezone date_time = datetime.datetime.utcfromtimestamp(hipoc) fp.close() year = int(date_time[0:4]) month = int(date_time[5:7]) dom = int(date_time[8:10]) hour = int(date_time[11:13]) min = int(date_time[14:16]) sec = int(date_time[17:19]) this_time = datetime.datetime(year, month, dom, hour, min, sec) if (this_time >= startDateTime_Reader and this_time <= endDateTime_Reader): filter_filenameList.append(filename) filter_filenameList.sort() self.filenameList = filter_filenameList return 1 def __getFilenameList(self): # print "hola" # print self.dirnameList dirList = [os.path.join(self.path, x) for x in self.dirnameList] self.filenameList = dirList # print self.filenameList # print "pase",len(self.filenameList) def __selectDataForTimes(self, online=False): if not(self.status): return None #---------------- self.__getFilenameList() #---------------- if not(online): if not(self.all): self.__getTimeFromData() if len(self.filenameList) > 0: self.status = 1 self.filenameList.sort() else: self.status = 0 return None else: if self.set != None: filename = getFileFromSet(self.path, self.ext, self.set) if self.flag_nextfile == True: self.dirnameList = [filename] fullfilename = self.path + "/" + filename self.filenameList = [fullfilename] self.filename_next_set = int(filename[6:16]) + 10 self.flag_nextfile = False else: print(filename) print("PRIMERA CONDICION") # if self.filename_next_set== int(filename[6:16]): print("TODO BIEN") if filename == None: raise ValueError("corregir") self.dirnameList = [filename] fullfilename = self.path + "/" + filename self.filenameList = [fullfilename] self.filename_next_set = int(filename[6:16]) + 10 print("Setting next file", self.filename_next_set) self.set = int(filename[6:16]) if True: pass else: print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO") else: filename = getlastFileFromPath(self.path, self.ext) if self.flag_nextfile == True: self.dirnameList = [filename] fullfilename = self.path + "/" + filename self.filenameList = [self.filenameList[-1]] self.filename_next_set = int(filename[6:16]) + 10 self.flag_nextfile = False else: filename = getFileFromSet(self.path, self.ext, self.set) print(filename) print("PRIMERA CONDICION") # if self.filename_next_set== int(filename[6:16]): print("TODO BIEN") if filename == None: raise ValueError("corregir") self.dirnameList = [filename] fullfilename = self.path + "/" + filename self.filenameList = [fullfilename] self.filename_next_set = int(filename[6:16]) + 10 print("Setting next file", self.filename_next_set) self.set = int(filename[6:16]) if True: pass else: print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO") def searchFilesOffLine(self, path, startDate, endDate, ext, startTime=datetime.time(0, 0, 0), endTime=datetime.time(23, 59, 59), walk=True): self.__setParameters(path, startDate, endDate, startTime, endTime, walk) self.__checkPath() self.__findDataForDates() # print self.dirnameList self.__selectDataForTimes() for i in range(len(self.filenameList)): print("%s" % (self.filenameList[i])) return def searchFilesOnLine(self, path, expLabel="", ext=None, startDate=None, endDate=None, walk=True, set=None): startDate = datetime.datetime.utcnow().date() endDate = datetime.datetime.utcnow().date() self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk) self.__checkPath() fullpath = path print("%s folder was found: " % (fullpath)) if set == None: self.set = None filename = getlastFileFromPath(fullpath, ext) startDate = datetime.datetime.utcnow().date endDate = datetime.datetime.utcnow().date() # else: filename = getFileFromSet(fullpath, ext, set) startDate = None endDate = None # if not (filename): return None, None, None, None, None # print "%s file was found" %(filename) # # dir_hf_filename= self.path+"/"+filename # fp= h5py.File(dir_hf_filename,'r') # hipoc=fp['t'].value # fp.close() # date_time=datetime.datetime.utcfromtimestamp(hipoc) # # year =int(date_time[0:4]) # month=int(date_time[5:7]) # dom =int(date_time[8:10]) # set= int(filename[4:10]) # self.set=set-1 # self.dirnameList=[filename] filenameList = fullpath + "/" + filename self.dirnameList = [filename] self.filenameList = [filenameList] self.flag_nextfile = True # self.__findDataForDates(online=True) # self.dirnameList=[self.dirnameList[-1]] # print self.dirnameList # self.__selectDataForTimes(online=True) # return fullpath,filename,year,month,dom,set return def __setNextFile(self, online=False): """ """ if not(online): newFile = self.__setNextFileOffline() else: newFile = self.__setNextFileOnline() if not(newFile): return 0 return 1 def __setNextFileOffline(self): """ """ idFile = self.fileIndex while(True): idFile += 1 if not (idFile < len(self.filenameList)): self.flagNoMoreFiles = 1 print("No more Files") return 0 filename = self.filenameList[idFile] hfFilePointer = h5py.File(filename, 'r') epoc = hfFilePointer['t'].value # this_time=datetime.datetime(year,month,dom,hour,min,sec) break self.flagIsNewFile = 1 self.fileIndex = idFile self.filename = filename self.hfFilePointer = hfFilePointer hfFilePointer.close() self.__t0 = epoc print("Setting the file: %s" % self.filename) return 1 def __setNextFileOnline(self): """ """ print("SOY NONE", self.set) if self.set == None: pass else: self.set += 10 filename = self.filenameList[0] # fullfilename if self.filename_online != None: self.__selectDataForTimes(online=True) filename = self.filenameList[0] while self.filename_online == filename: print('waiting %d seconds to get a new file...' % (self.__waitForNewFile)) time.sleep(self.__waitForNewFile) # self.__findDataForDates(online=True) self.set = self.filename_next_set self.__selectDataForTimes(online=True) filename = self.filenameList[0] sizeoffile = os.path.getsize(filename) # print filename sizeoffile = os.path.getsize(filename) if sizeoffile < 1670240: print("%s is not the rigth size" % filename) delay = 50 print('waiting %d seconds for delay...' % (delay)) time.sleep(delay) sizeoffile = os.path.getsize(filename) if sizeoffile < 1670240: delay = 50 print('waiting %d more seconds for delay...' % (delay)) time.sleep(delay) sizeoffile = os.path.getsize(filename) if sizeoffile < 1670240: delay = 50 print('waiting %d more seconds for delay...' % (delay)) time.sleep(delay) try: hfFilePointer = h5py.File(filename, 'r') except: print("Error reading file %s" % filename) self.filename_online = filename epoc = hfFilePointer['t'].value self.hfFilePointer = hfFilePointer hfFilePointer.close() self.__t0 = epoc self.flagIsNewFile = 1 self.filename = filename print("Setting the file: %s" % self.filename) return 1 def __getExpParameters(self): if not(self.status): return None def setup(self, path=None, startDate=None, endDate=None, startTime=datetime.time(0, 0, 0), endTime=datetime.time(23, 59, 59), set=None, expLabel="", ext=None, all=0, timezone=0, online=False, delay=60, walk=True): ''' In this method we should set all initial parameters. ''' if path == None: raise ValueError("The path is not valid") if ext == None: ext = self.ext self.timezone = timezone self.online = online self.all = all # if set==None: # print set if not(online): print("Searching files in offline mode...") self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk) else: print("Searching files in online mode...") self.searchFilesOnLine(path, walk, ext, set=set) if set == None: pass else: self.set = set - 10 # for nTries in range(self.nTries): # # fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set) # # if fullpath: # break # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1) # time.sleep(self.delay) # if not(fullpath): # print "There ins't valid files in %s" % path # return None if not(self.filenameList): print("There is no files into the folder: %s" % (path)) sys.exit(-1) self.__getExpParameters() self.fileIndex = -1 self.__setNextFile(online) self.__readMetadata() self.__setLocalVariables() self.__setHeaderDO() # self.profileIndex_offset= 0 # self.profileIndex = self.profileIndex_offset self.isConfig = True def __readMetadata(self): self.__readHeader() def __setLocalVariables(self): self.datablock = numpy.zeros((self.nChannels, self.nHeights, self.nProfiles), dtype=numpy.complex) # self.profileIndex = 9999 def __setHeaderDO(self): self.dataOut.radarControllerHeaderObj = RadarControllerHeader() self.dataOut.systemHeaderObj = SystemHeader() #--------------------------------------------------------- self.dataOut.systemHeaderObj.nProfiles = 100 self.dataOut.systemHeaderObj.nSamples = 1000 SAMPLING_STRUCTURE = [('h0', '= self.nProfiles: return 1 return 0 def readNextBlock(self): if not(self.__setNewBlock()): return 0 if not(self.readBlock()): return 0 return 1 def __setNewBlock(self): if self.hfFilePointer == None: return 0 if self.flagIsNewFile: return 1 if self.profileIndex < self.nProfiles: return 1 self.__setNextFile(self.online) return 1 def readBlock(self): fp = h5py.File(self.filename, 'r') # Puntero que apunta al archivo hdf5 ch0 = (fp['ch0']).value # Primer canal (100,1000)--(perfiles,alturas) ch1 = (fp['ch1']).value # Segundo canal (100,1000)--(perfiles,alturas) fp.close() ch0 = ch0.swapaxes(0, 1) # Primer canal (100,1000)--(alturas,perfiles) ch1 = ch1.swapaxes(0, 1) # Segundo canal (100,1000)--(alturas,perfiles) self.datablock = numpy.array([ch0, ch1]) self.flagIsNewFile = 0 self.profileIndex = 0 return 1 def getData(self): if self.flagNoMoreFiles: self.dataOut.flagNoData = True return 0 if self.__hasNotDataInBuffer(): if not(self.readNextBlock()): self.dataOut.flagNodata = True return 0 ############################## ############################## self.dataOut.data = self.datablock[:, :, self.profileIndex] self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds * self.profileIndex self.dataOut.profileIndex = self.profileIndex self.dataOut.flagNoData = False self.profileIndex += 1 return self.dataOut.data def run(self, **kwargs): ''' This method will be called many times so here you should put all your code ''' if not self.isConfig: self.setup(**kwargs) self.isConfig = True self.getData()