jroIO_hf.py
862 lines
| 24.6 KiB
| text/x-python
|
PythonLexer
Alexander Valdez
|
r538 | ''' | |
| Created on Jul 3, 2014 | |||
José Chávez
|
r897 | @author: roj-com0419 | |
|
Alexander Valdez
|
r538 | ''' | |
|
Alexander Valdez
|
r541 | import os,sys | |
|
Alexander Valdez
|
r538 | import time,datetime | |
| import h5py | |||
|
Alexander Valdez
|
r541 | import numpy | |
|
Alexander Valdez
|
r552 | import fnmatch | |
|
Alexander Valdez
|
r538 | import re | |
Miguel Valdez
|
r645 | from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader | |
| from schainpy.model.data.jrodata import Voltage | |||
| from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | |||
|
Alexander Valdez
|
r547 | ||
| def isNumber(str): | |||
| """ | |||
| Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. | |||
|
José Chávez
|
r897 | Excepciones: | |
|
Alexander Valdez
|
r547 | Si un determinado string no puede ser convertido a numero | |
| Input: | |||
| str, string al cual se le analiza para determinar si convertible a un numero o no | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | Return: | |
| True : si el string es uno numerico | |||
| False : no es un string numerico | |||
| """ | |||
| try: | |||
| float( str ) | |||
| return True | |||
| except: | |||
| return False | |||
|
Alexander Valdez
|
r615 | def getFileFromSet(path, ext, set=None): | |
|
Alexander Valdez
|
r547 | validFilelist = [] | |
| fileList = os.listdir(path) | |||
|
Alexander Valdez
|
r615 | ||
| if len(fileList) < 1: | |||
| return None | |||
|
Alexander Valdez
|
r547 | # 0 1234 567 89A BCDE | |
| # H YYYY DDD SSS .ext | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | for thisFile in fileList: | |
| try: | |||
|
Alexander Valdez
|
r557 | number= int(thisFile[6:16]) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | # year = int(thisFile[1:5]) | |
| # doy = int(thisFile[5:8]) | |||
| except: | |||
| continue | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): | |
| continue | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | validFilelist.append(thisFile) | |
|
Alexander Valdez
|
r615 | ||
| if len(validFilelist) < 1: | |||
| return None | |||
| validFilelist = sorted( validFilelist, key=str.lower ) | |||
| if set == None: | |||
| return validFilelist[-1] | |||
|
José Chávez
|
r897 | ||
George Yong
|
r1167 | print("set =" ,set) | |
|
Alexander Valdez
|
r615 | for thisFile in validFilelist: | |
| if set <= int(thisFile[6:16]): | |||
|
George Yong
|
r1167 | print(thisFile,int(thisFile[6:16])) | |
|
José Chávez
|
r897 | return thisFile | |
|
Alexander Valdez
|
r615 | ||
| return validFilelist[-1] | |||
| myfile = fnmatch.filter(validFilelist,'*%10d*'%(set)) | |||
|
Alexander Valdez
|
r547 | #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set)) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | if len(myfile)!= 0: | |
| return myfile[0] | |||
| else: | |||
|
Alexander Valdez
|
r557 | filename = '*%10.10d%s'%(set,ext.lower()) | |
|
George Yong
|
r1167 | print('the filename %s does not exist'%filename) | |
| print('...going to the last file: ') | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | if validFilelist: | |
| validFilelist = sorted( validFilelist, key=str.lower ) | |||
| return validFilelist[-1] | |||
| return None | |||
| def getlastFileFromPath(path, ext): | |||
| """ | |||
| Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext" | |||
|
José Chávez
|
r897 | al final de la depuracion devuelve el ultimo file de la lista que quedo. | |
| Input: | |||
|
Alexander Valdez
|
r547 | fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta | |
|
José Chávez
|
r897 | ext : extension de los files contenidos en una carpeta | |
|
Alexander Valdez
|
r547 | Return: | |
| El ultimo file de una determinada carpeta, no se considera el path. | |||
| """ | |||
| validFilelist = [] | |||
| fileList = os.listdir(path) | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | # 0 1234 567 89A BCDE | |
| # H YYYY DDD SSS .ext | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | for thisFile in fileList: | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | try: | |
|
Alexander Valdez
|
r557 | number= int(thisFile[6:16]) | |
|
Alexander Valdez
|
r547 | except: | |
|
George Yong
|
r1167 | print("There is a file or folder with different format") | |
|
Alexander Valdez
|
r547 | if not isNumber(number): | |
| continue | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | # year = thisFile[1:5] | |
| # if not isNumber(year): | |||
| # continue | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | # doy = thisFile[5:8] | |
| # if not isNumber(doy): | |||
| # continue | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | number= int(number) | |
| # year = int(year) | |||
| # doy = int(doy) | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): | |
| continue | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | validFilelist.append(thisFile) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | ||
| if validFilelist: | |||
| validFilelist = sorted( validFilelist, key=str.lower ) | |||
| return validFilelist[-1] | |||
| return None | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | class HFReader(ProcessingUnit): | |
| ''' | |||
| classdocs | |||
| ''' | |||
| path = None | |||
| startDate= None | |||
| endDate = None | |||
| startTime= None | |||
| endTime = None | |||
| walk = None | |||
| isConfig = False | |||
| dataOut=None | |||
|
Alexander Valdez
|
r547 | nTries = 3 | |
|
Alexander Valdez
|
r538 | ext = ".hdf5" | |
|
José Chávez
|
r897 | def __init__(self, **kwargs): | |
|
Alexander Valdez
|
r538 | ''' | |
| Constructor | |||
| ''' | |||
|
José Chávez
|
r897 | ProcessingUnit.__init__(self, **kwargs) | |
|
Alexander Valdez
|
r538 | self.isConfig =False | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.datablock = None | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r552 | self.filename_current=None | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.utc = 0 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.ext='.hdf5' | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.flagIsNewFile = 1 | |
| #------------------------------------------------- | |||
| self.fileIndex=None | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.profileIndex_offset=None | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.filenameList=[] | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.hfFilePointer= None | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | self.filename_online = None | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.status=True | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.flagNoMoreFiles= False | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r615 | self.__waitForNewFile = 20 | |
|
Alexander Valdez
|
r547 | ||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | #-------------------------------------------------- | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut = self.createObjByDefault() | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def createObjByDefault(self): | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | dataObj = Voltage() | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | return dataObj | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def setObjProperties(self): | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | pass | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def getBlockDimension(self): | |
| """ | |||
| Obtiene la cantidad de puntos a leer por cada bloque de datos | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | Affected: | |
| self.blocksize | |||
| Return: | |||
| None | |||
| """ | |||
| pts2read =self.nChannels*self.nHeights*self.nProfiles | |||
|
José Chávez
|
r897 | self.blocksize = pts2read | |
|
Alexander Valdez
|
r538 | def __readHeader(self): | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.nProfiles = 100 | |
| self.nHeights = 1000 | |||
|
José Chávez
|
r897 | self.nChannels = 2 | |
|
Alexander Valdez
|
r538 | self.__firstHeigth=0 | |
| self.__nSamples=1000 | |||
| self.__deltaHeigth=1.5 | |||
|
Alexander Valdez
|
r541 | self.__sample_rate=1e5 | |
|
Alexander Valdez
|
r551 | #self.__frequency=2.72e6 | |
|
Alexander Valdez
|
r547 | #self.__frequency=3.64e6 | |
|
Alexander Valdez
|
r551 | self.__frequency=None | |
|
Alexander Valdez
|
r538 | self.__online = False | |
|
Alexander Valdez
|
r552 | self.filename_next_set=None | |
|
José Chávez
|
r897 | ||
| #print "Frequency of Operation:", self.__frequency | |||
|
Alexander Valdez
|
r538 | def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''): | |
| self.path = path | |||
| self.startDate = startDate | |||
| self.endDate = endDate | |||
| self.startTime = startTime | |||
| self.endTime = endTime | |||
| self.walk = walk | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def __checkPath(self): | |
| if os.path.exists(self.path): | |||
| self.status=1 | |||
| else: | |||
| self.status=0 | |||
|
George Yong
|
r1167 | print('Path %s does not exits'%self.path) | |
|
Alexander Valdez
|
r551 | return | |
|
Alexander Valdez
|
r538 | return | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def __selDates(self, hf_dirname_format): | |
| try: | |||
| dir_hf_filename= self.path+"/"+hf_dirname_format | |||
| fp= h5py.File(dir_hf_filename,'r') | |||
| hipoc=fp['t'].value | |||
|
Alexander Valdez
|
r541 | fp.close() | |
|
Miguel Valdez
|
r645 | date_time=datetime.datetime.utcfromtimestamp(hipoc) | |
|
Alexander Valdez
|
r538 | year =int(date_time[0:4]) | |
| month=int(date_time[5:7]) | |||
| dom =int(date_time[8:10]) | |||
| thisDate= datetime.date(year,month,dom) | |||
| if (thisDate>=self.startDate and thisDate <= self.endDate): | |||
| return hf_dirname_format | |||
| except: | |||
| return None | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def __findDataForDates(self,online=False): | |
| if not(self.status): | |||
| return None | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | pat = '\d+.\d+' | |
| dirnameList = [re.search(pat,x) for x in os.listdir(self.path)] | |||
|
George Yong
|
r1167 | dirnameList = [x for x in dirnameList if x!=None] | |
|
Alexander Valdez
|
r538 | dirnameList = [x.string for x in dirnameList] | |
|
José Chávez
|
r897 | if not(online): | |
|
Alexander Valdez
|
r538 | dirnameList = [self.__selDates(x) for x in dirnameList] | |
|
George Yong
|
r1167 | dirnameList = [x for x in dirnameList if x!=None] | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | if len(dirnameList)>0: | |
|
Alexander Valdez
|
r547 | self.status = 1 | |
| self.dirnameList = dirnameList | |||
| self.dirnameList.sort() | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | else: | |
| self.status = 0 | |||
| return None | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def __getTimeFromData(self): | |
| startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime) | |||
| endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime) | |||
|
George Yong
|
r1167 | print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)) | |
| print('........................................') | |||
|
Alexander Valdez
|
r538 | filter_filenameList=[] | |
| self.filenameList.sort() | |||
| for i in range(len(self.filenameList)-1): | |||
| filename=self.filenameList[i] | |||
| dir_hf_filename= filename | |||
| fp= h5py.File(dir_hf_filename,'r') | |||
| hipoc=fp['t'].value | |||
|
Alexander Valdez
|
r547 | hipoc=hipoc+self.timezone | |
|
Miguel Valdez
|
r645 | date_time=datetime.datetime.utcfromtimestamp(hipoc) | |
|
Alexander Valdez
|
r541 | fp.close() | |
|
Alexander Valdez
|
r538 | year =int(date_time[0:4]) | |
| month=int(date_time[5:7]) | |||
| dom =int(date_time[8:10]) | |||
| hour =int(date_time[11:13]) | |||
| min =int(date_time[14:16]) | |||
| sec =int(date_time[17:19]) | |||
|
José Chávez
|
r897 | this_time=datetime.datetime(year,month,dom,hour,min,sec) | |
|
Alexander Valdez
|
r538 | if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader): | |
| filter_filenameList.append(filename) | |||
| filter_filenameList.sort() | |||
| self.filenameList = filter_filenameList | |||
| return 1 | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def __getFilenameList(self): | |
|
Alexander Valdez
|
r547 | #print "hola" | |
| #print self.dirnameList | |||
|
Alexander Valdez
|
r538 | dirList = [os.path.join(self.path,x) for x in self.dirnameList] | |
|
José Chávez
|
r897 | self.filenameList= dirList | |
|
Alexander Valdez
|
r552 | #print self.filenameList | |
| #print "pase",len(self.filenameList) | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def __selectDataForTimes(self, online=False): | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | if not(self.status): | |
|
Alexander Valdez
|
r541 | return None | |
|
Alexander Valdez
|
r551 | #---------------- | |
|
Alexander Valdez
|
r538 | self.__getFilenameList() | |
|
Alexander Valdez
|
r551 | #---------------- | |
|
Alexander Valdez
|
r538 | if not(online): | |
| if not(self.all): | |||
| self.__getTimeFromData() | |||
| if len(self.filenameList)>0: | |||
| self.status=1 | |||
| self.filenameList.sort() | |||
| else: | |||
| self.status=0 | |||
| return None | |||
| else: | |||
|
Alexander Valdez
|
r615 | if self.set != None: | |
| filename=getFileFromSet(self.path,self.ext,self.set) | |||
|
Alexander Valdez
|
r552 | if self.flag_nextfile==True: | |
| self.dirnameList=[filename] | |||
| fullfilename=self.path+"/"+filename | |||
|
Alexander Valdez
|
r615 | self.filenameList=[fullfilename] | |
|
Alexander Valdez
|
r557 | self.filename_next_set=int(filename[6:16])+10 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r552 | self.flag_nextfile=False | |
| else: | |||
|
George Yong
|
r1167 | print(filename) | |
| print("PRIMERA CONDICION") | |||
|
Alexander Valdez
|
r615 | #if self.filename_next_set== int(filename[6:16]): | |
|
George Yong
|
r1167 | print("TODO BIEN") | |
|
Alexander Valdez
|
r615 | ||
| if filename == None: | |||
|
George Yong
|
r1167 | raise ValueError("corregir") | |
|
Alexander Valdez
|
r615 | ||
| self.dirnameList=[filename] | |||
| fullfilename=self.path+"/"+filename | |||
| self.filenameList=[fullfilename] | |||
| self.filename_next_set=int(filename[6:16])+10 | |||
|
George Yong
|
r1167 | print("Setting next file",self.filename_next_set) | |
|
Alexander Valdez
|
r615 | self.set=int(filename[6:16]) | |
| if True: | |||
| pass | |||
|
Alexander Valdez
|
r552 | else: | |
|
George Yong
|
r1167 | print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO") | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r552 | else: | |
|
Alexander Valdez
|
r615 | filename =getlastFileFromPath(self.path,self.ext) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r615 | if self.flag_nextfile==True: | |
|
Alexander Valdez
|
r552 | self.dirnameList=[filename] | |
| fullfilename=self.path+"/"+filename | |||
| self.filenameList=[self.filenameList[-1]] | |||
|
Alexander Valdez
|
r615 | self.filename_next_set=int(filename[6:16])+10 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r615 | self.flag_nextfile=False | |
| else: | |||
|
José Chávez
|
r897 | filename=getFileFromSet(self.path,self.ext,self.set) | |
|
George Yong
|
r1167 | print(filename) | |
| print("PRIMERA CONDICION") | |||
|
Alexander Valdez
|
r615 | #if self.filename_next_set== int(filename[6:16]): | |
|
George Yong
|
r1167 | print("TODO BIEN") | |
|
Alexander Valdez
|
r615 | ||
| if filename == None: | |||
|
George Yong
|
r1167 | raise ValueError("corregir") | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r615 | self.dirnameList=[filename] | |
| fullfilename=self.path+"/"+filename | |||
| self.filenameList=[fullfilename] | |||
| self.filename_next_set=int(filename[6:16])+10 | |||
|
George Yong
|
r1167 | print("Setting next file",self.filename_next_set) | |
|
Alexander Valdez
|
r615 | self.set=int(filename[6:16]) | |
| if True: | |||
| pass | |||
| else: | |||
|
George Yong
|
r1167 | print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO") | |
|
Alexander Valdez
|
r547 | ||
|
José Chávez
|
r897 | ||
Juan C. Espinoza
|
r1052 | def searchFilesOffLine(self, | |
|
Alexander Valdez
|
r538 | path, | |
| startDate, | |||
| endDate, | |||
| ext, | |||
| startTime=datetime.time(0,0,0), | |||
| endTime=datetime.time(23,59,59), | |||
| walk=True): | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.__setParameters(path, startDate, endDate, startTime, endTime, walk) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.__checkPath() | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.__findDataForDates() | |
|
Alexander Valdez
|
r547 | #print self.dirnameList | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.__selectDataForTimes() | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | for i in range(len(self.filenameList)): | |
|
George Yong
|
r1167 | print("%s"% (self.filenameList[i])) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | return | |
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1052 | def searchFilesOnLine(self, | |
|
Alexander Valdez
|
r538 | path, | |
|
Alexander Valdez
|
r547 | expLabel= "", | |
| ext=None, | |||
| startDate=None, | |||
| endDate=None, | |||
| walk=True, | |||
| set=None): | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r551 | startDate = datetime.datetime.utcnow().date() | |
| endDate = datetime.datetime.utcnow().date() | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.__checkPath() | |
|
José Chávez
|
r897 | ||
| fullpath=path | |||
|
George Yong
|
r1167 | print("%s folder was found: " %(fullpath )) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r552 | if set == None: | |
| self.set=None | |||
| filename =getlastFileFromPath(fullpath,ext) | |||
| startDate= datetime.datetime.utcnow().date | |||
| endDate= datetime.datetime.utcnow().date() | |||
|
José Chávez
|
r897 | # | |
|
Alexander Valdez
|
r552 | else: | |
| filename= getFileFromSet(fullpath,ext,set) | |||
| startDate=None | |||
| endDate=None | |||
|
José Chávez
|
r897 | # | |
|
Alexander Valdez
|
r552 | if not (filename): | |
| return None,None,None,None,None | |||
|
José Chávez
|
r897 | #print "%s file was found" %(filename) | |
| # | |||
|
Alexander Valdez
|
r551 | # dir_hf_filename= self.path+"/"+filename | |
| # fp= h5py.File(dir_hf_filename,'r') | |||
| # hipoc=fp['t'].value | |||
| # fp.close() | |||
|
Miguel Valdez
|
r645 | # date_time=datetime.datetime.utcfromtimestamp(hipoc) | |
|
José Chávez
|
r897 | # | |
|
Alexander Valdez
|
r551 | # year =int(date_time[0:4]) | |
| # month=int(date_time[5:7]) | |||
| # dom =int(date_time[8:10]) | |||
| # set= int(filename[4:10]) | |||
| # self.set=set-1 | |||
|
Alexander Valdez
|
r547 | #self.dirnameList=[filename] | |
|
Alexander Valdez
|
r552 | filenameList= fullpath+"/"+filename | |
| self.dirnameList=[filename] | |||
| self.filenameList=[filenameList] | |||
| self.flag_nextfile=True | |||
| #self.__findDataForDates(online=True) | |||
| #self.dirnameList=[self.dirnameList[-1]] | |||
|
Alexander Valdez
|
r547 | #print self.dirnameList | |
|
Alexander Valdez
|
r552 | #self.__selectDataForTimes(online=True) | |
|
Alexander Valdez
|
r551 | #return fullpath,filename,year,month,dom,set | |
| return | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def __setNextFile(self,online=False): | |
| """ | |||
| """ | |||
| if not(online): | |||
| newFile = self.__setNextFileOffline() | |||
| else: | |||
| newFile = self.__setNextFileOnline() | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | if not(newFile): | |
| return 0 | |||
|
José Chávez
|
r897 | return 1 | |
|
Alexander Valdez
|
r538 | def __setNextFileOffline(self): | |
| """ | |||
| """ | |||
| idFile= self.fileIndex | |||
| while(True): | |||
| idFile += 1 | |||
| if not (idFile < len(self.filenameList)): | |||
| self.flagNoMoreFiles = 1 | |||
|
George Yong
|
r1167 | print("No more Files") | |
|
Alexander Valdez
|
r538 | return 0 | |
| filename = self.filenameList[idFile] | |||
| hfFilePointer =h5py.File(filename,'r') | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | epoc=hfFilePointer['t'].value | |
| #this_time=datetime.datetime(year,month,dom,hour,min,sec) | |||
| break | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.flagIsNewFile = 1 | |
| self.fileIndex = idFile | |||
| self.filename = filename | |||
| self.hfFilePointer = hfFilePointer | |||
|
Alexander Valdez
|
r541 | hfFilePointer.close() | |
|
Alexander Valdez
|
r538 | self.__t0=epoc | |
|
George Yong
|
r1167 | print("Setting the file: %s"%self.filename) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | return 1 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def __setNextFileOnline(self): | |
| """ | |||
|
José Chávez
|
r897 | """ | |
|
George Yong
|
r1167 | print("SOY NONE",self.set) | |
|
Alexander Valdez
|
r552 | if self.set==None: | |
| pass | |||
| else: | |||
|
Alexander Valdez
|
r557 | self.set +=10 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r552 | filename = self.filenameList[0]#fullfilename | |
|
Alexander Valdez
|
r547 | if self.filename_online != None: | |
| self.__selectDataForTimes(online=True) | |||
|
Alexander Valdez
|
r538 | filename = self.filenameList[0] | |
|
Alexander Valdez
|
r547 | while self.filename_online == filename: | |
|
George Yong
|
r1167 | print('waiting %d seconds to get a new file...'%(self.__waitForNewFile)) | |
|
Alexander Valdez
|
r615 | time.sleep(self.__waitForNewFile) | |
|
Alexander Valdez
|
r552 | #self.__findDataForDates(online=True) | |
|
Alexander Valdez
|
r615 | self.set=self.filename_next_set | |
|
Alexander Valdez
|
r538 | self.__selectDataForTimes(online=True) | |
| filename = self.filenameList[0] | |||
|
Alexander Valdez
|
r552 | sizeoffile=os.path.getsize(filename) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r552 | #print filename | |
| sizeoffile=os.path.getsize(filename) | |||
|
Alexander Valdez
|
r557 | if sizeoffile<1670240: | |
|
George Yong
|
r1167 | print("%s is not the rigth size"%filename) | |
|
Alexander Valdez
|
r615 | delay=50 | |
|
George Yong
|
r1167 | print('waiting %d seconds for delay...'%(delay)) | |
|
José Chávez
|
r897 | time.sleep(delay) | |
|
Alexander Valdez
|
r615 | sizeoffile=os.path.getsize(filename) | |
| if sizeoffile<1670240: | |||
| delay=50 | |||
|
George Yong
|
r1167 | print('waiting %d more seconds for delay...'%(delay)) | |
|
José Chávez
|
r897 | time.sleep(delay) | |
|
Alexander Valdez
|
r615 | ||
|
Alexander Valdez
|
r552 | sizeoffile=os.path.getsize(filename) | |
|
Alexander Valdez
|
r557 | if sizeoffile<1670240: | |
|
Alexander Valdez
|
r615 | delay=50 | |
|
George Yong
|
r1167 | print('waiting %d more seconds for delay...'%(delay)) | |
|
José Chávez
|
r897 | time.sleep(delay) | |
| try: | |||
|
Alexander Valdez
|
r552 | hfFilePointer=h5py.File(filename,'r') | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r552 | except: | |
|
George Yong
|
r1167 | print("Error reading file %s"%filename) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | self.filename_online=filename | |
| epoc=hfFilePointer['t'].value | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | self.hfFilePointer=hfFilePointer | |
| hfFilePointer.close() | |||
| self.__t0=epoc | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.flagIsNewFile = 1 | |
| self.filename = filename | |||
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | print("Setting the file: %s"%self.filename) | |
|
José Chávez
|
r897 | return 1 | |
|
Alexander Valdez
|
r538 | def __getExpParameters(self): | |
| if not(self.status): | |||
|
José Chávez
|
r897 | return None | |
|
Alexander Valdez
|
r547 | def setup(self, | |
| path = None, | |||
|
Alexander Valdez
|
r538 | startDate = None, | |
| endDate = None, | |||
| startTime = datetime.time(0,0,0), | |||
|
José Chávez
|
r897 | endTime = datetime.time(23,59,59), | |
| set = None, | |||
|
Alexander Valdez
|
r538 | expLabel = "", | |
| ext = None, | |||
| all=0, | |||
|
Alexander Valdez
|
r541 | timezone=0, | |
|
José Chávez
|
r897 | online = False, | |
|
Alexander Valdez
|
r538 | delay = 60, | |
| walk = True): | |||
| ''' | |||
| In this method we should set all initial parameters. | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | ''' | |
|
Alexander Valdez
|
r547 | if path==None: | |
|
George Yong
|
r1167 | raise ValueError("The path is not valid") | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r547 | if ext==None: | |
| ext = self.ext | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.timezone= timezone | |
| self.online= online | |||
| self.all=all | |||
|
Alexander Valdez
|
r552 | #if set==None: | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r552 | #print set | |
|
Alexander Valdez
|
r538 | if not(online): | |
|
George Yong
|
r1167 | print("Searching files in offline mode...") | |
|
Alexander Valdez
|
r547 | ||
|
Juan C. Espinoza
|
r1052 | self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk) | |
|
Alexander Valdez
|
r538 | else: | |
|
George Yong
|
r1167 | print("Searching files in online mode...") | |
|
Juan C. Espinoza
|
r1052 | self.searchFilesOnLine(path, walk,ext,set=set) | |
|
Alexander Valdez
|
r552 | if set==None: | |
| pass | |||
| else: | |||
|
Alexander Valdez
|
r557 | self.set=set-10 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r551 | # for nTries in range(self.nTries): | |
|
José Chávez
|
r897 | # | |
|
Juan C. Espinoza
|
r1052 | # fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set) | |
|
José Chávez
|
r897 | # | |
|
Alexander Valdez
|
r551 | # if fullpath: | |
| # break | |||
| # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1) | |||
| # time.sleep(self.delay) | |||
| # if not(fullpath): | |||
| # print "There ins't valid files in %s" % path | |||
| # return None | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | if not(self.filenameList): | |
|
George Yong
|
r1167 | print("There is no files into the folder: %s"%(path)) | |
|
Alexander Valdez
|
r538 | sys.exit(-1) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.__getExpParameters() | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.fileIndex = -1 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.__setNextFile(online) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.__readMetadata() | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.__setLocalVariables() | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.__setHeaderDO() | |
| #self.profileIndex_offset= 0 | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | #self.profileIndex = self.profileIndex_offset | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.isConfig = True | |
| def __readMetadata(self): | |||
| self.__readHeader() | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def __setLocalVariables(self): | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex) | |
| # | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.profileIndex = 9999 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def __setHeaderDO(self): | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.radarControllerHeaderObj = RadarControllerHeader() | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.systemHeaderObj = SystemHeader() | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r555 | #--------------------------------------------------------- | |
| self.dataOut.systemHeaderObj.nProfiles=100 | |||
| self.dataOut.systemHeaderObj.nSamples=1000 | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r555 | SAMPLING_STRUCTURE=[('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')] | |
| self.dataOut.radarControllerHeaderObj.samplingWindow=numpy.zeros((1,),SAMPLING_STRUCTURE) | |||
| self.dataOut.radarControllerHeaderObj.samplingWindow['h0']=0 | |||
| self.dataOut.radarControllerHeaderObj.samplingWindow['dh']=1.5 | |||
| self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']=1000 | |||
| self.dataOut.radarControllerHeaderObj.nHeights=int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']) | |||
| self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0'] | |||
| self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh'] | |||
| self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'] | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r555 | self.dataOut.radarControllerHeaderObj.nWindows=1 | |
| self.dataOut.radarControllerHeaderObj.codetype=0 | |||
| self.dataOut.radarControllerHeaderObj.numTaus=0 | |||
| #self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4') | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r555 | #self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4') | |
| #self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4') | |||
| #self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0) | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r555 | self.dataOut.radarControllerHeaderObj.code_size=0 | |
| self.dataOut.nBaud=0 | |||
| self.dataOut.nCode=0 | |||
| self.dataOut.nPairs=0 | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r555 | #--------------------------------------------------------- | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.type = "Voltage" | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.data = None | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r555 | self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')]) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.nProfiles = 1 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth | |
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | self.dataOut.channelList = list(range(self.nChannels)) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r541 | #self.dataOut.channelIndexList = None | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.flagNoData = True | |
|
José Chávez
|
r897 | ||
| #Set to TRUE if the data is discontinuous | |||
|
Alexander Valdez
|
r538 | self.dataOut.flagDiscontinuousBlock = False | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.utctime = None | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r615 | self.dataOut.timeZone = self.timezone | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.dstFlag = 0 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.errorCount = 0 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r541 | self.dataOut.nCohInt = 1 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights() | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.flagDecodeData = False #asumo que la data esta decodificada | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.flagDeflipData = False #asumo que la data esta sin flip | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.flagShiftFFT = False | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate | |
|
José Chávez
|
r897 | ||
| #Time interval between profiles | |||
|
Alexander Valdez
|
r538 | #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.frequency = self.__frequency | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.dataOut.realtime = self.__online | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def __hasNotDataInBuffer(self): | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | if self.profileIndex >= self.nProfiles: | |
| return 1 | |||
| return 0 | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def readNextBlock(self): | |
| if not(self.__setNewBlock()): | |||
| return 0 | |||
| if not(self.readBlock()): | |||
| return 0 | |||
| return 1 | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def __setNewBlock(self): | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | if self.hfFilePointer==None: | |
| return 0 | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | if self.flagIsNewFile: | |
| return 1 | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | if self.profileIndex < self.nProfiles: | |
|
José Chávez
|
r897 | return 1 | |
|
Alexander Valdez
|
r538 | self.__setNextFile(self.online) | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | return 1 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def readBlock(self): | |
|
Alexander Valdez
|
r541 | fp=h5py.File(self.filename,'r') | |
|
José Chávez
|
r897 | #Puntero que apunta al archivo hdf5 | |
|
Alexander Valdez
|
r538 | ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas) | |
| ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas) | |||
|
Alexander Valdez
|
r541 | fp.close() | |
|
Alexander Valdez
|
r538 | ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles) | |
| ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles) | |||
| self.datablock = numpy.array([ch0,ch1]) | |||
| self.flagIsNewFile=0 | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | self.profileIndex=0 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | return 1 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def getData(self): | |
| if self.flagNoMoreFiles: | |||
| self.dataOut.flagNoData = True | |||
|
George Yong
|
r1167 | print('Process finished') | |
|
Alexander Valdez
|
r538 | return 0 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | if self.__hasNotDataInBuffer(): | |
| if not(self.readNextBlock()): | |||
| self.dataOut.flagNodata=True | |||
| return 0 | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | ############################## | |
| ############################## | |||
| self.dataOut.data = self.datablock[:,:,self.profileIndex] | |||
|
Alexander Valdez
|
r615 | self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds*self.profileIndex | |
|
Alexander Valdez
|
r538 | self.dataOut.profileIndex= self.profileIndex | |
| self.dataOut.flagNoData=False | |||
|
José Chávez
|
r897 | self.profileIndex +=1 | |
|
Alexander Valdez
|
r538 | return self.dataOut.data | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | def run(self, **kwargs): | |
| ''' | |||
| This method will be called many times so here you should put all your code | |||
| ''' | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r538 | if not self.isConfig: | |
| self.setup(**kwargs) | |||
| self.isConfig = True | |||
|
George Yong
|
r1167 | self.getData() |