jroIO_hf.py
863 lines
| 24.5 KiB
| text/x-python
|
PythonLexer
Alexander Valdez
|
r538 | ''' | ||
| Created on Jul 3, 2014 | ||||
José Chávez
|
r897 | @author: roj-com0419 | ||
|
Alexander Valdez
|
r538 | ''' | ||
|
Alexander Valdez
|
r541 | import os,sys | ||
|
Alexander Valdez
|
r538 | import time,datetime | ||
| import h5py | ||||
|
Alexander Valdez
|
r541 | import numpy | ||
|
Alexander Valdez
|
r552 | import fnmatch | ||
|
Alexander Valdez
|
r538 | import re | ||
Miguel Valdez
|
r645 | from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader | ||
| from schainpy.model.data.jrodata import Voltage | ||||
| from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | ||||
|
Alexander Valdez
|
r547 | |||
| def isNumber(str): | ||||
| """ | ||||
| Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. | ||||
|
José Chávez
|
r897 | Excepciones: | ||
|
Alexander Valdez
|
r547 | Si un determinado string no puede ser convertido a numero | ||
| Input: | ||||
| str, string al cual se le analiza para determinar si convertible a un numero o no | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | Return: | ||
| True : si el string es uno numerico | ||||
| False : no es un string numerico | ||||
| """ | ||||
| try: | ||||
| float( str ) | ||||
| return True | ||||
| except: | ||||
| return False | ||||
|
Alexander Valdez
|
r615 | def getFileFromSet(path, ext, set=None): | ||
|
Alexander Valdez
|
r547 | validFilelist = [] | ||
| fileList = os.listdir(path) | ||||
|
Alexander Valdez
|
r615 | |||
| if len(fileList) < 1: | ||||
| return None | ||||
|
Alexander Valdez
|
r547 | # 0 1234 567 89A BCDE | ||
| # H YYYY DDD SSS .ext | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | for thisFile in fileList: | ||
| try: | ||||
|
Alexander Valdez
|
r557 | number= int(thisFile[6:16]) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | # year = int(thisFile[1:5]) | ||
| # doy = int(thisFile[5:8]) | ||||
| except: | ||||
| continue | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): | ||
| continue | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | validFilelist.append(thisFile) | ||
|
Alexander Valdez
|
r615 | |||
| if len(validFilelist) < 1: | ||||
| return None | ||||
| validFilelist = sorted( validFilelist, key=str.lower ) | ||||
| if set == None: | ||||
| return validFilelist[-1] | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r615 | print "set =" ,set | ||
| for thisFile in validFilelist: | ||||
| if set <= int(thisFile[6:16]): | ||||
| print thisFile,int(thisFile[6:16]) | ||||
|
José Chávez
|
r897 | return thisFile | ||
|
Alexander Valdez
|
r615 | |||
| return validFilelist[-1] | ||||
| myfile = fnmatch.filter(validFilelist,'*%10d*'%(set)) | ||||
|
Alexander Valdez
|
r547 | #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set)) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | if len(myfile)!= 0: | ||
| return myfile[0] | ||||
| else: | ||||
|
Alexander Valdez
|
r557 | filename = '*%10.10d%s'%(set,ext.lower()) | ||
|
Alexander Valdez
|
r547 | print 'the filename %s does not exist'%filename | ||
| print '...going to the last file: ' | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | if validFilelist: | ||
| validFilelist = sorted( validFilelist, key=str.lower ) | ||||
| return validFilelist[-1] | ||||
| return None | ||||
| def getlastFileFromPath(path, ext): | ||||
| """ | ||||
| Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext" | ||||
|
José Chávez
|
r897 | al final de la depuracion devuelve el ultimo file de la lista que quedo. | ||
| Input: | ||||
|
Alexander Valdez
|
r547 | fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta | ||
|
José Chávez
|
r897 | ext : extension de los files contenidos en una carpeta | ||
|
Alexander Valdez
|
r547 | Return: | ||
| El ultimo file de una determinada carpeta, no se considera el path. | ||||
| """ | ||||
| validFilelist = [] | ||||
| fileList = os.listdir(path) | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | # 0 1234 567 89A BCDE | ||
| # H YYYY DDD SSS .ext | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | for thisFile in fileList: | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | try: | ||
|
Alexander Valdez
|
r557 | number= int(thisFile[6:16]) | ||
|
Alexander Valdez
|
r547 | except: | ||
| print "There is a file or folder with different format" | ||||
| if not isNumber(number): | ||||
| continue | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | # year = thisFile[1:5] | ||
| # if not isNumber(year): | ||||
| # continue | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | # doy = thisFile[5:8] | ||
| # if not isNumber(doy): | ||||
| # continue | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | number= int(number) | ||
| # year = int(year) | ||||
| # doy = int(doy) | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): | ||
| continue | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | validFilelist.append(thisFile) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | |||
| if validFilelist: | ||||
| validFilelist = sorted( validFilelist, key=str.lower ) | ||||
| return validFilelist[-1] | ||||
| return None | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | class HFReader(ProcessingUnit): | ||
| ''' | ||||
| classdocs | ||||
| ''' | ||||
| path = None | ||||
| startDate= None | ||||
| endDate = None | ||||
| startTime= None | ||||
| endTime = None | ||||
| walk = None | ||||
| isConfig = False | ||||
| dataOut=None | ||||
|
Alexander Valdez
|
r547 | nTries = 3 | ||
|
Alexander Valdez
|
r538 | ext = ".hdf5" | ||
|
José Chávez
|
r897 | def __init__(self, **kwargs): | ||
|
Alexander Valdez
|
r538 | ''' | ||
| Constructor | ||||
| ''' | ||||
|
José Chávez
|
r897 | ProcessingUnit.__init__(self, **kwargs) | ||
|
Alexander Valdez
|
r538 | self.isConfig =False | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.datablock = None | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r552 | self.filename_current=None | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.utc = 0 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.ext='.hdf5' | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.flagIsNewFile = 1 | ||
| #------------------------------------------------- | ||||
| self.fileIndex=None | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.profileIndex_offset=None | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.filenameList=[] | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.hfFilePointer= None | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | self.filename_online = None | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.status=True | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.flagNoMoreFiles= False | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r615 | self.__waitForNewFile = 20 | ||
|
Alexander Valdez
|
r547 | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | #-------------------------------------------------- | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut = self.createObjByDefault() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def createObjByDefault(self): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | dataObj = Voltage() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | return dataObj | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def setObjProperties(self): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | pass | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def getBlockDimension(self): | ||
| """ | ||||
| Obtiene la cantidad de puntos a leer por cada bloque de datos | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | Affected: | ||
| self.blocksize | ||||
| Return: | ||||
| None | ||||
| """ | ||||
| pts2read =self.nChannels*self.nHeights*self.nProfiles | ||||
|
José Chávez
|
r897 | self.blocksize = pts2read | ||
|
Alexander Valdez
|
r538 | def __readHeader(self): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.nProfiles = 100 | ||
| self.nHeights = 1000 | ||||
|
José Chávez
|
r897 | self.nChannels = 2 | ||
|
Alexander Valdez
|
r538 | self.__firstHeigth=0 | ||
| self.__nSamples=1000 | ||||
| self.__deltaHeigth=1.5 | ||||
|
Alexander Valdez
|
r541 | self.__sample_rate=1e5 | ||
|
Alexander Valdez
|
r551 | #self.__frequency=2.72e6 | ||
|
Alexander Valdez
|
r547 | #self.__frequency=3.64e6 | ||
|
Alexander Valdez
|
r551 | self.__frequency=None | ||
|
Alexander Valdez
|
r538 | self.__online = False | ||
|
Alexander Valdez
|
r552 | self.filename_next_set=None | ||
|
José Chávez
|
r897 | |||
| #print "Frequency of Operation:", self.__frequency | ||||
|
Alexander Valdez
|
r538 | def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''): | ||
| self.path = path | ||||
| self.startDate = startDate | ||||
| self.endDate = endDate | ||||
| self.startTime = startTime | ||||
| self.endTime = endTime | ||||
| self.walk = walk | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __checkPath(self): | ||
| if os.path.exists(self.path): | ||||
| self.status=1 | ||||
| else: | ||||
| self.status=0 | ||||
| print 'Path %s does not exits'%self.path | ||||
|
Alexander Valdez
|
r551 | return | ||
|
Alexander Valdez
|
r538 | return | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __selDates(self, hf_dirname_format): | ||
| try: | ||||
| dir_hf_filename= self.path+"/"+hf_dirname_format | ||||
| fp= h5py.File(dir_hf_filename,'r') | ||||
| hipoc=fp['t'].value | ||||
|
Alexander Valdez
|
r541 | fp.close() | ||
|
Miguel Valdez
|
r645 | date_time=datetime.datetime.utcfromtimestamp(hipoc) | ||
|
Alexander Valdez
|
r538 | year =int(date_time[0:4]) | ||
| month=int(date_time[5:7]) | ||||
| dom =int(date_time[8:10]) | ||||
| thisDate= datetime.date(year,month,dom) | ||||
| if (thisDate>=self.startDate and thisDate <= self.endDate): | ||||
| return hf_dirname_format | ||||
| except: | ||||
| return None | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __findDataForDates(self,online=False): | ||
| if not(self.status): | ||||
| return None | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | pat = '\d+.\d+' | ||
| dirnameList = [re.search(pat,x) for x in os.listdir(self.path)] | ||||
| dirnameList = filter(lambda x:x!=None,dirnameList) | ||||
| dirnameList = [x.string for x in dirnameList] | ||||
|
José Chávez
|
r897 | if not(online): | ||
|
Alexander Valdez
|
r538 | dirnameList = [self.__selDates(x) for x in dirnameList] | ||
| dirnameList = filter(lambda x:x!=None,dirnameList) | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if len(dirnameList)>0: | ||
|
Alexander Valdez
|
r547 | self.status = 1 | ||
| self.dirnameList = dirnameList | ||||
| self.dirnameList.sort() | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | else: | ||
| self.status = 0 | ||||
| return None | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __getTimeFromData(self): | ||
| startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime) | ||||
| endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime) | ||||
| print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader) | ||||
| print '........................................' | ||||
| filter_filenameList=[] | ||||
| self.filenameList.sort() | ||||
| for i in range(len(self.filenameList)-1): | ||||
| filename=self.filenameList[i] | ||||
| dir_hf_filename= filename | ||||
| fp= h5py.File(dir_hf_filename,'r') | ||||
| hipoc=fp['t'].value | ||||
|
Alexander Valdez
|
r547 | hipoc=hipoc+self.timezone | ||
|
Miguel Valdez
|
r645 | date_time=datetime.datetime.utcfromtimestamp(hipoc) | ||
|
Alexander Valdez
|
r541 | fp.close() | ||
|
Alexander Valdez
|
r538 | year =int(date_time[0:4]) | ||
| month=int(date_time[5:7]) | ||||
| dom =int(date_time[8:10]) | ||||
| hour =int(date_time[11:13]) | ||||
| min =int(date_time[14:16]) | ||||
| sec =int(date_time[17:19]) | ||||
|
José Chávez
|
r897 | this_time=datetime.datetime(year,month,dom,hour,min,sec) | ||
|
Alexander Valdez
|
r538 | if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader): | ||
| filter_filenameList.append(filename) | ||||
| filter_filenameList.sort() | ||||
| self.filenameList = filter_filenameList | ||||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __getFilenameList(self): | ||
|
Alexander Valdez
|
r547 | #print "hola" | ||
| #print self.dirnameList | ||||
|
Alexander Valdez
|
r538 | dirList = [os.path.join(self.path,x) for x in self.dirnameList] | ||
|
José Chávez
|
r897 | self.filenameList= dirList | ||
|
Alexander Valdez
|
r552 | #print self.filenameList | ||
| #print "pase",len(self.filenameList) | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __selectDataForTimes(self, online=False): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if not(self.status): | ||
|
Alexander Valdez
|
r541 | return None | ||
|
Alexander Valdez
|
r551 | #---------------- | ||
|
Alexander Valdez
|
r538 | self.__getFilenameList() | ||
|
Alexander Valdez
|
r551 | #---------------- | ||
|
Alexander Valdez
|
r538 | if not(online): | ||
| if not(self.all): | ||||
| self.__getTimeFromData() | ||||
| if len(self.filenameList)>0: | ||||
| self.status=1 | ||||
| self.filenameList.sort() | ||||
| else: | ||||
| self.status=0 | ||||
| return None | ||||
| else: | ||||
|
Alexander Valdez
|
r615 | if self.set != None: | ||
| filename=getFileFromSet(self.path,self.ext,self.set) | ||||
|
Alexander Valdez
|
r552 | if self.flag_nextfile==True: | ||
| self.dirnameList=[filename] | ||||
| fullfilename=self.path+"/"+filename | ||||
|
Alexander Valdez
|
r615 | self.filenameList=[fullfilename] | ||
|
Alexander Valdez
|
r557 | self.filename_next_set=int(filename[6:16])+10 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r552 | self.flag_nextfile=False | ||
| else: | ||||
|
Alexander Valdez
|
r615 | print filename | ||
| print "PRIMERA CONDICION" | ||||
| #if self.filename_next_set== int(filename[6:16]): | ||||
| print "TODO BIEN" | ||||
| if filename == None: | ||||
| raise ValueError, "corregir" | ||||
| self.dirnameList=[filename] | ||||
| fullfilename=self.path+"/"+filename | ||||
| self.filenameList=[fullfilename] | ||||
| self.filename_next_set=int(filename[6:16])+10 | ||||
| print "Setting next file",self.filename_next_set | ||||
| self.set=int(filename[6:16]) | ||||
| if True: | ||||
| pass | ||||
|
Alexander Valdez
|
r552 | else: | ||
|
Alexander Valdez
|
r615 | print "ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO" | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r552 | else: | ||
|
Alexander Valdez
|
r615 | filename =getlastFileFromPath(self.path,self.ext) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r615 | if self.flag_nextfile==True: | ||
|
Alexander Valdez
|
r552 | self.dirnameList=[filename] | ||
| fullfilename=self.path+"/"+filename | ||||
| self.filenameList=[self.filenameList[-1]] | ||||
|
Alexander Valdez
|
r615 | self.filename_next_set=int(filename[6:16])+10 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r615 | self.flag_nextfile=False | ||
| else: | ||||
|
José Chávez
|
r897 | filename=getFileFromSet(self.path,self.ext,self.set) | ||
|
Alexander Valdez
|
r615 | print filename | ||
| print "PRIMERA CONDICION" | ||||
| #if self.filename_next_set== int(filename[6:16]): | ||||
| print "TODO BIEN" | ||||
| if filename == None: | ||||
| raise ValueError, "corregir" | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r615 | self.dirnameList=[filename] | ||
| fullfilename=self.path+"/"+filename | ||||
| self.filenameList=[fullfilename] | ||||
| self.filename_next_set=int(filename[6:16])+10 | ||||
| print "Setting next file",self.filename_next_set | ||||
| self.set=int(filename[6:16]) | ||||
| if True: | ||||
| pass | ||||
| else: | ||||
| print "ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO" | ||||
|
Alexander Valdez
|
r547 | |||
|
José Chávez
|
r897 | |||
Juan C. Espinoza
|
r1052 | def searchFilesOffLine(self, | ||
|
Alexander Valdez
|
r538 | path, | ||
| startDate, | ||||
| endDate, | ||||
| ext, | ||||
| startTime=datetime.time(0,0,0), | ||||
| endTime=datetime.time(23,59,59), | ||||
| walk=True): | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__setParameters(path, startDate, endDate, startTime, endTime, walk) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__checkPath() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__findDataForDates() | ||
|
Alexander Valdez
|
r547 | #print self.dirnameList | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__selectDataForTimes() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | for i in range(len(self.filenameList)): | ||
|
José Chávez
|
r897 | print "%s"% (self.filenameList[i]) | ||
|
Alexander Valdez
|
r538 | return | ||
|
José Chávez
|
r897 | |||
|
Juan C. Espinoza
|
r1052 | def searchFilesOnLine(self, | ||
|
Alexander Valdez
|
r538 | path, | ||
|
Alexander Valdez
|
r547 | expLabel= "", | ||
| ext=None, | ||||
| startDate=None, | ||||
| endDate=None, | ||||
| walk=True, | ||||
| set=None): | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r551 | startDate = datetime.datetime.utcnow().date() | ||
| endDate = datetime.datetime.utcnow().date() | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__checkPath() | ||
|
José Chávez
|
r897 | |||
| fullpath=path | ||||
|
Alexander Valdez
|
r552 | print "%s folder was found: " %(fullpath ) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r552 | if set == None: | ||
| self.set=None | ||||
| filename =getlastFileFromPath(fullpath,ext) | ||||
| startDate= datetime.datetime.utcnow().date | ||||
| endDate= datetime.datetime.utcnow().date() | ||||
|
José Chávez
|
r897 | # | ||
|
Alexander Valdez
|
r552 | else: | ||
| filename= getFileFromSet(fullpath,ext,set) | ||||
| startDate=None | ||||
| endDate=None | ||||
|
José Chávez
|
r897 | # | ||
|
Alexander Valdez
|
r552 | if not (filename): | ||
| return None,None,None,None,None | ||||
|
José Chávez
|
r897 | #print "%s file was found" %(filename) | ||
| # | ||||
|
Alexander Valdez
|
r551 | # dir_hf_filename= self.path+"/"+filename | ||
| # fp= h5py.File(dir_hf_filename,'r') | ||||
| # hipoc=fp['t'].value | ||||
| # fp.close() | ||||
|
Miguel Valdez
|
r645 | # date_time=datetime.datetime.utcfromtimestamp(hipoc) | ||
|
José Chávez
|
r897 | # | ||
|
Alexander Valdez
|
r551 | # year =int(date_time[0:4]) | ||
| # month=int(date_time[5:7]) | ||||
| # dom =int(date_time[8:10]) | ||||
| # set= int(filename[4:10]) | ||||
| # self.set=set-1 | ||||
|
Alexander Valdez
|
r547 | #self.dirnameList=[filename] | ||
|
Alexander Valdez
|
r552 | filenameList= fullpath+"/"+filename | ||
| self.dirnameList=[filename] | ||||
| self.filenameList=[filenameList] | ||||
| self.flag_nextfile=True | ||||
| #self.__findDataForDates(online=True) | ||||
| #self.dirnameList=[self.dirnameList[-1]] | ||||
|
Alexander Valdez
|
r547 | #print self.dirnameList | ||
|
Alexander Valdez
|
r552 | #self.__selectDataForTimes(online=True) | ||
|
Alexander Valdez
|
r551 | #return fullpath,filename,year,month,dom,set | ||
| return | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __setNextFile(self,online=False): | ||
| """ | ||||
| """ | ||||
| if not(online): | ||||
| newFile = self.__setNextFileOffline() | ||||
| else: | ||||
| newFile = self.__setNextFileOnline() | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if not(newFile): | ||
| return 0 | ||||
|
José Chávez
|
r897 | return 1 | ||
|
Alexander Valdez
|
r538 | def __setNextFileOffline(self): | ||
| """ | ||||
| """ | ||||
| idFile= self.fileIndex | ||||
| while(True): | ||||
| idFile += 1 | ||||
| if not (idFile < len(self.filenameList)): | ||||
| self.flagNoMoreFiles = 1 | ||||
| print "No more Files" | ||||
| return 0 | ||||
| filename = self.filenameList[idFile] | ||||
| hfFilePointer =h5py.File(filename,'r') | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | epoc=hfFilePointer['t'].value | ||
| #this_time=datetime.datetime(year,month,dom,hour,min,sec) | ||||
| break | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.flagIsNewFile = 1 | ||
| self.fileIndex = idFile | ||||
| self.filename = filename | ||||
| self.hfFilePointer = hfFilePointer | ||||
|
Alexander Valdez
|
r541 | hfFilePointer.close() | ||
|
Alexander Valdez
|
r538 | self.__t0=epoc | ||
| print "Setting the file: %s"%self.filename | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __setNextFileOnline(self): | ||
| """ | ||||
|
José Chávez
|
r897 | """ | ||
|
Alexander Valdez
|
r615 | print "SOY NONE",self.set | ||
|
Alexander Valdez
|
r552 | if self.set==None: | ||
| pass | ||||
| else: | ||||
|
Alexander Valdez
|
r557 | self.set +=10 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r552 | filename = self.filenameList[0]#fullfilename | ||
|
Alexander Valdez
|
r547 | if self.filename_online != None: | ||
| self.__selectDataForTimes(online=True) | ||||
|
Alexander Valdez
|
r538 | filename = self.filenameList[0] | ||
|
Alexander Valdez
|
r547 | while self.filename_online == filename: | ||
|
Alexander Valdez
|
r538 | print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile) | ||
|
Alexander Valdez
|
r615 | time.sleep(self.__waitForNewFile) | ||
|
Alexander Valdez
|
r552 | #self.__findDataForDates(online=True) | ||
|
Alexander Valdez
|
r615 | self.set=self.filename_next_set | ||
|
Alexander Valdez
|
r538 | self.__selectDataForTimes(online=True) | ||
| filename = self.filenameList[0] | ||||
|
Alexander Valdez
|
r552 | sizeoffile=os.path.getsize(filename) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r552 | #print filename | ||
| sizeoffile=os.path.getsize(filename) | ||||
|
Alexander Valdez
|
r557 | if sizeoffile<1670240: | ||
|
Alexander Valdez
|
r552 | print "%s is not the rigth size"%filename | ||
|
Alexander Valdez
|
r615 | delay=50 | ||
|
Alexander Valdez
|
r552 | print 'waiting %d seconds for delay...'%(delay) | ||
|
José Chávez
|
r897 | time.sleep(delay) | ||
|
Alexander Valdez
|
r615 | sizeoffile=os.path.getsize(filename) | ||
| if sizeoffile<1670240: | ||||
| delay=50 | ||||
| print 'waiting %d more seconds for delay...'%(delay) | ||||
|
José Chávez
|
r897 | time.sleep(delay) | ||
|
Alexander Valdez
|
r615 | |||
|
Alexander Valdez
|
r552 | sizeoffile=os.path.getsize(filename) | ||
|
Alexander Valdez
|
r557 | if sizeoffile<1670240: | ||
|
Alexander Valdez
|
r615 | delay=50 | ||
|
Alexander Valdez
|
r552 | print 'waiting %d more seconds for delay...'%(delay) | ||
|
José Chávez
|
r897 | time.sleep(delay) | ||
| try: | ||||
|
Alexander Valdez
|
r552 | hfFilePointer=h5py.File(filename,'r') | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r552 | except: | ||
| print "Error reading file %s"%filename | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | self.filename_online=filename | ||
| epoc=hfFilePointer['t'].value | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | self.hfFilePointer=hfFilePointer | ||
| hfFilePointer.close() | ||||
| self.__t0=epoc | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.flagIsNewFile = 1 | ||
| self.filename = filename | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | print "Setting the file: %s"%self.filename | ||
|
José Chávez
|
r897 | return 1 | ||
|
Alexander Valdez
|
r538 | def __getExpParameters(self): | ||
| if not(self.status): | ||||
|
José Chávez
|
r897 | return None | ||
|
Alexander Valdez
|
r547 | def setup(self, | ||
| path = None, | ||||
|
Alexander Valdez
|
r538 | startDate = None, | ||
| endDate = None, | ||||
| startTime = datetime.time(0,0,0), | ||||
|
José Chávez
|
r897 | endTime = datetime.time(23,59,59), | ||
| set = None, | ||||
|
Alexander Valdez
|
r538 | expLabel = "", | ||
| ext = None, | ||||
| all=0, | ||||
|
Alexander Valdez
|
r541 | timezone=0, | ||
|
José Chávez
|
r897 | online = False, | ||
|
Alexander Valdez
|
r538 | delay = 60, | ||
| walk = True): | ||||
| ''' | ||||
| In this method we should set all initial parameters. | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | ''' | ||
|
Alexander Valdez
|
r547 | if path==None: | ||
| raise ValueError,"The path is not valid" | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | if ext==None: | ||
| ext = self.ext | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.timezone= timezone | ||
| self.online= online | ||||
| self.all=all | ||||
|
Alexander Valdez
|
r552 | #if set==None: | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r552 | #print set | ||
|
Alexander Valdez
|
r538 | if not(online): | ||
|
Alexander Valdez
|
r547 | print "Searching files in offline mode..." | ||
|
Juan C. Espinoza
|
r1052 | self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk) | ||
|
Alexander Valdez
|
r538 | else: | ||
|
Alexander Valdez
|
r547 | print "Searching files in online mode..." | ||
|
Juan C. Espinoza
|
r1052 | self.searchFilesOnLine(path, walk,ext,set=set) | ||
|
Alexander Valdez
|
r552 | if set==None: | ||
| pass | ||||
| else: | ||||
|
Alexander Valdez
|
r557 | self.set=set-10 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r551 | # for nTries in range(self.nTries): | ||
|
José Chávez
|
r897 | # | ||
|
Juan C. Espinoza
|
r1052 | # fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set) | ||
|
José Chávez
|
r897 | # | ||
|
Alexander Valdez
|
r551 | # if fullpath: | ||
| # break | ||||
| # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1) | ||||
| # time.sleep(self.delay) | ||||
| # if not(fullpath): | ||||
| # print "There ins't valid files in %s" % path | ||||
| # return None | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if not(self.filenameList): | ||
| print "There is no files into the folder: %s"%(path) | ||||
| sys.exit(-1) | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__getExpParameters() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.fileIndex = -1 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__setNextFile(online) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__readMetadata() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__setLocalVariables() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__setHeaderDO() | ||
| #self.profileIndex_offset= 0 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | #self.profileIndex = self.profileIndex_offset | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.isConfig = True | ||
| def __readMetadata(self): | ||||
| self.__readHeader() | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __setLocalVariables(self): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex) | ||
| # | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.profileIndex = 9999 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __setHeaderDO(self): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.radarControllerHeaderObj = RadarControllerHeader() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.systemHeaderObj = SystemHeader() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r555 | #--------------------------------------------------------- | ||
| self.dataOut.systemHeaderObj.nProfiles=100 | ||||
| self.dataOut.systemHeaderObj.nSamples=1000 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r555 | SAMPLING_STRUCTURE=[('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')] | ||
| self.dataOut.radarControllerHeaderObj.samplingWindow=numpy.zeros((1,),SAMPLING_STRUCTURE) | ||||
| self.dataOut.radarControllerHeaderObj.samplingWindow['h0']=0 | ||||
| self.dataOut.radarControllerHeaderObj.samplingWindow['dh']=1.5 | ||||
| self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']=1000 | ||||
| self.dataOut.radarControllerHeaderObj.nHeights=int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']) | ||||
| self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0'] | ||||
| self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh'] | ||||
| self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'] | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r555 | self.dataOut.radarControllerHeaderObj.nWindows=1 | ||
| self.dataOut.radarControllerHeaderObj.codetype=0 | ||||
| self.dataOut.radarControllerHeaderObj.numTaus=0 | ||||
| #self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4') | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r555 | #self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4') | ||
| #self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4') | ||||
| #self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0) | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r555 | self.dataOut.radarControllerHeaderObj.code_size=0 | ||
| self.dataOut.nBaud=0 | ||||
| self.dataOut.nCode=0 | ||||
| self.dataOut.nPairs=0 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r555 | #--------------------------------------------------------- | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.type = "Voltage" | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.data = None | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r555 | self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')]) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.nProfiles = 1 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r541 | self.dataOut.channelList = range(self.nChannels) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r541 | #self.dataOut.channelIndexList = None | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.flagNoData = True | ||
|
José Chávez
|
r897 | |||
| #Set to TRUE if the data is discontinuous | ||||
|
Alexander Valdez
|
r538 | self.dataOut.flagDiscontinuousBlock = False | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.utctime = None | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r615 | self.dataOut.timeZone = self.timezone | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.dstFlag = 0 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.errorCount = 0 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r541 | self.dataOut.nCohInt = 1 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.flagDecodeData = False #asumo que la data esta decodificada | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.flagDeflipData = False #asumo que la data esta sin flip | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.flagShiftFFT = False | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate | ||
|
José Chávez
|
r897 | |||
| #Time interval between profiles | ||||
|
Alexander Valdez
|
r538 | #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.frequency = self.__frequency | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.realtime = self.__online | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __hasNotDataInBuffer(self): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if self.profileIndex >= self.nProfiles: | ||
| return 1 | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def readNextBlock(self): | ||
| if not(self.__setNewBlock()): | ||||
| return 0 | ||||
| if not(self.readBlock()): | ||||
| return 0 | ||||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __setNewBlock(self): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if self.hfFilePointer==None: | ||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if self.flagIsNewFile: | ||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if self.profileIndex < self.nProfiles: | ||
|
José Chávez
|
r897 | return 1 | ||
|
Alexander Valdez
|
r538 | self.__setNextFile(self.online) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def readBlock(self): | ||
|
Alexander Valdez
|
r541 | fp=h5py.File(self.filename,'r') | ||
|
José Chávez
|
r897 | #Puntero que apunta al archivo hdf5 | ||
|
Alexander Valdez
|
r538 | ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas) | ||
| ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas) | ||||
|
Alexander Valdez
|
r541 | fp.close() | ||
|
Alexander Valdez
|
r538 | ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles) | ||
| ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles) | ||||
| self.datablock = numpy.array([ch0,ch1]) | ||||
| self.flagIsNewFile=0 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.profileIndex=0 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def getData(self): | ||
| if self.flagNoMoreFiles: | ||||
| self.dataOut.flagNoData = True | ||||
| print 'Process finished' | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if self.__hasNotDataInBuffer(): | ||
| if not(self.readNextBlock()): | ||||
| self.dataOut.flagNodata=True | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | ############################## | ||
| ############################## | ||||
| self.dataOut.data = self.datablock[:,:,self.profileIndex] | ||||
|
Alexander Valdez
|
r615 | self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds*self.profileIndex | ||
|
Alexander Valdez
|
r538 | self.dataOut.profileIndex= self.profileIndex | ||
| self.dataOut.flagNoData=False | ||||
|
José Chávez
|
r897 | self.profileIndex +=1 | ||
|
Alexander Valdez
|
r538 | return self.dataOut.data | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def run(self, **kwargs): | ||
| ''' | ||||
| This method will be called many times so here you should put all your code | ||||
| ''' | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if not self.isConfig: | ||
| self.setup(**kwargs) | ||||
| self.isConfig = True | ||||
| self.getData() | ||||