jroIO_hf.py
862 lines
| 24.9 KiB
| text/x-python
|
PythonLexer
Alexander Valdez
|
r538 | ''' | ||
| Created on Jul 3, 2014 | ||||
José Chávez
|
r897 | @author: roj-com0419 | ||
|
Alexander Valdez
|
r538 | ''' | ||
| r1396 | import os, sys | |||
| import time, datetime | ||||
|
Alexander Valdez
|
r538 | import h5py | ||
|
Alexander Valdez
|
r541 | import numpy | ||
|
Alexander Valdez
|
r552 | import fnmatch | ||
|
Alexander Valdez
|
r538 | import re | ||
Miguel Valdez
|
r645 | from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader | ||
| from schainpy.model.data.jrodata import Voltage | ||||
| from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | ||||
|
Alexander Valdez
|
r547 | |||
| def isNumber(str): | ||||
| """ | ||||
| Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. | ||||
|
José Chávez
|
r897 | Excepciones: | ||
|
Alexander Valdez
|
r547 | Si un determinado string no puede ser convertido a numero | ||
| Input: | ||||
| str, string al cual se le analiza para determinar si convertible a un numero o no | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | Return: | ||
| True : si el string es uno numerico | ||||
| False : no es un string numerico | ||||
| """ | ||||
| try: | ||||
| r1396 | float(str) | |||
|
Alexander Valdez
|
r547 | return True | ||
| except: | ||||
| return False | ||||
|
Alexander Valdez
|
r615 | def getFileFromSet(path, ext, set=None): | ||
|
Alexander Valdez
|
r547 | validFilelist = [] | ||
| fileList = os.listdir(path) | ||||
|
Alexander Valdez
|
r615 | |||
| if len(fileList) < 1: | ||||
| return None | ||||
|
Alexander Valdez
|
r547 | # 0 1234 567 89A BCDE | ||
| # H YYYY DDD SSS .ext | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | for thisFile in fileList: | ||
| try: | ||||
| r1396 | number = int(thisFile[6:16]) | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | # year = int(thisFile[1:5]) | ||
| # doy = int(thisFile[5:8]) | ||||
| except: | ||||
| continue | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): | ||
| continue | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | validFilelist.append(thisFile) | ||
|
Alexander Valdez
|
r615 | |||
| if len(validFilelist) < 1: | ||||
| return None | ||||
| r1396 | validFilelist = sorted(validFilelist, key=str.lower) | |||
|
Alexander Valdez
|
r615 | |||
| if set == None: | ||||
| return validFilelist[-1] | ||||
|
José Chávez
|
r897 | |||
| r1396 | print("set =" , set) | |||
|
Alexander Valdez
|
r615 | for thisFile in validFilelist: | ||
| if set <= int(thisFile[6:16]): | ||||
| r1396 | print(thisFile, int(thisFile[6:16])) | |||
|
José Chávez
|
r897 | return thisFile | ||
|
Alexander Valdez
|
r615 | |||
| return validFilelist[-1] | ||||
| r1396 | myfile = fnmatch.filter(validFilelist, '*%10d*' % (set)) | |||
| # myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set)) | ||||
|
José Chávez
|
r897 | |||
| r1396 | if len(myfile) != 0: | |||
|
Alexander Valdez
|
r547 | return myfile[0] | ||
| else: | ||||
| r1396 | filename = '*%10.10d%s' % (set, ext.lower()) | |||
| print('the filename %s does not exist' % filename) | ||||
George Yong
|
r1167 | print('...going to the last file: ') | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | if validFilelist: | ||
| r1396 | validFilelist = sorted(validFilelist, key=str.lower) | |||
|
Alexander Valdez
|
r547 | return validFilelist[-1] | ||
| return None | ||||
| def getlastFileFromPath(path, ext): | ||||
| """ | ||||
| Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext" | ||||
|
José Chávez
|
r897 | al final de la depuracion devuelve el ultimo file de la lista que quedo. | ||
| Input: | ||||
|
Alexander Valdez
|
r547 | fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta | ||
|
José Chávez
|
r897 | ext : extension de los files contenidos en una carpeta | ||
|
Alexander Valdez
|
r547 | Return: | ||
| El ultimo file de una determinada carpeta, no se considera el path. | ||||
| """ | ||||
| validFilelist = [] | ||||
| fileList = os.listdir(path) | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | # 0 1234 567 89A BCDE | ||
| # H YYYY DDD SSS .ext | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | for thisFile in fileList: | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | try: | ||
| r1396 | number = int(thisFile[6:16]) | |||
|
Alexander Valdez
|
r547 | except: | ||
|
George Yong
|
r1167 | print("There is a file or folder with different format") | ||
|
Alexander Valdez
|
r547 | if not isNumber(number): | ||
| continue | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | # year = thisFile[1:5] | ||
| # if not isNumber(year): | ||||
| # continue | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | # doy = thisFile[5:8] | ||
| # if not isNumber(doy): | ||||
| # continue | ||||
|
José Chávez
|
r897 | |||
| r1396 | number = int(number) | |||
|
Alexander Valdez
|
r547 | # year = int(year) | ||
| # doy = int(doy) | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): | ||
| continue | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | validFilelist.append(thisFile) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | |||
| if validFilelist: | ||||
| r1396 | validFilelist = sorted(validFilelist, key=str.lower) | |||
|
Alexander Valdez
|
r547 | return validFilelist[-1] | ||
| return None | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | class HFReader(ProcessingUnit): | ||
| ''' | ||||
| classdocs | ||||
| ''' | ||||
| r1396 | path = None | |||
| startDate = None | ||||
| endDate = None | ||||
| startTime = None | ||||
| endTime = None | ||||
| walk = None | ||||
|
Alexander Valdez
|
r538 | isConfig = False | ||
| r1396 | dataOut = None | |||
|
Alexander Valdez
|
r547 | nTries = 3 | ||
| r1396 | ext = ".hdf5" | |||
|
Alexander Valdez
|
r538 | |||
|
José Chávez
|
r897 | def __init__(self, **kwargs): | ||
|
Alexander Valdez
|
r538 | ''' | ||
| Constructor | ||||
| ''' | ||||
|
José Chávez
|
r897 | ProcessingUnit.__init__(self, **kwargs) | ||
| r1396 | self.isConfig = False | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.datablock = None | ||
|
José Chávez
|
r897 | |||
| r1396 | self.filename_current = None | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.utc = 0 | ||
|
José Chávez
|
r897 | |||
| r1396 | self.ext = '.hdf5' | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.flagIsNewFile = 1 | ||
| #------------------------------------------------- | ||||
| r1396 | self.fileIndex = None | |||
|
José Chávez
|
r897 | |||
| r1396 | self.profileIndex_offset = None | |||
|
José Chávez
|
r897 | |||
| r1396 | self.filenameList = [] | |||
|
José Chávez
|
r897 | |||
| r1396 | self.hfFilePointer = None | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r547 | self.filename_online = None | ||
|
José Chávez
|
r897 | |||
| r1396 | self.status = True | |||
|
José Chávez
|
r897 | |||
| r1396 | self.flagNoMoreFiles = False | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r615 | self.__waitForNewFile = 20 | ||
|
Alexander Valdez
|
r547 | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | #-------------------------------------------------- | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut = self.createObjByDefault() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def createObjByDefault(self): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | dataObj = Voltage() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | return dataObj | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def setObjProperties(self): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | pass | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def getBlockDimension(self): | ||
| """ | ||||
| Obtiene la cantidad de puntos a leer por cada bloque de datos | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | Affected: | ||
| self.blocksize | ||||
| Return: | ||||
| None | ||||
| """ | ||||
| r1396 | pts2read = self.nChannels * self.nHeights * self.nProfiles | |||
|
José Chávez
|
r897 | self.blocksize = pts2read | ||
|
Alexander Valdez
|
r538 | def __readHeader(self): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.nProfiles = 100 | ||
| self.nHeights = 1000 | ||||
|
José Chávez
|
r897 | self.nChannels = 2 | ||
| r1396 | self.__firstHeigth = 0 | |||
| self.__nSamples = 1000 | ||||
| self.__deltaHeigth = 1.5 | ||||
| self.__sample_rate = 1e5 | ||||
| # self.__frequency=2.72e6 | ||||
| # self.__frequency=3.64e6 | ||||
| self.__frequency = None | ||||
|
Alexander Valdez
|
r538 | self.__online = False | ||
| r1396 | self.filename_next_set = None | |||
|
José Chávez
|
r897 | |||
| r1396 | # print "Frequency of Operation:", self.__frequency | |||
|
José Chávez
|
r897 | |||
| r1396 | def __setParameters(self, path='', startDate='', endDate='', startTime='', endTime='', walk=''): | |||
|
Alexander Valdez
|
r538 | self.path = path | ||
| self.startDate = startDate | ||||
| self.endDate = endDate | ||||
| self.startTime = startTime | ||||
| self.endTime = endTime | ||||
| self.walk = walk | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __checkPath(self): | ||
| if os.path.exists(self.path): | ||||
| r1396 | self.status = 1 | |||
|
Alexander Valdez
|
r538 | else: | ||
| r1396 | self.status = 0 | |||
| print('Path %s does not exits' % self.path) | ||||
|
Alexander Valdez
|
r551 | return | ||
|
Alexander Valdez
|
r538 | return | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __selDates(self, hf_dirname_format): | ||
| try: | ||||
| r1396 | dir_hf_filename = self.path + "/" + hf_dirname_format | |||
| fp = h5py.File(dir_hf_filename, 'r') | ||||
| hipoc = fp['t'].value | ||||
|
Alexander Valdez
|
r541 | fp.close() | ||
| r1396 | date_time = datetime.datetime.utcfromtimestamp(hipoc) | |||
| year = int(date_time[0:4]) | ||||
| month = int(date_time[5:7]) | ||||
| dom = int(date_time[8:10]) | ||||
| thisDate = datetime.date(year, month, dom) | ||||
| if (thisDate >= self.startDate and thisDate <= self.endDate): | ||||
|
Alexander Valdez
|
r538 | return hf_dirname_format | ||
| except: | ||||
| return None | ||||
|
José Chávez
|
r897 | |||
| r1396 | def __findDataForDates(self, online=False): | |||
|
Alexander Valdez
|
r538 | if not(self.status): | ||
| return None | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | pat = '\d+.\d+' | ||
| r1396 | dirnameList = [re.search(pat, x) for x in os.listdir(self.path)] | |||
| dirnameList = [x for x in dirnameList if x != None] | ||||
|
Alexander Valdez
|
r538 | dirnameList = [x.string for x in dirnameList] | ||
|
José Chávez
|
r897 | if not(online): | ||
|
Alexander Valdez
|
r538 | dirnameList = [self.__selDates(x) for x in dirnameList] | ||
| r1396 | dirnameList = [x for x in dirnameList if x != None] | |||
|
José Chávez
|
r897 | |||
| r1396 | if len(dirnameList) > 0: | |||
|
Alexander Valdez
|
r547 | self.status = 1 | ||
| self.dirnameList = dirnameList | ||||
| self.dirnameList.sort() | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | else: | ||
| self.status = 0 | ||||
| return None | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __getTimeFromData(self): | ||
| r1396 | startDateTime_Reader = datetime.datetime.combine(self.startDate, self.startTime) | |||
| endDateTime_Reader = datetime.datetime.combine(self.endDate, self.endTime) | ||||
| print('Filtering Files from %s to %s' % (startDateTime_Reader, endDateTime_Reader)) | ||||
|
George Yong
|
r1167 | print('........................................') | ||
| r1396 | filter_filenameList = [] | |||
|
Alexander Valdez
|
r538 | self.filenameList.sort() | ||
| r1396 | for i in range(len(self.filenameList) - 1): | |||
| filename = self.filenameList[i] | ||||
| dir_hf_filename = filename | ||||
| fp = h5py.File(dir_hf_filename, 'r') | ||||
| hipoc = fp['t'].value | ||||
| hipoc = hipoc + self.timezone | ||||
| date_time = datetime.datetime.utcfromtimestamp(hipoc) | ||||
|
Alexander Valdez
|
r541 | fp.close() | ||
| r1396 | year = int(date_time[0:4]) | |||
| month = int(date_time[5:7]) | ||||
| dom = int(date_time[8:10]) | ||||
| hour = int(date_time[11:13]) | ||||
| min = int(date_time[14:16]) | ||||
| sec = int(date_time[17:19]) | ||||
| this_time = datetime.datetime(year, month, dom, hour, min, sec) | ||||
| if (this_time >= startDateTime_Reader and this_time <= endDateTime_Reader): | ||||
|
Alexander Valdez
|
r538 | filter_filenameList.append(filename) | ||
| filter_filenameList.sort() | ||||
| self.filenameList = filter_filenameList | ||||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __getFilenameList(self): | ||
| r1396 | # print "hola" | |||
| # print self.dirnameList | ||||
| dirList = [os.path.join(self.path, x) for x in self.dirnameList] | ||||
| self.filenameList = dirList | ||||
| # print self.filenameList | ||||
| # print "pase",len(self.filenameList) | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __selectDataForTimes(self, online=False): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if not(self.status): | ||
|
Alexander Valdez
|
r541 | return None | ||
|
Alexander Valdez
|
r551 | #---------------- | ||
|
Alexander Valdez
|
r538 | self.__getFilenameList() | ||
|
Alexander Valdez
|
r551 | #---------------- | ||
|
Alexander Valdez
|
r538 | if not(online): | ||
| if not(self.all): | ||||
| self.__getTimeFromData() | ||||
| r1396 | if len(self.filenameList) > 0: | |||
| self.status = 1 | ||||
|
Alexander Valdez
|
r538 | self.filenameList.sort() | ||
| else: | ||||
| r1396 | self.status = 0 | |||
|
Alexander Valdez
|
r538 | return None | ||
| else: | ||||
|
Alexander Valdez
|
r615 | if self.set != None: | ||
| r1396 | filename = getFileFromSet(self.path, self.ext, self.set) | |||
|
Alexander Valdez
|
r615 | |||
| r1396 | if self.flag_nextfile == True: | |||
| self.dirnameList = [filename] | ||||
| fullfilename = self.path + "/" + filename | ||||
| self.filenameList = [fullfilename] | ||||
| self.filename_next_set = int(filename[6:16]) + 10 | ||||
|
José Chávez
|
r897 | |||
| r1396 | self.flag_nextfile = False | |||
|
Alexander Valdez
|
r552 | else: | ||
|
George Yong
|
r1167 | print(filename) | ||
| print("PRIMERA CONDICION") | ||||
| r1396 | # if self.filename_next_set== int(filename[6:16]): | |||
|
George Yong
|
r1167 | print("TODO BIEN") | ||
|
Alexander Valdez
|
r615 | |||
| if filename == None: | ||||
|
George Yong
|
r1167 | raise ValueError("corregir") | ||
|
Alexander Valdez
|
r615 | |||
| r1396 | self.dirnameList = [filename] | |||
| fullfilename = self.path + "/" + filename | ||||
| self.filenameList = [fullfilename] | ||||
| self.filename_next_set = int(filename[6:16]) + 10 | ||||
| print("Setting next file", self.filename_next_set) | ||||
| self.set = int(filename[6:16]) | ||||
|
Alexander Valdez
|
r615 | if True: | ||
| pass | ||||
|
Alexander Valdez
|
r552 | else: | ||
|
George Yong
|
r1167 | print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO") | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r552 | else: | ||
| r1396 | filename = getlastFileFromPath(self.path, self.ext) | |||
|
José Chávez
|
r897 | |||
| r1396 | if self.flag_nextfile == True: | |||
| self.dirnameList = [filename] | ||||
| fullfilename = self.path + "/" + filename | ||||
| self.filenameList = [self.filenameList[-1]] | ||||
| self.filename_next_set = int(filename[6:16]) + 10 | ||||
|
José Chávez
|
r897 | |||
| r1396 | self.flag_nextfile = False | |||
|
Alexander Valdez
|
r615 | else: | ||
| r1396 | filename = getFileFromSet(self.path, self.ext, self.set) | |||
|
George Yong
|
r1167 | print(filename) | ||
| print("PRIMERA CONDICION") | ||||
| r1396 | # if self.filename_next_set== int(filename[6:16]): | |||
|
George Yong
|
r1167 | print("TODO BIEN") | ||
|
Alexander Valdez
|
r615 | |||
| if filename == None: | ||||
|
George Yong
|
r1167 | raise ValueError("corregir") | ||
|
José Chávez
|
r897 | |||
| r1396 | self.dirnameList = [filename] | |||
| fullfilename = self.path + "/" + filename | ||||
| self.filenameList = [fullfilename] | ||||
| self.filename_next_set = int(filename[6:16]) + 10 | ||||
| print("Setting next file", self.filename_next_set) | ||||
| self.set = int(filename[6:16]) | ||||
|
Alexander Valdez
|
r615 | if True: | ||
| pass | ||||
| else: | ||||
|
George Yong
|
r1167 | print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO") | ||
|
Alexander Valdez
|
r547 | |||
|
José Chávez
|
r897 | |||
Juan C. Espinoza
|
r1052 | def searchFilesOffLine(self, | ||
|
Alexander Valdez
|
r538 | path, | ||
| startDate, | ||||
| endDate, | ||||
| ext, | ||||
| r1396 | startTime=datetime.time(0, 0, 0), | |||
| endTime=datetime.time(23, 59, 59), | ||||
|
Alexander Valdez
|
r538 | walk=True): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__setParameters(path, startDate, endDate, startTime, endTime, walk) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__checkPath() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__findDataForDates() | ||
| r1396 | # print self.dirnameList | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__selectDataForTimes() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | for i in range(len(self.filenameList)): | ||
| r1396 | print("%s" % (self.filenameList[i])) | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | return | ||
|
José Chávez
|
r897 | |||
|
Juan C. Espinoza
|
r1052 | def searchFilesOnLine(self, | ||
|
Alexander Valdez
|
r538 | path, | ||
| r1396 | expLabel="", | |||
|
Alexander Valdez
|
r547 | ext=None, | ||
| startDate=None, | ||||
| endDate=None, | ||||
| walk=True, | ||||
| set=None): | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r551 | startDate = datetime.datetime.utcnow().date() | ||
| endDate = datetime.datetime.utcnow().date() | ||||
|
José Chávez
|
r897 | |||
| r1396 | self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk) | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__checkPath() | ||
|
José Chávez
|
r897 | |||
| r1396 | fullpath = path | |||
| print("%s folder was found: " % (fullpath)) | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r552 | if set == None: | ||
| r1396 | self.set = None | |||
| filename = getlastFileFromPath(fullpath, ext) | ||||
| startDate = datetime.datetime.utcnow().date | ||||
| endDate = datetime.datetime.utcnow().date() | ||||
|
José Chávez
|
r897 | # | ||
|
Alexander Valdez
|
r552 | else: | ||
| r1396 | filename = getFileFromSet(fullpath, ext, set) | |||
| startDate = None | ||||
| endDate = None | ||||
|
José Chávez
|
r897 | # | ||
|
Alexander Valdez
|
r552 | if not (filename): | ||
| r1396 | return None, None, None, None, None | |||
| # print "%s file was found" %(filename) | ||||
|
José Chávez
|
r897 | |||
| # | ||||
|
Alexander Valdez
|
r551 | # dir_hf_filename= self.path+"/"+filename | ||
| # fp= h5py.File(dir_hf_filename,'r') | ||||
| # hipoc=fp['t'].value | ||||
| # fp.close() | ||||
|
Miguel Valdez
|
r645 | # date_time=datetime.datetime.utcfromtimestamp(hipoc) | ||
|
José Chávez
|
r897 | # | ||
|
Alexander Valdez
|
r551 | # year =int(date_time[0:4]) | ||
| # month=int(date_time[5:7]) | ||||
| # dom =int(date_time[8:10]) | ||||
| # set= int(filename[4:10]) | ||||
| # self.set=set-1 | ||||
| r1396 | # self.dirnameList=[filename] | |||
| filenameList = fullpath + "/" + filename | ||||
| self.dirnameList = [filename] | ||||
| self.filenameList = [filenameList] | ||||
| self.flag_nextfile = True | ||||
| # self.__findDataForDates(online=True) | ||||
| # self.dirnameList=[self.dirnameList[-1]] | ||||
| # print self.dirnameList | ||||
| # self.__selectDataForTimes(online=True) | ||||
| # return fullpath,filename,year,month,dom,set | ||||
|
Alexander Valdez
|
r551 | return | ||
|
José Chávez
|
r897 | |||
| r1396 | def __setNextFile(self, online=False): | |||
|
Alexander Valdez
|
r538 | """ | ||
| """ | ||||
| if not(online): | ||||
| newFile = self.__setNextFileOffline() | ||||
| else: | ||||
| newFile = self.__setNextFileOnline() | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if not(newFile): | ||
| return 0 | ||||
|
José Chávez
|
r897 | return 1 | ||
|
Alexander Valdez
|
r538 | def __setNextFileOffline(self): | ||
| """ | ||||
| """ | ||||
| r1396 | idFile = self.fileIndex | |||
|
Alexander Valdez
|
r538 | while(True): | ||
| idFile += 1 | ||||
| if not (idFile < len(self.filenameList)): | ||||
| self.flagNoMoreFiles = 1 | ||||
|
George Yong
|
r1167 | print("No more Files") | ||
|
Alexander Valdez
|
r538 | return 0 | ||
| filename = self.filenameList[idFile] | ||||
| r1396 | hfFilePointer = h5py.File(filename, 'r') | |||
|
José Chávez
|
r897 | |||
| r1396 | epoc = hfFilePointer['t'].value | |||
| # this_time=datetime.datetime(year,month,dom,hour,min,sec) | ||||
|
Alexander Valdez
|
r538 | break | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.flagIsNewFile = 1 | ||
| self.fileIndex = idFile | ||||
| self.filename = filename | ||||
| self.hfFilePointer = hfFilePointer | ||||
|
Alexander Valdez
|
r541 | hfFilePointer.close() | ||
| r1396 | self.__t0 = epoc | |||
| print("Setting the file: %s" % self.filename) | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __setNextFileOnline(self): | ||
| """ | ||||
|
José Chávez
|
r897 | """ | ||
| r1396 | print("SOY NONE", self.set) | |||
| if self.set == None: | ||||
|
Alexander Valdez
|
r552 | pass | ||
| else: | ||||
| r1396 | self.set += 10 | |||
|
José Chávez
|
r897 | |||
| r1396 | filename = self.filenameList[0] # fullfilename | |||
|
Alexander Valdez
|
r547 | if self.filename_online != None: | ||
| self.__selectDataForTimes(online=True) | ||||
|
Alexander Valdez
|
r538 | filename = self.filenameList[0] | ||
|
Alexander Valdez
|
r547 | while self.filename_online == filename: | ||
| r1396 | print('waiting %d seconds to get a new file...' % (self.__waitForNewFile)) | |||
|
Alexander Valdez
|
r615 | time.sleep(self.__waitForNewFile) | ||
| r1396 | # self.__findDataForDates(online=True) | |||
| self.set = self.filename_next_set | ||||
|
Alexander Valdez
|
r538 | self.__selectDataForTimes(online=True) | ||
| filename = self.filenameList[0] | ||||
| r1396 | sizeoffile = os.path.getsize(filename) | |||
| # print filename | ||||
| sizeoffile = os.path.getsize(filename) | ||||
| if sizeoffile < 1670240: | ||||
| print("%s is not the rigth size" % filename) | ||||
| delay = 50 | ||||
| print('waiting %d seconds for delay...' % (delay)) | ||||
|
José Chávez
|
r897 | time.sleep(delay) | ||
| r1396 | sizeoffile = os.path.getsize(filename) | |||
| if sizeoffile < 1670240: | ||||
| delay = 50 | ||||
| print('waiting %d more seconds for delay...' % (delay)) | ||||
|
José Chávez
|
r897 | time.sleep(delay) | ||
|
Alexander Valdez
|
r615 | |||
| r1396 | sizeoffile = os.path.getsize(filename) | |||
| if sizeoffile < 1670240: | ||||
| delay = 50 | ||||
| print('waiting %d more seconds for delay...' % (delay)) | ||||
|
José Chávez
|
r897 | time.sleep(delay) | ||
| try: | ||||
| r1396 | hfFilePointer = h5py.File(filename, 'r') | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r552 | except: | ||
| r1396 | print("Error reading file %s" % filename) | |||
|
José Chávez
|
r897 | |||
| r1396 | self.filename_online = filename | |||
| epoc = hfFilePointer['t'].value | ||||
|
José Chávez
|
r897 | |||
| r1396 | self.hfFilePointer = hfFilePointer | |||
|
Alexander Valdez
|
r547 | hfFilePointer.close() | ||
| r1396 | self.__t0 = epoc | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.flagIsNewFile = 1 | ||
| self.filename = filename | ||||
|
José Chávez
|
r897 | |||
| r1396 | print("Setting the file: %s" % self.filename) | |||
|
José Chávez
|
r897 | return 1 | ||
|
Alexander Valdez
|
r538 | def __getExpParameters(self): | ||
| if not(self.status): | ||||
|
José Chávez
|
r897 | return None | ||
|
Alexander Valdez
|
r547 | def setup(self, | ||
| r1396 | path=None, | |||
| startDate=None, | ||||
| endDate=None, | ||||
| startTime=datetime.time(0, 0, 0), | ||||
| endTime=datetime.time(23, 59, 59), | ||||
| set=None, | ||||
| expLabel="", | ||||
| ext=None, | ||||
|
Alexander Valdez
|
r538 | all=0, | ||
|
Alexander Valdez
|
r541 | timezone=0, | ||
| r1396 | online=False, | |||
| delay=60, | ||||
| walk=True): | ||||
|
Alexander Valdez
|
r538 | ''' | ||
| In this method we should set all initial parameters. | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | ''' | ||
| r1396 | if path == None: | |||
|
George Yong
|
r1167 | raise ValueError("The path is not valid") | ||
|
José Chávez
|
r897 | |||
| r1396 | if ext == None: | |||
|
Alexander Valdez
|
r547 | ext = self.ext | ||
|
José Chávez
|
r897 | |||
| r1396 | self.timezone = timezone | |||
| self.online = online | ||||
| self.all = all | ||||
| # if set==None: | ||||
|
José Chávez
|
r897 | |||
| r1396 | # print set | |||
|
Alexander Valdez
|
r538 | if not(online): | ||
|
George Yong
|
r1167 | print("Searching files in offline mode...") | ||
|
Alexander Valdez
|
r547 | |||
|
Juan C. Espinoza
|
r1052 | self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk) | ||
|
Alexander Valdez
|
r538 | else: | ||
|
George Yong
|
r1167 | print("Searching files in online mode...") | ||
| r1396 | self.searchFilesOnLine(path, walk, ext, set=set) | |||
| if set == None: | ||||
|
Alexander Valdez
|
r552 | pass | ||
| else: | ||||
| r1396 | self.set = set - 10 | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r551 | # for nTries in range(self.nTries): | ||
|
José Chávez
|
r897 | # | ||
|
Juan C. Espinoza
|
r1052 | # fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set) | ||
|
José Chávez
|
r897 | # | ||
|
Alexander Valdez
|
r551 | # if fullpath: | ||
| # break | ||||
| # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1) | ||||
| # time.sleep(self.delay) | ||||
| # if not(fullpath): | ||||
| # print "There ins't valid files in %s" % path | ||||
| # return None | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if not(self.filenameList): | ||
| r1396 | print("There is no files into the folder: %s" % (path)) | |||
|
Alexander Valdez
|
r538 | sys.exit(-1) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__getExpParameters() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.fileIndex = -1 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__setNextFile(online) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__readMetadata() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__setLocalVariables() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.__setHeaderDO() | ||
| r1396 | # self.profileIndex_offset= 0 | |||
|
José Chávez
|
r897 | |||
| r1396 | # self.profileIndex = self.profileIndex_offset | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.isConfig = True | ||
| def __readMetadata(self): | ||||
| self.__readHeader() | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __setLocalVariables(self): | ||
|
José Chávez
|
r897 | |||
| r1396 | self.datablock = numpy.zeros((self.nChannels, self.nHeights, self.nProfiles), dtype=numpy.complex) | |||
|
Alexander Valdez
|
r538 | # | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.profileIndex = 9999 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __setHeaderDO(self): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.radarControllerHeaderObj = RadarControllerHeader() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.systemHeaderObj = SystemHeader() | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r555 | #--------------------------------------------------------- | ||
| r1396 | self.dataOut.systemHeaderObj.nProfiles = 100 | |||
| self.dataOut.systemHeaderObj.nSamples = 1000 | ||||
|
José Chávez
|
r897 | |||
| r1396 | SAMPLING_STRUCTURE = [('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')] | |||
| self.dataOut.radarControllerHeaderObj.samplingWindow = numpy.zeros((1,), SAMPLING_STRUCTURE) | ||||
| self.dataOut.radarControllerHeaderObj.samplingWindow['h0'] = 0 | ||||
| self.dataOut.radarControllerHeaderObj.samplingWindow['dh'] = 1.5 | ||||
| self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'] = 1000 | ||||
| self.dataOut.radarControllerHeaderObj.nHeights = int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']) | ||||
|
Alexander Valdez
|
r555 | self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0'] | ||
| self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh'] | ||||
| self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'] | ||||
|
José Chávez
|
r897 | |||
| r1396 | self.dataOut.radarControllerHeaderObj.nWindows = 1 | |||
| self.dataOut.radarControllerHeaderObj.codetype = 0 | ||||
| self.dataOut.radarControllerHeaderObj.numTaus = 0 | ||||
| # self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4') | ||||
|
José Chávez
|
r897 | |||
| r1396 | # self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4') | |||
| # self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4') | ||||
| # self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0) | ||||
|
José Chávez
|
r897 | |||
| r1396 | self.dataOut.radarControllerHeaderObj.code_size = 0 | |||
| self.dataOut.nBaud = 0 | ||||
| self.dataOut.nCode = 0 | ||||
| self.dataOut.nPairs = 0 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r555 | #--------------------------------------------------------- | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.type = "Voltage" | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.data = None | ||
|
José Chávez
|
r897 | |||
| r1396 | self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')]) | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.nProfiles = 1 | ||
|
José Chávez
|
r897 | |||
| r1396 | self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype=numpy.float) * self.__deltaHeigth | |||
|
José Chávez
|
r897 | |||
|
George Yong
|
r1167 | self.dataOut.channelList = list(range(self.nChannels)) | ||
|
José Chávez
|
r897 | |||
| r1396 | # self.dataOut.channelIndexList = None | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.flagNoData = True | ||
|
José Chávez
|
r897 | |||
| r1396 | # Set to TRUE if the data is discontinuous | |||
|
Alexander Valdez
|
r538 | self.dataOut.flagDiscontinuousBlock = False | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.utctime = None | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r615 | self.dataOut.timeZone = self.timezone | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.dstFlag = 0 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.errorCount = 0 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r541 | self.dataOut.nCohInt = 1 | ||
|
José Chávez
|
r897 | |||
| r1338 | self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights | |||
|
José Chávez
|
r897 | |||
| r1396 | self.dataOut.flagDecodeData = False # asumo que la data esta decodificada | |||
|
José Chávez
|
r897 | |||
| r1396 | self.dataOut.flagDeflipData = False # asumo que la data esta sin flip | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.flagShiftFFT = False | ||
|
José Chávez
|
r897 | |||
| r1396 | self.dataOut.ippSeconds = 1.0 * self.__nSamples / self.__sample_rate | |||
|
José Chávez
|
r897 | |||
| r1396 | # Time interval between profiles | |||
| # self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.frequency = self.__frequency | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | self.dataOut.realtime = self.__online | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __hasNotDataInBuffer(self): | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if self.profileIndex >= self.nProfiles: | ||
| return 1 | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def readNextBlock(self): | ||
| if not(self.__setNewBlock()): | ||||
| return 0 | ||||
| if not(self.readBlock()): | ||||
| return 0 | ||||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def __setNewBlock(self): | ||
|
José Chávez
|
r897 | |||
| r1396 | if self.hfFilePointer == None: | |||
|
Alexander Valdez
|
r538 | return 0 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if self.flagIsNewFile: | ||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if self.profileIndex < self.nProfiles: | ||
|
José Chávez
|
r897 | return 1 | ||
|
Alexander Valdez
|
r538 | self.__setNextFile(self.online) | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def readBlock(self): | ||
| r1396 | fp = h5py.File(self.filename, 'r') | |||
| # Puntero que apunta al archivo hdf5 | ||||
| ch0 = (fp['ch0']).value # Primer canal (100,1000)--(perfiles,alturas) | ||||
| ch1 = (fp['ch1']).value # Segundo canal (100,1000)--(perfiles,alturas) | ||||
|
Alexander Valdez
|
r541 | fp.close() | ||
| r1396 | ch0 = ch0.swapaxes(0, 1) # Primer canal (100,1000)--(alturas,perfiles) | |||
| ch1 = ch1.swapaxes(0, 1) # Segundo canal (100,1000)--(alturas,perfiles) | ||||
| self.datablock = numpy.array([ch0, ch1]) | ||||
| self.flagIsNewFile = 0 | ||||
|
José Chávez
|
r897 | |||
| r1396 | self.profileIndex = 0 | |||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def getData(self): | ||
| if self.flagNoMoreFiles: | ||||
| self.dataOut.flagNoData = True | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if self.__hasNotDataInBuffer(): | ||
| if not(self.readNextBlock()): | ||||
| r1396 | self.dataOut.flagNodata = True | |||
|
Alexander Valdez
|
r538 | return 0 | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | ############################## | ||
| ############################## | ||||
| r1396 | self.dataOut.data = self.datablock[:, :, self.profileIndex] | |||
| self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds * self.profileIndex | ||||
| self.dataOut.profileIndex = self.profileIndex | ||||
| self.dataOut.flagNoData = False | ||||
| self.profileIndex += 1 | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | return self.dataOut.data | ||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | def run(self, **kwargs): | ||
| ''' | ||||
| This method will be called many times so here you should put all your code | ||||
| ''' | ||||
|
José Chávez
|
r897 | |||
|
Alexander Valdez
|
r538 | if not self.isConfig: | ||
| self.setup(**kwargs) | ||||
| self.isConfig = True | ||||
| r1396 | self.getData() | |||
