##// END OF EJS Templates
schain-jc
RSS
Python 2to3, Spectra (all operations) working
Python 2to3, Spectra (all operations) working
George Yong - - Load All Authors

File last commit:

r1167:1f521b07c958
r1167:1f521b07c958
Show More
jroIO_hf.py
862 lines | 24.6 KiB | text/x-python | PythonLexer
/ schainpy / model / io / jroIO_hf.py
History | Annotation | Raw |Copy content |Copy permalink
'''
Created on Jul 3, 2014
@author: roj-com0419
'''
import os,sys
import time,datetime
import h5py
import numpy
import fnmatch
import re
from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
from schainpy.model.data.jrodata import Voltage
from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
def isNumber(str):
"""
Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
Excepciones:
Si un determinado string no puede ser convertido a numero
Input:
str, string al cual se le analiza para determinar si convertible a un numero o no
Return:
True : si el string es uno numerico
False : no es un string numerico
"""
try:
float( str )
return True
except:
return False
def getFileFromSet(path, ext, set=None):
validFilelist = []
fileList = os.listdir(path)
if len(fileList) < 1:
return None
# 0 1234 567 89A BCDE
# H YYYY DDD SSS .ext
for thisFile in fileList:
try:
number= int(thisFile[6:16])
# year = int(thisFile[1:5])
# doy = int(thisFile[5:8])
except:
continue
if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
continue
validFilelist.append(thisFile)
if len(validFilelist) < 1:
return None
validFilelist = sorted( validFilelist, key=str.lower )
if set == None:
return validFilelist[-1]
print("set =" ,set)
for thisFile in validFilelist:
if set <= int(thisFile[6:16]):
print(thisFile,int(thisFile[6:16]))
return thisFile
return validFilelist[-1]
myfile = fnmatch.filter(validFilelist,'*%10d*'%(set))
#myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
if len(myfile)!= 0:
return myfile[0]
else:
filename = '*%10.10d%s'%(set,ext.lower())
print('the filename %s does not exist'%filename)
print('...going to the last file: ')
if validFilelist:
validFilelist = sorted( validFilelist, key=str.lower )
return validFilelist[-1]
return None
def getlastFileFromPath(path, ext):
"""
Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
al final de la depuracion devuelve el ultimo file de la lista que quedo.
Input:
fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
ext : extension de los files contenidos en una carpeta
Return:
El ultimo file de una determinada carpeta, no se considera el path.
"""
validFilelist = []
fileList = os.listdir(path)
# 0 1234 567 89A BCDE
# H YYYY DDD SSS .ext
for thisFile in fileList:
try:
number= int(thisFile[6:16])
except:
print("There is a file or folder with different format")
if not isNumber(number):
continue
# year = thisFile[1:5]
# if not isNumber(year):
# continue
# doy = thisFile[5:8]
# if not isNumber(doy):
# continue
number= int(number)
# year = int(year)
# doy = int(doy)
if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
continue
validFilelist.append(thisFile)
if validFilelist:
validFilelist = sorted( validFilelist, key=str.lower )
return validFilelist[-1]
return None
class HFReader(ProcessingUnit):
'''
classdocs
'''
path = None
startDate= None
endDate = None
startTime= None
endTime = None
walk = None
isConfig = False
dataOut=None
nTries = 3
ext = ".hdf5"
def __init__(self, **kwargs):
'''
Constructor
'''
ProcessingUnit.__init__(self, **kwargs)
self.isConfig =False
self.datablock = None
self.filename_current=None
self.utc = 0
self.ext='.hdf5'
self.flagIsNewFile = 1
#-------------------------------------------------
self.fileIndex=None
self.profileIndex_offset=None
self.filenameList=[]
self.hfFilePointer= None
self.filename_online = None
self.status=True
self.flagNoMoreFiles= False
self.__waitForNewFile = 20
#--------------------------------------------------
self.dataOut = self.createObjByDefault()
def createObjByDefault(self):
dataObj = Voltage()
return dataObj
def setObjProperties(self):
pass
def getBlockDimension(self):
"""
Obtiene la cantidad de puntos a leer por cada bloque de datos
Affected:
self.blocksize
Return:
None
"""
pts2read =self.nChannels*self.nHeights*self.nProfiles
self.blocksize = pts2read
def __readHeader(self):
self.nProfiles = 100
self.nHeights = 1000
self.nChannels = 2
self.__firstHeigth=0
self.__nSamples=1000
self.__deltaHeigth=1.5
self.__sample_rate=1e5
#self.__frequency=2.72e6
#self.__frequency=3.64e6
self.__frequency=None
self.__online = False
self.filename_next_set=None
#print "Frequency of Operation:", self.__frequency
def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
self.path = path
self.startDate = startDate
self.endDate = endDate
self.startTime = startTime
self.endTime = endTime
self.walk = walk
def __checkPath(self):
if os.path.exists(self.path):
self.status=1
else:
self.status=0
print('Path %s does not exits'%self.path)
return
return
def __selDates(self, hf_dirname_format):
try:
dir_hf_filename= self.path+"/"+hf_dirname_format
fp= h5py.File(dir_hf_filename,'r')
hipoc=fp['t'].value
fp.close()
date_time=datetime.datetime.utcfromtimestamp(hipoc)
year =int(date_time[0:4])
month=int(date_time[5:7])
dom =int(date_time[8:10])
thisDate= datetime.date(year,month,dom)
if (thisDate>=self.startDate and thisDate <= self.endDate):
return hf_dirname_format
except:
return None
def __findDataForDates(self,online=False):
if not(self.status):
return None
pat = '\d+.\d+'
dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
dirnameList = [x for x in dirnameList if x!=None]
dirnameList = [x.string for x in dirnameList]
if not(online):
dirnameList = [self.__selDates(x) for x in dirnameList]
dirnameList = [x for x in dirnameList if x!=None]
if len(dirnameList)>0:
self.status = 1
self.dirnameList = dirnameList
self.dirnameList.sort()
else:
self.status = 0
return None
def __getTimeFromData(self):
startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
print('........................................')
filter_filenameList=[]
self.filenameList.sort()
for i in range(len(self.filenameList)-1):
filename=self.filenameList[i]
dir_hf_filename= filename
fp= h5py.File(dir_hf_filename,'r')
hipoc=fp['t'].value
hipoc=hipoc+self.timezone
date_time=datetime.datetime.utcfromtimestamp(hipoc)
fp.close()
year =int(date_time[0:4])
month=int(date_time[5:7])
dom =int(date_time[8:10])
hour =int(date_time[11:13])
min =int(date_time[14:16])
sec =int(date_time[17:19])
this_time=datetime.datetime(year,month,dom,hour,min,sec)
if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
filter_filenameList.append(filename)
filter_filenameList.sort()
self.filenameList = filter_filenameList
return 1
def __getFilenameList(self):
#print "hola"
#print self.dirnameList
dirList = [os.path.join(self.path,x) for x in self.dirnameList]
self.filenameList= dirList
#print self.filenameList
#print "pase",len(self.filenameList)
def __selectDataForTimes(self, online=False):
if not(self.status):
return None
#----------------
self.__getFilenameList()
#----------------
if not(online):
if not(self.all):
self.__getTimeFromData()
if len(self.filenameList)>0:
self.status=1
self.filenameList.sort()
else:
self.status=0
return None
else:
if self.set != None:
filename=getFileFromSet(self.path,self.ext,self.set)
if self.flag_nextfile==True:
self.dirnameList=[filename]
fullfilename=self.path+"/"+filename
self.filenameList=[fullfilename]
self.filename_next_set=int(filename[6:16])+10
self.flag_nextfile=False
else:
print(filename)
print("PRIMERA CONDICION")
#if self.filename_next_set== int(filename[6:16]):
print("TODO BIEN")
if filename == None:
raise ValueError("corregir")
self.dirnameList=[filename]
fullfilename=self.path+"/"+filename
self.filenameList=[fullfilename]
self.filename_next_set=int(filename[6:16])+10
print("Setting next file",self.filename_next_set)
self.set=int(filename[6:16])
if True:
pass
else:
print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO")
else:
filename =getlastFileFromPath(self.path,self.ext)
if self.flag_nextfile==True:
self.dirnameList=[filename]
fullfilename=self.path+"/"+filename
self.filenameList=[self.filenameList[-1]]
self.filename_next_set=int(filename[6:16])+10
self.flag_nextfile=False
else:
filename=getFileFromSet(self.path,self.ext,self.set)
print(filename)
print("PRIMERA CONDICION")
#if self.filename_next_set== int(filename[6:16]):
print("TODO BIEN")
if filename == None:
raise ValueError("corregir")
self.dirnameList=[filename]
fullfilename=self.path+"/"+filename
self.filenameList=[fullfilename]
self.filename_next_set=int(filename[6:16])+10
print("Setting next file",self.filename_next_set)
self.set=int(filename[6:16])
if True:
pass
else:
print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO")
def searchFilesOffLine(self,
path,
startDate,
endDate,
ext,
startTime=datetime.time(0,0,0),
endTime=datetime.time(23,59,59),
walk=True):
self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
self.__checkPath()
self.__findDataForDates()
#print self.dirnameList
self.__selectDataForTimes()
for i in range(len(self.filenameList)):
print("%s"% (self.filenameList[i]))
return
def searchFilesOnLine(self,
path,
expLabel= "",
ext=None,
startDate=None,
endDate=None,
walk=True,
set=None):
startDate = datetime.datetime.utcnow().date()
endDate = datetime.datetime.utcnow().date()
self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
self.__checkPath()
fullpath=path
print("%s folder was found: " %(fullpath ))
if set == None:
self.set=None
filename =getlastFileFromPath(fullpath,ext)
startDate= datetime.datetime.utcnow().date
endDate= datetime.datetime.utcnow().date()
#
else:
filename= getFileFromSet(fullpath,ext,set)
startDate=None
endDate=None
#
if not (filename):
return None,None,None,None,None
#print "%s file was found" %(filename)
#
# dir_hf_filename= self.path+"/"+filename
# fp= h5py.File(dir_hf_filename,'r')
# hipoc=fp['t'].value
# fp.close()
# date_time=datetime.datetime.utcfromtimestamp(hipoc)
#
# year =int(date_time[0:4])
# month=int(date_time[5:7])
# dom =int(date_time[8:10])
# set= int(filename[4:10])
# self.set=set-1
#self.dirnameList=[filename]
filenameList= fullpath+"/"+filename
self.dirnameList=[filename]
self.filenameList=[filenameList]
self.flag_nextfile=True
#self.__findDataForDates(online=True)
#self.dirnameList=[self.dirnameList[-1]]
#print self.dirnameList
#self.__selectDataForTimes(online=True)
#return fullpath,filename,year,month,dom,set
return
def __setNextFile(self,online=False):
"""
"""
if not(online):
newFile = self.__setNextFileOffline()
else:
newFile = self.__setNextFileOnline()
if not(newFile):
return 0
return 1
def __setNextFileOffline(self):
"""
"""
idFile= self.fileIndex
while(True):
idFile += 1
if not (idFile < len(self.filenameList)):
self.flagNoMoreFiles = 1
print("No more Files")
return 0
filename = self.filenameList[idFile]
hfFilePointer =h5py.File(filename,'r')
epoc=hfFilePointer['t'].value
#this_time=datetime.datetime(year,month,dom,hour,min,sec)
break
self.flagIsNewFile = 1
self.fileIndex = idFile
self.filename = filename
self.hfFilePointer = hfFilePointer
hfFilePointer.close()
self.__t0=epoc
print("Setting the file: %s"%self.filename)
return 1
def __setNextFileOnline(self):
"""
"""
print("SOY NONE",self.set)
if self.set==None:
pass
else:
self.set +=10
filename = self.filenameList[0]#fullfilename
if self.filename_online != None:
self.__selectDataForTimes(online=True)
filename = self.filenameList[0]
while self.filename_online == filename:
print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
time.sleep(self.__waitForNewFile)
#self.__findDataForDates(online=True)
self.set=self.filename_next_set
self.__selectDataForTimes(online=True)
filename = self.filenameList[0]
sizeoffile=os.path.getsize(filename)
#print filename
sizeoffile=os.path.getsize(filename)
if sizeoffile<1670240:
print("%s is not the rigth size"%filename)
delay=50
print('waiting %d seconds for delay...'%(delay))
time.sleep(delay)
sizeoffile=os.path.getsize(filename)
if sizeoffile<1670240:
delay=50
print('waiting %d more seconds for delay...'%(delay))
time.sleep(delay)
sizeoffile=os.path.getsize(filename)
if sizeoffile<1670240:
delay=50
print('waiting %d more seconds for delay...'%(delay))
time.sleep(delay)
try:
hfFilePointer=h5py.File(filename,'r')
except:
print("Error reading file %s"%filename)
self.filename_online=filename
epoc=hfFilePointer['t'].value
self.hfFilePointer=hfFilePointer
hfFilePointer.close()
self.__t0=epoc
self.flagIsNewFile = 1
self.filename = filename
print("Setting the file: %s"%self.filename)
return 1
def __getExpParameters(self):
if not(self.status):
return None
def setup(self,
path = None,
startDate = None,
endDate = None,
startTime = datetime.time(0,0,0),
endTime = datetime.time(23,59,59),
set = None,
expLabel = "",
ext = None,
all=0,
timezone=0,
online = False,
delay = 60,
walk = True):
'''
In this method we should set all initial parameters.
'''
if path==None:
raise ValueError("The path is not valid")
if ext==None:
ext = self.ext
self.timezone= timezone
self.online= online
self.all=all
#if set==None:
#print set
if not(online):
print("Searching files in offline mode...")
self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk)
else:
print("Searching files in online mode...")
self.searchFilesOnLine(path, walk,ext,set=set)
if set==None:
pass
else:
self.set=set-10
# for nTries in range(self.nTries):
#
# fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
#
# if fullpath:
# break
# print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
# time.sleep(self.delay)
# if not(fullpath):
# print "There ins't valid files in %s" % path
# return None
if not(self.filenameList):
print("There is no files into the folder: %s"%(path))
sys.exit(-1)
self.__getExpParameters()
self.fileIndex = -1
self.__setNextFile(online)
self.__readMetadata()
self.__setLocalVariables()
self.__setHeaderDO()
#self.profileIndex_offset= 0
#self.profileIndex = self.profileIndex_offset
self.isConfig = True
def __readMetadata(self):
self.__readHeader()
def __setLocalVariables(self):
self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
#
self.profileIndex = 9999
def __setHeaderDO(self):
self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
self.dataOut.systemHeaderObj = SystemHeader()
#---------------------------------------------------------
self.dataOut.systemHeaderObj.nProfiles=100
self.dataOut.systemHeaderObj.nSamples=1000
SAMPLING_STRUCTURE=[('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')]
self.dataOut.radarControllerHeaderObj.samplingWindow=numpy.zeros((1,),SAMPLING_STRUCTURE)
self.dataOut.radarControllerHeaderObj.samplingWindow['h0']=0
self.dataOut.radarControllerHeaderObj.samplingWindow['dh']=1.5
self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']=1000
self.dataOut.radarControllerHeaderObj.nHeights=int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'])
self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0']
self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh']
self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']
self.dataOut.radarControllerHeaderObj.nWindows=1
self.dataOut.radarControllerHeaderObj.codetype=0
self.dataOut.radarControllerHeaderObj.numTaus=0
#self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4')
#self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4')
#self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4')
#self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0)
self.dataOut.radarControllerHeaderObj.code_size=0
self.dataOut.nBaud=0
self.dataOut.nCode=0
self.dataOut.nPairs=0
#---------------------------------------------------------
self.dataOut.type = "Voltage"
self.dataOut.data = None
self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
self.dataOut.nProfiles = 1
self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
self.dataOut.channelList = list(range(self.nChannels))
#self.dataOut.channelIndexList = None
self.dataOut.flagNoData = True
#Set to TRUE if the data is discontinuous
self.dataOut.flagDiscontinuousBlock = False
self.dataOut.utctime = None
self.dataOut.timeZone = self.timezone
self.dataOut.dstFlag = 0
self.dataOut.errorCount = 0
self.dataOut.nCohInt = 1
self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
self.dataOut.flagShiftFFT = False
self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
#Time interval between profiles
#self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
self.dataOut.frequency = self.__frequency
self.dataOut.realtime = self.__online
def __hasNotDataInBuffer(self):
if self.profileIndex >= self.nProfiles:
return 1
return 0
def readNextBlock(self):
if not(self.__setNewBlock()):
return 0
if not(self.readBlock()):
return 0
return 1
def __setNewBlock(self):
if self.hfFilePointer==None:
return 0
if self.flagIsNewFile:
return 1
if self.profileIndex < self.nProfiles:
return 1
self.__setNextFile(self.online)
return 1
def readBlock(self):
fp=h5py.File(self.filename,'r')
#Puntero que apunta al archivo hdf5
ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
fp.close()
ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
self.datablock = numpy.array([ch0,ch1])
self.flagIsNewFile=0
self.profileIndex=0
return 1
def getData(self):
if self.flagNoMoreFiles:
self.dataOut.flagNoData = True
print('Process finished')
return 0
if self.__hasNotDataInBuffer():
if not(self.readNextBlock()):
self.dataOut.flagNodata=True
return 0
##############################
##############################
self.dataOut.data = self.datablock[:,:,self.profileIndex]
self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds*self.profileIndex
self.dataOut.profileIndex= self.profileIndex
self.dataOut.flagNoData=False
self.profileIndex +=1
return self.dataOut.data
def run(self, **kwargs):
'''
This method will be called many times so here you should put all your code
'''
if not self.isConfig:
self.setup(**kwargs)
self.isConfig = True
self.getData()