##// END OF EJS Templates
schain
RSS
Se realizar la lectura en modo online llamando al metodo digitalRFReader(self.path) en reemplazo del metodo reload(), grabando previamente el path de lectura o directorio superior donde se almacena la data. Adicionalmente, se ha definido un tiempo de espera de 3 segundos para dar tiempo suficiente al programa de adquisicion de generar archivos. ...
Se realizar la lectura en modo online llamando al metodo digitalRFReader(self.path) en reemplazo del metodo reload(), grabando previamente el path de lectura o directorio superior donde se almacena la data. Adicionalmente, se ha definido un tiempo de espera de 3 segundos para dar tiempo suficiente al programa de adquisicion de generar archivos. El archivo jroIO_digitalRF.py utiliza la libreria digital_rf cuya version actual es la 2.62( 2017 ) ,esta libreria no tiene definido el metodo o clase reload, este metodo existe en la version 2.0(2014), si uno revisa el archivo jroIO_usrp.py, esta unidad de lectura trabaja con la version 2.0 llamada digital_rf_hdf5, para hacer uso de esta unidad de lectura se instalan los programas correspondiente pero el formato y la informacion difiere un poco de la version actual. Se infiere entonces que al desarrollar del archivo jroIO_digitalRF.py, esperaba que la libreria aun tenga incluido el metodo reload con el update de las versiones pero este ya no es parte del desarrollo, Se realizo la consulta al desarrollador actual de digitalRF Ryan Voltz si se iba a incluir a futuro pero indico que no era necesario.
Juan C. Espinoza - - Load All Authors

File last commit:

r1052:3fb7c359028f
r1234:b6a76136b1f3
Show More
jroIO_hf.py
863 lines | 24.5 KiB | text/x-python | PythonLexer
/ schainpy / model / io / jroIO_hf.py
History | Annotation | Raw |Copy content |Copy permalink
'''
Created on Jul 3, 2014
@author: roj-com0419
'''
import os,sys
import time,datetime
import h5py
import numpy
import fnmatch
import re
from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
from schainpy.model.data.jrodata import Voltage
from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
def isNumber(str):
"""
Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
Excepciones:
Si un determinado string no puede ser convertido a numero
Input:
str, string al cual se le analiza para determinar si convertible a un numero o no
Return:
True : si el string es uno numerico
False : no es un string numerico
"""
try:
float( str )
return True
except:
return False
def getFileFromSet(path, ext, set=None):
validFilelist = []
fileList = os.listdir(path)
if len(fileList) < 1:
return None
# 0 1234 567 89A BCDE
# H YYYY DDD SSS .ext
for thisFile in fileList:
try:
number= int(thisFile[6:16])
# year = int(thisFile[1:5])
# doy = int(thisFile[5:8])
except:
continue
if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
continue
validFilelist.append(thisFile)
if len(validFilelist) < 1:
return None
validFilelist = sorted( validFilelist, key=str.lower )
if set == None:
return validFilelist[-1]
print "set =" ,set
for thisFile in validFilelist:
if set <= int(thisFile[6:16]):
print thisFile,int(thisFile[6:16])
return thisFile
return validFilelist[-1]
myfile = fnmatch.filter(validFilelist,'*%10d*'%(set))
#myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
if len(myfile)!= 0:
return myfile[0]
else:
filename = '*%10.10d%s'%(set,ext.lower())
print 'the filename %s does not exist'%filename
print '...going to the last file: '
if validFilelist:
validFilelist = sorted( validFilelist, key=str.lower )
return validFilelist[-1]
return None
def getlastFileFromPath(path, ext):
"""
Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
al final de la depuracion devuelve el ultimo file de la lista que quedo.
Input:
fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
ext : extension de los files contenidos en una carpeta
Return:
El ultimo file de una determinada carpeta, no se considera el path.
"""
validFilelist = []
fileList = os.listdir(path)
# 0 1234 567 89A BCDE
# H YYYY DDD SSS .ext
for thisFile in fileList:
try:
number= int(thisFile[6:16])
except:
print "There is a file or folder with different format"
if not isNumber(number):
continue
# year = thisFile[1:5]
# if not isNumber(year):
# continue
# doy = thisFile[5:8]
# if not isNumber(doy):
# continue
number= int(number)
# year = int(year)
# doy = int(doy)
if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
continue
validFilelist.append(thisFile)
if validFilelist:
validFilelist = sorted( validFilelist, key=str.lower )
return validFilelist[-1]
return None
class HFReader(ProcessingUnit):
'''
classdocs
'''
path = None
startDate= None
endDate = None
startTime= None
endTime = None
walk = None
isConfig = False
dataOut=None
nTries = 3
ext = ".hdf5"
def __init__(self, **kwargs):
'''
Constructor
'''
ProcessingUnit.__init__(self, **kwargs)
self.isConfig =False
self.datablock = None
self.filename_current=None
self.utc = 0
self.ext='.hdf5'
self.flagIsNewFile = 1
#-------------------------------------------------
self.fileIndex=None
self.profileIndex_offset=None
self.filenameList=[]
self.hfFilePointer= None
self.filename_online = None
self.status=True
self.flagNoMoreFiles= False
self.__waitForNewFile = 20
#--------------------------------------------------
self.dataOut = self.createObjByDefault()
def createObjByDefault(self):
dataObj = Voltage()
return dataObj
def setObjProperties(self):
pass
def getBlockDimension(self):
"""
Obtiene la cantidad de puntos a leer por cada bloque de datos
Affected:
self.blocksize
Return:
None
"""
pts2read =self.nChannels*self.nHeights*self.nProfiles
self.blocksize = pts2read
def __readHeader(self):
self.nProfiles = 100
self.nHeights = 1000
self.nChannels = 2
self.__firstHeigth=0
self.__nSamples=1000
self.__deltaHeigth=1.5
self.__sample_rate=1e5
#self.__frequency=2.72e6
#self.__frequency=3.64e6
self.__frequency=None
self.__online = False
self.filename_next_set=None
#print "Frequency of Operation:", self.__frequency
def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
self.path = path
self.startDate = startDate
self.endDate = endDate
self.startTime = startTime
self.endTime = endTime
self.walk = walk
def __checkPath(self):
if os.path.exists(self.path):
self.status=1
else:
self.status=0
print 'Path %s does not exits'%self.path
return
return
def __selDates(self, hf_dirname_format):
try:
dir_hf_filename= self.path+"/"+hf_dirname_format
fp= h5py.File(dir_hf_filename,'r')
hipoc=fp['t'].value
fp.close()
date_time=datetime.datetime.utcfromtimestamp(hipoc)
year =int(date_time[0:4])
month=int(date_time[5:7])
dom =int(date_time[8:10])
thisDate= datetime.date(year,month,dom)
if (thisDate>=self.startDate and thisDate <= self.endDate):
return hf_dirname_format
except:
return None
def __findDataForDates(self,online=False):
if not(self.status):
return None
pat = '\d+.\d+'
dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
dirnameList = filter(lambda x:x!=None,dirnameList)
dirnameList = [x.string for x in dirnameList]
if not(online):
dirnameList = [self.__selDates(x) for x in dirnameList]
dirnameList = filter(lambda x:x!=None,dirnameList)
if len(dirnameList)>0:
self.status = 1
self.dirnameList = dirnameList
self.dirnameList.sort()
else:
self.status = 0
return None
def __getTimeFromData(self):
startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
print '........................................'
filter_filenameList=[]
self.filenameList.sort()
for i in range(len(self.filenameList)-1):
filename=self.filenameList[i]
dir_hf_filename= filename
fp= h5py.File(dir_hf_filename,'r')
hipoc=fp['t'].value
hipoc=hipoc+self.timezone
date_time=datetime.datetime.utcfromtimestamp(hipoc)
fp.close()
year =int(date_time[0:4])
month=int(date_time[5:7])
dom =int(date_time[8:10])
hour =int(date_time[11:13])
min =int(date_time[14:16])
sec =int(date_time[17:19])
this_time=datetime.datetime(year,month,dom,hour,min,sec)
if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
filter_filenameList.append(filename)
filter_filenameList.sort()
self.filenameList = filter_filenameList
return 1
def __getFilenameList(self):
#print "hola"
#print self.dirnameList
dirList = [os.path.join(self.path,x) for x in self.dirnameList]
self.filenameList= dirList
#print self.filenameList
#print "pase",len(self.filenameList)
def __selectDataForTimes(self, online=False):
if not(self.status):
return None
#----------------
self.__getFilenameList()
#----------------
if not(online):
if not(self.all):
self.__getTimeFromData()
if len(self.filenameList)>0:
self.status=1
self.filenameList.sort()
else:
self.status=0
return None
else:
if self.set != None:
filename=getFileFromSet(self.path,self.ext,self.set)
if self.flag_nextfile==True:
self.dirnameList=[filename]
fullfilename=self.path+"/"+filename
self.filenameList=[fullfilename]
self.filename_next_set=int(filename[6:16])+10
self.flag_nextfile=False
else:
print filename
print "PRIMERA CONDICION"
#if self.filename_next_set== int(filename[6:16]):
print "TODO BIEN"
if filename == None:
raise ValueError, "corregir"
self.dirnameList=[filename]
fullfilename=self.path+"/"+filename
self.filenameList=[fullfilename]
self.filename_next_set=int(filename[6:16])+10
print "Setting next file",self.filename_next_set
self.set=int(filename[6:16])
if True:
pass
else:
print "ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO"
else:
filename =getlastFileFromPath(self.path,self.ext)
if self.flag_nextfile==True:
self.dirnameList=[filename]
fullfilename=self.path+"/"+filename
self.filenameList=[self.filenameList[-1]]
self.filename_next_set=int(filename[6:16])+10
self.flag_nextfile=False
else:
filename=getFileFromSet(self.path,self.ext,self.set)
print filename
print "PRIMERA CONDICION"
#if self.filename_next_set== int(filename[6:16]):
print "TODO BIEN"
if filename == None:
raise ValueError, "corregir"
self.dirnameList=[filename]
fullfilename=self.path+"/"+filename
self.filenameList=[fullfilename]
self.filename_next_set=int(filename[6:16])+10
print "Setting next file",self.filename_next_set
self.set=int(filename[6:16])
if True:
pass
else:
print "ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO"
def searchFilesOffLine(self,
path,
startDate,
endDate,
ext,
startTime=datetime.time(0,0,0),
endTime=datetime.time(23,59,59),
walk=True):
self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
self.__checkPath()
self.__findDataForDates()
#print self.dirnameList
self.__selectDataForTimes()
for i in range(len(self.filenameList)):
print "%s"% (self.filenameList[i])
return
def searchFilesOnLine(self,
path,
expLabel= "",
ext=None,
startDate=None,
endDate=None,
walk=True,
set=None):
startDate = datetime.datetime.utcnow().date()
endDate = datetime.datetime.utcnow().date()
self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
self.__checkPath()
fullpath=path
print "%s folder was found: " %(fullpath )
if set == None:
self.set=None
filename =getlastFileFromPath(fullpath,ext)
startDate= datetime.datetime.utcnow().date
endDate= datetime.datetime.utcnow().date()
#
else:
filename= getFileFromSet(fullpath,ext,set)
startDate=None
endDate=None
#
if not (filename):
return None,None,None,None,None
#print "%s file was found" %(filename)
#
# dir_hf_filename= self.path+"/"+filename
# fp= h5py.File(dir_hf_filename,'r')
# hipoc=fp['t'].value
# fp.close()
# date_time=datetime.datetime.utcfromtimestamp(hipoc)
#
# year =int(date_time[0:4])
# month=int(date_time[5:7])
# dom =int(date_time[8:10])
# set= int(filename[4:10])
# self.set=set-1
#self.dirnameList=[filename]
filenameList= fullpath+"/"+filename
self.dirnameList=[filename]
self.filenameList=[filenameList]
self.flag_nextfile=True
#self.__findDataForDates(online=True)
#self.dirnameList=[self.dirnameList[-1]]
#print self.dirnameList
#self.__selectDataForTimes(online=True)
#return fullpath,filename,year,month,dom,set
return
def __setNextFile(self,online=False):
"""
"""
if not(online):
newFile = self.__setNextFileOffline()
else:
newFile = self.__setNextFileOnline()
if not(newFile):
return 0
return 1
def __setNextFileOffline(self):
"""
"""
idFile= self.fileIndex
while(True):
idFile += 1
if not (idFile < len(self.filenameList)):
self.flagNoMoreFiles = 1
print "No more Files"
return 0
filename = self.filenameList[idFile]
hfFilePointer =h5py.File(filename,'r')
epoc=hfFilePointer['t'].value
#this_time=datetime.datetime(year,month,dom,hour,min,sec)
break
self.flagIsNewFile = 1
self.fileIndex = idFile
self.filename = filename
self.hfFilePointer = hfFilePointer
hfFilePointer.close()
self.__t0=epoc
print "Setting the file: %s"%self.filename
return 1
def __setNextFileOnline(self):
"""
"""
print "SOY NONE",self.set
if self.set==None:
pass
else:
self.set +=10
filename = self.filenameList[0]#fullfilename
if self.filename_online != None:
self.__selectDataForTimes(online=True)
filename = self.filenameList[0]
while self.filename_online == filename:
print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
time.sleep(self.__waitForNewFile)
#self.__findDataForDates(online=True)
self.set=self.filename_next_set
self.__selectDataForTimes(online=True)
filename = self.filenameList[0]
sizeoffile=os.path.getsize(filename)
#print filename
sizeoffile=os.path.getsize(filename)
if sizeoffile<1670240:
print "%s is not the rigth size"%filename
delay=50
print 'waiting %d seconds for delay...'%(delay)
time.sleep(delay)
sizeoffile=os.path.getsize(filename)
if sizeoffile<1670240:
delay=50
print 'waiting %d more seconds for delay...'%(delay)
time.sleep(delay)
sizeoffile=os.path.getsize(filename)
if sizeoffile<1670240:
delay=50
print 'waiting %d more seconds for delay...'%(delay)
time.sleep(delay)
try:
hfFilePointer=h5py.File(filename,'r')
except:
print "Error reading file %s"%filename
self.filename_online=filename
epoc=hfFilePointer['t'].value
self.hfFilePointer=hfFilePointer
hfFilePointer.close()
self.__t0=epoc
self.flagIsNewFile = 1
self.filename = filename
print "Setting the file: %s"%self.filename
return 1
def __getExpParameters(self):
if not(self.status):
return None
def setup(self,
path = None,
startDate = None,
endDate = None,
startTime = datetime.time(0,0,0),
endTime = datetime.time(23,59,59),
set = None,
expLabel = "",
ext = None,
all=0,
timezone=0,
online = False,
delay = 60,
walk = True):
'''
In this method we should set all initial parameters.
'''
if path==None:
raise ValueError,"The path is not valid"
if ext==None:
ext = self.ext
self.timezone= timezone
self.online= online
self.all=all
#if set==None:
#print set
if not(online):
print "Searching files in offline mode..."
self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk)
else:
print "Searching files in online mode..."
self.searchFilesOnLine(path, walk,ext,set=set)
if set==None:
pass
else:
self.set=set-10
# for nTries in range(self.nTries):
#
# fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
#
# if fullpath:
# break
# print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
# time.sleep(self.delay)
# if not(fullpath):
# print "There ins't valid files in %s" % path
# return None
if not(self.filenameList):
print "There is no files into the folder: %s"%(path)
sys.exit(-1)
self.__getExpParameters()
self.fileIndex = -1
self.__setNextFile(online)
self.__readMetadata()
self.__setLocalVariables()
self.__setHeaderDO()
#self.profileIndex_offset= 0
#self.profileIndex = self.profileIndex_offset
self.isConfig = True
def __readMetadata(self):
self.__readHeader()
def __setLocalVariables(self):
self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
#
self.profileIndex = 9999
def __setHeaderDO(self):
self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
self.dataOut.systemHeaderObj = SystemHeader()
#---------------------------------------------------------
self.dataOut.systemHeaderObj.nProfiles=100
self.dataOut.systemHeaderObj.nSamples=1000
SAMPLING_STRUCTURE=[('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')]
self.dataOut.radarControllerHeaderObj.samplingWindow=numpy.zeros((1,),SAMPLING_STRUCTURE)
self.dataOut.radarControllerHeaderObj.samplingWindow['h0']=0
self.dataOut.radarControllerHeaderObj.samplingWindow['dh']=1.5
self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']=1000
self.dataOut.radarControllerHeaderObj.nHeights=int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'])
self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0']
self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh']
self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']
self.dataOut.radarControllerHeaderObj.nWindows=1
self.dataOut.radarControllerHeaderObj.codetype=0
self.dataOut.radarControllerHeaderObj.numTaus=0
#self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4')
#self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4')
#self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4')
#self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0)
self.dataOut.radarControllerHeaderObj.code_size=0
self.dataOut.nBaud=0
self.dataOut.nCode=0
self.dataOut.nPairs=0
#---------------------------------------------------------
self.dataOut.type = "Voltage"
self.dataOut.data = None
self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
self.dataOut.nProfiles = 1
self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
self.dataOut.channelList = range(self.nChannels)
#self.dataOut.channelIndexList = None
self.dataOut.flagNoData = True
#Set to TRUE if the data is discontinuous
self.dataOut.flagDiscontinuousBlock = False
self.dataOut.utctime = None
self.dataOut.timeZone = self.timezone
self.dataOut.dstFlag = 0
self.dataOut.errorCount = 0
self.dataOut.nCohInt = 1
self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
self.dataOut.flagShiftFFT = False
self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
#Time interval between profiles
#self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
self.dataOut.frequency = self.__frequency
self.dataOut.realtime = self.__online
def __hasNotDataInBuffer(self):
if self.profileIndex >= self.nProfiles:
return 1
return 0
def readNextBlock(self):
if not(self.__setNewBlock()):
return 0
if not(self.readBlock()):
return 0
return 1
def __setNewBlock(self):
if self.hfFilePointer==None:
return 0
if self.flagIsNewFile:
return 1
if self.profileIndex < self.nProfiles:
return 1
self.__setNextFile(self.online)
return 1
def readBlock(self):
fp=h5py.File(self.filename,'r')
#Puntero que apunta al archivo hdf5
ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
fp.close()
ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
self.datablock = numpy.array([ch0,ch1])
self.flagIsNewFile=0
self.profileIndex=0
return 1
def getData(self):
if self.flagNoMoreFiles:
self.dataOut.flagNoData = True
print 'Process finished'
return 0
if self.__hasNotDataInBuffer():
if not(self.readNextBlock()):
self.dataOut.flagNodata=True
return 0
##############################
##############################
self.dataOut.data = self.datablock[:,:,self.profileIndex]
self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds*self.profileIndex
self.dataOut.profileIndex= self.profileIndex
self.dataOut.flagNoData=False
self.profileIndex +=1
return self.dataOut.data
def run(self, **kwargs):
'''
This method will be called many times so here you should put all your code
'''
if not self.isConfig:
self.setup(**kwargs)
self.isConfig = True
self.getData()