diff --git a/schainpy/model/data/jroamisr.py b/schainpy/model/data/jroamisr.py new file mode 100644 index 0000000..2fab480 --- /dev/null +++ b/schainpy/model/data/jroamisr.py @@ -0,0 +1,59 @@ +import numpy + +class AMISR: + def __init__(self): + self.flagNoData = True + self.data = None + self.utctime = None + self.type = "AMISR" + + #propiedades para compatibilidad con Voltages + self.timeZone = 300#timezone like jroheader, difference in minutes between UTC and localtime + self.dstFlag = 0#self.dataIn.dstFlag + self.errorCount = 0#self.dataIn.errorCount + self.useLocalTime = True#self.dataIn.useLocalTime + + self.radarControllerHeaderObj = None#self.dataIn.radarControllerHeaderObj.copy() + self.systemHeaderObj = None#self.dataIn.systemHeaderObj.copy() + self.channelList = [0]#self.dataIn.channelList esto solo aplica para el caso de AMISR + self.dtype = numpy.dtype([('real','=self.startDate and thisDate <= self.endDate): + return amisr_dirname_format + except: + return None + + def __findDataForDates(self): + + + + if not(self.status): + return None + + pat = '\d+.\d+' + dirnameList = [re.search(pat,x) for x in os.listdir(self.path)] + dirnameList = filter(lambda x:x!=None,dirnameList) + dirnameList = [x.string for x in dirnameList] + dirnameList = [self.__selDates(x) for x in dirnameList] + dirnameList = filter(lambda x:x!=None,dirnameList) + if len(dirnameList)>0: + self.status = 1 + self.dirnameList = dirnameList + self.dirnameList.sort() + else: + self.status = 0 + return None + + def __getTimeFromData(self): + pass + + def __filterByGlob1(self, dirName): + filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file) + filterDict = {} + filterDict.setdefault(dirName) + filterDict[dirName] = filter_files + return filterDict + + def __getFilenameList(self, fileListInKeys, dirList): + for value in fileListInKeys: + dirName = value.keys()[0] + for file in value[dirName]: + filename = os.path.join(dirName, file) + self.filenameList.append(filename) + + + def __selectDataForTimes(self): + #aun no esta implementado el filtro for tiempo + if not(self.status): + return None + + dirList = [os.path.join(self.path,x) for x in self.dirnameList] + + fileListInKeys = [self.__filterByGlob1(x) for x in dirList] + + self.__getFilenameList(fileListInKeys, dirList) + + if len(self.filenameList)>0: + self.status = 1 + self.filenameList.sort() + else: + self.status = 0 + return None + + + def __searchFilesOffline(self, + path, + startDate, + endDate, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + walk=True): + + self.__setParameters(path, startDate, endDate, startTime, endTime, walk) + + self.__checkPath() + + self.__findDataForDates() + + self.__selectDataForTimes() + + for i in range(len(self.filenameList)): + print "%s" %(self.filenameList[i]) + + return + + def __setNextFileOffline(self): + idFile = self.fileIndex + + while (True): + idFile += 1 + if not(idFile < len(self.filenameList)): + self.flagNoMoreFiles = 1 + print "No more Files" + return 0 + + filename = self.filenameList[idFile] + + amisrFilePointer = h5py.File(filename,'r') + + break + + self.flagIsNewFile = 1 + self.fileIndex = idFile + self.filename = filename + + self.amisrFilePointer = amisrFilePointer + + print "Setting the file: %s"%self.filename + + return 1 + + def __readHeader(self): + self.radacHeaderObj = RadacHeader(self.amisrFilePointer) + + #update values from experiment cfg file + if self.radacHeaderObj.nrecords == self.recordsperfile_fromfile: + self.radacHeaderObj.nrecords = self.recordsperfile_fromfile + self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile + self.radacHeaderObj.npulses = self.npulsesint_fromfile + self.radacHeaderObj.nsamples = self.ngates_fromfile + + #get tuning frequency + frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency') + self.frequency_h5file = frequency_h5file_dataset[0,0] + + self.flagIsNewFile = 1 + + def __getBeamCode(self): + self.beamCodeDict = {} + self.beamRangeDict = {} + + for i in range(len(self.radacHeaderObj.beamCode[0,:])): + self.beamCodeDict.setdefault(i) + self.beamRangeDict.setdefault(i) + self.beamCodeDict[i] = self.radacHeaderObj.beamCode[0,i] + + + just4record0 = self.radacHeaderObj.beamCodeByPulse[0,:] + + for i in range(len(self.beamCodeDict.values())): + xx = numpy.where(just4record0==self.beamCodeDict.values()[i]) + self.beamRangeDict[i] = xx[0] + + def __getExpParameters(self): + if not(self.status): + return None + + experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup') + + expFinder = glob.glob1(experimentCfgPath,'*.exp') + if len(expFinder)== 0: + self.status = 0 + return None + + experimentFilename = os.path.join(experimentCfgPath,expFinder[0]) + + f = open(experimentFilename) + lines = f.readlines() + f.close() + + parmsList = ['npulsesint*','recordsperfile*','nbeamcodes*','ngates*'] + filterList = [fnmatch.filter(lines, x) for x in parmsList] + + + values = [re.sub(r'\D',"",x[0]) for x in filterList] + + self.npulsesint_fromfile = int(values[0]) + self.recordsperfile_fromfile = int(values[1]) + self.nbeamcodes_fromfile = int(values[2]) + self.ngates_fromfile = int(values[3]) + + tufileFinder = fnmatch.filter(lines, 'tufile=*') + tufile = tufileFinder[0].split('=')[1].split('\n')[0] + tufilename = os.path.join(experimentCfgPath,tufile) + + f = open(tufilename) + lines = f.readlines() + f.close() + self.ippSeconds_fromfile = float(lines[1].split()[2])/1E6 + + + self.status = 1 + + def __setIdsAndArrays(self): + self.dataByFrame = self.__setDataByFrame() + self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :] + self.readRanges() + self.idpulse_range1, self.idpulse_range2 = self.radacHeaderObj.getIndexRangeToPulse(0) + self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses) + self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame) + + + def __setNextFile(self): + + newFile = self.__setNextFileOffline() + + if not(newFile): + return 0 + + self.__readHeader() + + if self.__firstFile: + self.__setIdsAndArrays() + self.__firstFile = False + + self.__getBeamCode() + self.readDataBlock() + + + def setup(self,path=None, + startDate=None, + endDate=None, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + walk=True): + + #Busqueda de archivos offline + self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk) + + if not(self.filenameList): + print "There is no files into the folder: %s"%(path) + + sys.exit(-1) + + self.__getExpParameters() + + self.fileIndex = -1 + + self.__setNextFile() + + def readRanges(self): + dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range') + #self.rangeFromFile = dataset.value + self.rangeFromFile = numpy.reshape(dataset.value,(-1)) + return range + + + def readRadacTime(self,idrecord, range1, range2): + self.radacTimeFromFile = self.radacHeaderObj.radacTime.value + + radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses)) + #radacTimeByFrame = dataset[idrecord - 1,range1] + #radacTimeByFrame = dataset[idrecord,range2] + + return radacTimeByFrame + + def readBeamCode(self, idrecord, range1, range2): + dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode') + beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses)) + self.beamCodesFromFile = dataset.value + + #beamcodeByFrame[range1] = dataset[idrecord - 1, range1] + #beamcodeByFrame[range2] = dataset[idrecord, range2] + beamcodeByFrame[range1] = dataset[idrecord, range1] + beamcodeByFrame[range2] = dataset[idrecord, range2] + + return beamcodeByFrame + + + def __setDataByFrame(self): + ndata = 2 # porque es complejo + dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata)) + return dataByFrame + + def __readDataSet(self): + dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data') + return dataset + + def __setDataBlock(self,): + real = self.dataByFrame[:,:,0] #asumo que 0 es real + imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario + datablock = real + imag*1j #armo el complejo + return datablock + + def readSamples_version1(self,idrecord): + #estas tres primeras lineas solo se deben ejecutar una vez + if self.flagIsNewFile: + #reading dataset + self.dataset = self.__readDataSet() + self.flagIsNewFile = 0 + + if idrecord == 0: + #if self.buffer_last_record == None: + selectorById = self.radacHeaderObj.pulseCount[0,self.idpulse_range2] + + self.dataByFrame[selectorById,:,:] = self.dataset[0, self.idpulse_range2,:,:] + + self.radacTimeByFrame[selectorById] = self.radacHeaderObj.radacTime[0, self.idpulse_range2] + + selectorById = self.radacHeaderObj.pulseCount[0,self.idpulse_range1] + + self.radacTimeByFrame[selectorById] = self.buffer_radactime[selectorById] + + datablock = self.__setDataBlock() + + return datablock + + selectorById = self.radacHeaderObj.pulseCount[idrecord-1,self.idpulse_range1] + self.dataByFrame[selectorById,:,:] = self.dataset[idrecord-1, self.idpulse_range1, :, :] + self.radacTimeByFrame[selectorById] = self.radacHeaderObj.radacTime[idrecord-1, self.idpulse_range1] + + selectorById = self.radacHeaderObj.pulseCount[idrecord,self.idpulse_range2]#data incompleta ultimo archivo de carpeta, verifica el record real segun la dimension del arreglo de datos + self.dataByFrame[selectorById,:,:] = self.dataset[idrecord, self.idpulse_range2, :, :] + self.radacTimeByFrame[selectorById] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range2] + + datablock = self.__setDataBlock() + + selectorById = self.radacHeaderObj.pulseCount[idrecord,self.idpulse_range1] + self.dataByFrame[selectorById,:,:] = self.dataset[idrecord, self.idpulse_range1, :, :] + self.buffer_radactime[selectorById] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range1] + + return datablock + + + def readSamples(self,idrecord): + if self.flagIsNewFile: + self.dataByFrame = self.__setDataByFrame() + self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :] + + #reading ranges + self.readRanges() + #reading dataset + self.dataset = self.__readDataSet() + + self.flagIsNewFile = 0 + self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :] + self.dataByFrame = self.dataset[idrecord, :, :, :] + datablock = self.__setDataBlock() + return datablock + + + def readDataBlock(self): + + self.datablock = self.readSamples_version1(self.idrecord_count) + #self.datablock = self.readSamples(self.idrecord_count) + #print 'record:', self.idrecord_count + + self.idrecord_count += 1 + self.profileIndex = 0 + + if self.idrecord_count >= self.radacHeaderObj.nrecords: + self.idrecord_count = 0 + self.flagIsNewFile = 1 + + def readNextBlock(self): + + self.readDataBlock() + + if self.flagIsNewFile: + self.__setNextFile() + pass + + def __hasNotDataInBuffer(self): + #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record + if self.profileIndex >= self.radacHeaderObj.npulses: + return 1 + return 0 + + def printUTC(self): + print self.dataOut.utctime + print '' + + def setObjProperties(self): + self.dataOut.heightList = self.rangeFromFile/1000.0 #km + self.dataOut.nProfiles = self.radacHeaderObj.npulses + self.dataOut.nRecords = self.radacHeaderObj.nrecords + self.dataOut.nBeams = self.radacHeaderObj.nbeams + self.dataOut.ippSeconds = self.ippSeconds_fromfile + self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt + self.dataOut.frequency = self.frequency_h5file + self.dataOut.nBaud = None + self.dataOut.nCode = None + self.dataOut.code = None + + self.dataOut.beamCodeDict = self.beamCodeDict + self.dataOut.beamRangeDict = self.beamRangeDict + + def getData(self): + + if self.flagNoMoreFiles: + self.dataOut.flagNoData = True + print 'Process finished' + return 0 + + if self.__hasNotDataInBuffer(): + self.readNextBlock() + + + if self.datablock == None: # setear esta condicion cuando no hayan datos por leers + self.dataOut.flagNoData = True + return 0 + + self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1)) + + self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex] + + self.dataOut.flagNoData = False + + self.profileIndex += 1 + + return self.dataOut.data + + + def run(self, **kwargs): + if not(self.isConfig): + self.setup(**kwargs) + self.setObjProperties() + self.isConfig = True + + self.getData() diff --git a/schainpy/model/io/jroIO_base.py b/schainpy/model/io/jroIO_base.py index fd15626..22595f5 100644 --- a/schainpy/model/io/jroIO_base.py +++ b/schainpy/model/io/jroIO_base.py @@ -8,7 +8,7 @@ import time import numpy import fnmatch import time, datetime -import h5py +#import h5py import traceback #try: diff --git a/schainpy/model/io/jrodataIO.py b/schainpy/model/io/jrodataIO.py index 8d1b368..43a7263 100644 --- a/schainpy/model/io/jrodataIO.py +++ b/schainpy/model/io/jrodataIO.py @@ -1,3 +1,4 @@ from jroIO_voltage import * from jroIO_spectra import * from jroIO_heispectra import * +from jroIO_amisr import * \ No newline at end of file diff --git a/schainpy/model/proc/jroproc_amisr.py b/schainpy/model/proc/jroproc_amisr.py new file mode 100644 index 0000000..a5ad0d4 --- /dev/null +++ b/schainpy/model/proc/jroproc_amisr.py @@ -0,0 +1,82 @@ +''' +@author: Daniel Suarez +''' + +from jroproc_base import ProcessingUnit, Operation +from model.data.jroamisr import AMISR + +class AMISRProc(ProcessingUnit): + def __init__(self): + ProcessingUnit.__init__(self) + self.objectDict = {} + self.dataOut = AMISR() + + def run(self): + if self.dataIn.type == 'AMISR': + self.dataOut.copy(self.dataIn) + + +class PrintInfo(Operation): + def __init__(self): + pass + + def run(self, dataOut): + + print 'Number of Records by File: %d'%dataOut.nRecords + print 'Number of Pulses: %d'%dataOut.nProfiles + print 'Number of Samples by Pulse: %d'%len(dataOut.heightList) + print 'Ipp Seconds: %f'%dataOut.ippSeconds + print 'Number of Beams: %d'%dataOut.nBeams + print 'BeamCodes:' + beamStrList = ['Beam %d -> Code %d'%(k,v) for k,v in dataOut.beamCodeDict.items()] + for b in beamStrList: + print b + + +class BeamSelector(Operation): + profileIndex = None + nProfiles = None + + def __init__(self): + + self.profileIndex = 0 + + def incIndex(self): + self.profileIndex += 1 + + if self.profileIndex >= self.nProfiles: + self.profileIndex = 0 + + def isProfileInRange(self, minIndex, maxIndex): + + if self.profileIndex < minIndex: + return False + + if self.profileIndex > maxIndex: + return False + + return True + + def isProfileInList(self, profileList): + + if self.profileIndex not in profileList: + return False + + return True + + def run(self, dataOut, beam=None): + + dataOut.flagNoData = True + self.nProfiles = dataOut.nProfiles + + if beam != None: + if self.isProfileInList(dataOut.beamRangeDict[beam]): + dataOut.flagNoData = False + + self.incIndex() + return 1 + + else: + raise ValueError, "BeamSelector needs beam value" + + return 0 \ No newline at end of file diff --git a/schainpy/model/proc/jroproc_spectra.py b/schainpy/model/proc/jroproc_spectra.py index 813d9ce..fef9f4b 100644 --- a/schainpy/model/proc/jroproc_spectra.py +++ b/schainpy/model/proc/jroproc_spectra.py @@ -2,6 +2,7 @@ import numpy from jroproc_base import ProcessingUnit, Operation from model.data.jrodata import Spectra +from model.data.jrodata import hildebrand_sekhon class SpectraProc(ProcessingUnit): diff --git a/schainpy/model/proc/jroproc_voltage.py b/schainpy/model/proc/jroproc_voltage.py index 3c5853b..f1a9a4f 100644 --- a/schainpy/model/proc/jroproc_voltage.py +++ b/schainpy/model/proc/jroproc_voltage.py @@ -15,26 +15,32 @@ class VoltageProc(ProcessingUnit): self.flip = 1 def run(self): - self.dataOut.copy(self.dataIn) + if self.dataIn.type == 'AMISR': + self.__updateObjFromAmisrInput() + + if self.dataIn.type == 'Voltage': + self.dataOut.copy(self.dataIn) + +# self.dataOut.copy(self.dataIn) -# def __updateObjFromAmisrInput(self): -# -# self.dataOut.timeZone = self.dataIn.timeZone -# self.dataOut.dstFlag = self.dataIn.dstFlag -# self.dataOut.errorCount = self.dataIn.errorCount -# self.dataOut.useLocalTime = self.dataIn.useLocalTime -# -# self.dataOut.flagNoData = self.dataIn.flagNoData -# self.dataOut.data = self.dataIn.data -# self.dataOut.utctime = self.dataIn.utctime -# self.dataOut.channelList = self.dataIn.channelList -# self.dataOut.timeInterval = self.dataIn.timeInterval -# self.dataOut.heightList = self.dataIn.heightList -# self.dataOut.nProfiles = self.dataIn.nProfiles -# -# self.dataOut.nCohInt = self.dataIn.nCohInt -# self.dataOut.ippSeconds = self.dataIn.ippSeconds -# self.dataOut.frequency = self.dataIn.frequency + def __updateObjFromAmisrInput(self): + + self.dataOut.timeZone = self.dataIn.timeZone + self.dataOut.dstFlag = self.dataIn.dstFlag + self.dataOut.errorCount = self.dataIn.errorCount + self.dataOut.useLocalTime = self.dataIn.useLocalTime + + self.dataOut.flagNoData = self.dataIn.flagNoData + self.dataOut.data = self.dataIn.data + self.dataOut.utctime = self.dataIn.utctime + self.dataOut.channelList = self.dataIn.channelList + self.dataOut.timeInterval = self.dataIn.timeInterval + self.dataOut.heightList = self.dataIn.heightList + self.dataOut.nProfiles = self.dataIn.nProfiles + + self.dataOut.nCohInt = self.dataIn.nCohInt + self.dataOut.ippSeconds = self.dataIn.ippSeconds + self.dataOut.frequency = self.dataIn.frequency # # pass# # diff --git a/schainpy/model/proc/jroprocessing.py b/schainpy/model/proc/jroprocessing.py index ed54ca9..ab197bb 100644 --- a/schainpy/model/proc/jroprocessing.py +++ b/schainpy/model/proc/jroprocessing.py @@ -1,3 +1,4 @@ from jroproc_voltage import * from jroproc_spectra import * -from jroproc_heispectra import * \ No newline at end of file +from jroproc_heispectra import * +from jroproc_amisr import * \ No newline at end of file diff --git a/schainpy/test/amisr_proc.py b/schainpy/test/amisr_proc.py index d9e980f..f931f4a 100644 --- a/schainpy/test/amisr_proc.py +++ b/schainpy/test/amisr_proc.py @@ -17,7 +17,7 @@ path = os.path.join(os.environ['HOME'],'Documents/amisr') figpath = os.path.join(os.environ['HOME'],'Pictures/amisr') -readUnitConfObj = controllerObj.addReadUnit(datatype='AMISR', +readUnitConfObj = controllerObj.addReadUnit(datatype='AMISRReader', path=path, startDate='2014/08/18', endDate='2014/08/18', @@ -26,19 +26,19 @@ readUnitConfObj = controllerObj.addReadUnit(datatype='AMISR', walk=1) #AMISR Processing Unit -procUnitAMISRBeam0 = controllerObj.addProcUnit(datatype='AMISR', inputId=readUnitConfObj.getId()) +procUnitAMISRBeam0 = controllerObj.addProcUnit(datatype='AMISRProc', inputId=readUnitConfObj.getId()) #Beam Selector opObj11 = procUnitAMISRBeam0.addOperation(name='BeamSelector', optype='other') opObj11.addParameter(name='beam', value='0', format='int') #Voltage Processing Unit -procUnitConfObjBeam0 = controllerObj.addProcUnit(datatype='Voltage', inputId=procUnitAMISRBeam0.getId()) +procUnitConfObjBeam0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=procUnitAMISRBeam0.getId()) #Coherent Integration opObj11 = procUnitConfObjBeam0.addOperation(name='CohInt', optype='other') opObj11.addParameter(name='n', value='128', format='int') #Spectra Unit Processing, getting spectras with nProfiles and nFFTPoints -procUnitConfObjSpectraBeam0 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObjBeam0.getId()) +procUnitConfObjSpectraBeam0 = controllerObj.addProcUnit(datatype='SpectraProc', inputId=procUnitConfObjBeam0.getId()) procUnitConfObjSpectraBeam0.addParameter(name='nFFTPoints', value=32, format='int') procUnitConfObjSpectraBeam0.addParameter(name='nProfiles', value=32, format='int') #Noise Estimation @@ -66,7 +66,7 @@ opObj11.addParameter(name='zmax', value='70', format='int') #Save RTI figfile0 = 'amisr_rti_beam0.png' opObj11.addParameter(name='figpath', value=figpath, format='str') -opObj11.addParameter(name='figfile', value=figfile0, format='str') +#opObj11.addParameter(name='figfile', value=figfile0, format='str') print "Escribiendo el archivo XML" diff --git a/schainpy/test/amisr_reader.py b/schainpy/test/amisr_reader.py index ffbb148..3a547ba 100644 --- a/schainpy/test/amisr_reader.py +++ b/schainpy/test/amisr_reader.py @@ -17,7 +17,7 @@ path = os.path.join(os.environ['HOME'],'Documents/amisr') #'/home/signalchain/Do figpath = os.path.join(os.environ['HOME'],'Pictures/amisr') -readUnitConfObj = controllerObj.addReadUnit(datatype='AMISR', +readUnitConfObj = controllerObj.addReadUnit(datatype='AMISRReader', path=path, startDate='2014/08/18', endDate='2014/08/18', @@ -25,7 +25,7 @@ readUnitConfObj = controllerObj.addReadUnit(datatype='AMISR', endTime='23:59:59', walk=1) -procUnitAMISR = controllerObj.addProcUnit(datatype='AMISR', inputId=readUnitConfObj.getId()) +procUnitAMISR = controllerObj.addProcUnit(datatype='AMISRProc', inputId=readUnitConfObj.getId()) opObj11 = procUnitAMISR.addOperation(name='PrintInfo', optype='other')