jroIO_amisr.py
542 lines
| 17.9 KiB
| text/x-python
|
PythonLexer
Daniel Valdez
|
r491 | ''' | ||
| @author: Daniel Suarez | ||||
| ''' | ||||
| import os | ||||
| import sys | ||||
| import glob | ||||
| import fnmatch | ||||
| import datetime | ||||
| import re | ||||
| import h5py | ||||
| import numpy | ||||
| from model.proc.jroproc_base import ProcessingUnit, Operation | ||||
| from model.data.jroamisr import AMISR | ||||
| class RadacHeader(): | ||||
| def __init__(self, fp): | ||||
| header = 'Raw11/Data/RadacHeader' | ||||
| self.beamCodeByPulse = fp.get(header+'/BeamCode') | ||||
| self.beamCode = fp.get('Raw11/Data/Beamcodes') | ||||
| self.code = fp.get(header+'/Code') | ||||
| self.frameCount = fp.get(header+'/FrameCount') | ||||
| self.modeGroup = fp.get(header+'/ModeGroup') | ||||
| self.nsamplesPulse = fp.get(header+'/NSamplesPulse') | ||||
| self.pulseCount = fp.get(header+'/PulseCount') | ||||
| self.radacTime = fp.get(header+'/RadacTime') | ||||
| self.timeCount = fp.get(header+'/TimeCount') | ||||
| self.timeStatus = fp.get(header+'/TimeStatus') | ||||
| self.nrecords = self.pulseCount.shape[0] #nblocks | ||||
| self.npulses = self.pulseCount.shape[1] #nprofile | ||||
| self.nsamples = self.nsamplesPulse[0,0] #ngates | ||||
| self.nbeams = self.beamCode.shape[1] | ||||
| def getIndexRangeToPulse(self, idrecord=0): | ||||
| indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0) | ||||
| startPulseCountId = indexToZero[0][0] | ||||
| endPulseCountId = startPulseCountId - 1 | ||||
| range1 = numpy.arange(startPulseCountId,self.npulses,1) | ||||
| range2 = numpy.arange(0,startPulseCountId,1) | ||||
| return range1, range2 | ||||
| class AMISRReader(ProcessingUnit): | ||||
| path = None | ||||
| startDate = None | ||||
| endDate = None | ||||
| startTime = None | ||||
| endTime = None | ||||
| walk = None | ||||
| isConfig = False | ||||
| def __init__(self): | ||||
| self.set = None | ||||
| self.subset = None | ||||
| self.extension_file = '.h5' | ||||
| self.dtc_str = 'dtc' | ||||
| self.dtc_id = 0 | ||||
| self.status = True | ||||
| self.isConfig = False | ||||
| self.dirnameList = [] | ||||
| self.filenameList = [] | ||||
| self.fileIndex = None | ||||
| self.flagNoMoreFiles = False | ||||
| self.flagIsNewFile = 0 | ||||
| self.filename = '' | ||||
| self.amisrFilePointer = None | ||||
| self.radacHeaderObj = None | ||||
| self.dataOut = self.__createObjByDefault() | ||||
| self.datablock = None | ||||
| self.rest_datablock = None | ||||
| self.range = None | ||||
| self.idrecord_count = 0 | ||||
| self.profileIndex = 0 | ||||
| self.idpulse_range1 = None | ||||
| self.idpulse_range2 = None | ||||
| self.beamCodeByFrame = None | ||||
| self.radacTimeByFrame = None | ||||
| #atributos originales tal y como esta en el archivo de datos | ||||
| self.beamCodesFromFile = None | ||||
| self.radacTimeFromFile = None | ||||
| self.rangeFromFile = None | ||||
| self.dataByFrame = None | ||||
| self.dataset = None | ||||
| self.beamCodeDict = {} | ||||
| self.beamRangeDict = {} | ||||
| #experiment cgf file | ||||
| self.npulsesint_fromfile = None | ||||
| self.recordsperfile_fromfile = None | ||||
| self.nbeamcodes_fromfile = None | ||||
| self.ngates_fromfile = None | ||||
| self.ippSeconds_fromfile = None | ||||
| self.frequency_h5file = None | ||||
| self.__firstFile = True | ||||
| self.buffer_radactime = None | ||||
| def __createObjByDefault(self): | ||||
| dataObj = AMISR() | ||||
| return dataObj | ||||
| def __setParameters(self,path,startDate,endDate,startTime,endTime,walk): | ||||
| self.path = path | ||||
| self.startDate = startDate | ||||
| self.endDate = endDate | ||||
| self.startTime = startTime | ||||
| self.endTime = endTime | ||||
| self.walk = walk | ||||
| def __checkPath(self): | ||||
| if os.path.exists(self.path): | ||||
| self.status = 1 | ||||
| else: | ||||
| self.status = 0 | ||||
| print 'Path:%s does not exists'%self.path | ||||
| return | ||||
| def __selDates(self, amisr_dirname_format): | ||||
| try: | ||||
| year = int(amisr_dirname_format[0:4]) | ||||
| month = int(amisr_dirname_format[4:6]) | ||||
| dom = int(amisr_dirname_format[6:8]) | ||||
| thisDate = datetime.date(year,month,dom) | ||||
| if (thisDate>=self.startDate and thisDate <= self.endDate): | ||||
| return amisr_dirname_format | ||||
| except: | ||||
| return None | ||||
| def __findDataForDates(self): | ||||
| if not(self.status): | ||||
| return None | ||||
| pat = '\d+.\d+' | ||||
| dirnameList = [re.search(pat,x) for x in os.listdir(self.path)] | ||||
| dirnameList = filter(lambda x:x!=None,dirnameList) | ||||
| dirnameList = [x.string for x in dirnameList] | ||||
| dirnameList = [self.__selDates(x) for x in dirnameList] | ||||
| dirnameList = filter(lambda x:x!=None,dirnameList) | ||||
| if len(dirnameList)>0: | ||||
| self.status = 1 | ||||
| self.dirnameList = dirnameList | ||||
| self.dirnameList.sort() | ||||
| else: | ||||
| self.status = 0 | ||||
| return None | ||||
| def __getTimeFromData(self): | ||||
| pass | ||||
| def __filterByGlob1(self, dirName): | ||||
| filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file) | ||||
| filterDict = {} | ||||
| filterDict.setdefault(dirName) | ||||
| filterDict[dirName] = filter_files | ||||
| return filterDict | ||||
| def __getFilenameList(self, fileListInKeys, dirList): | ||||
| for value in fileListInKeys: | ||||
| dirName = value.keys()[0] | ||||
| for file in value[dirName]: | ||||
| filename = os.path.join(dirName, file) | ||||
| self.filenameList.append(filename) | ||||
| def __selectDataForTimes(self): | ||||
| #aun no esta implementado el filtro for tiempo | ||||
| if not(self.status): | ||||
| return None | ||||
| dirList = [os.path.join(self.path,x) for x in self.dirnameList] | ||||
| fileListInKeys = [self.__filterByGlob1(x) for x in dirList] | ||||
| self.__getFilenameList(fileListInKeys, dirList) | ||||
| if len(self.filenameList)>0: | ||||
| self.status = 1 | ||||
| self.filenameList.sort() | ||||
| else: | ||||
| self.status = 0 | ||||
| return None | ||||
| def __searchFilesOffline(self, | ||||
| path, | ||||
| startDate, | ||||
| endDate, | ||||
| startTime=datetime.time(0,0,0), | ||||
| endTime=datetime.time(23,59,59), | ||||
| walk=True): | ||||
| self.__setParameters(path, startDate, endDate, startTime, endTime, walk) | ||||
| self.__checkPath() | ||||
| self.__findDataForDates() | ||||
| self.__selectDataForTimes() | ||||
| for i in range(len(self.filenameList)): | ||||
| print "%s" %(self.filenameList[i]) | ||||
| return | ||||
| def __setNextFileOffline(self): | ||||
| idFile = self.fileIndex | ||||
| while (True): | ||||
| idFile += 1 | ||||
| if not(idFile < len(self.filenameList)): | ||||
| self.flagNoMoreFiles = 1 | ||||
| print "No more Files" | ||||
| return 0 | ||||
| filename = self.filenameList[idFile] | ||||
| amisrFilePointer = h5py.File(filename,'r') | ||||
| break | ||||
| self.flagIsNewFile = 1 | ||||
| self.fileIndex = idFile | ||||
| self.filename = filename | ||||
| self.amisrFilePointer = amisrFilePointer | ||||
| print "Setting the file: %s"%self.filename | ||||
| return 1 | ||||
| def __readHeader(self): | ||||
| self.radacHeaderObj = RadacHeader(self.amisrFilePointer) | ||||
| #update values from experiment cfg file | ||||
| if self.radacHeaderObj.nrecords == self.recordsperfile_fromfile: | ||||
| self.radacHeaderObj.nrecords = self.recordsperfile_fromfile | ||||
| self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile | ||||
| self.radacHeaderObj.npulses = self.npulsesint_fromfile | ||||
| self.radacHeaderObj.nsamples = self.ngates_fromfile | ||||
| #get tuning frequency | ||||
| frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency') | ||||
| self.frequency_h5file = frequency_h5file_dataset[0,0] | ||||
| self.flagIsNewFile = 1 | ||||
| def __getBeamCode(self): | ||||
| self.beamCodeDict = {} | ||||
| self.beamRangeDict = {} | ||||
| for i in range(len(self.radacHeaderObj.beamCode[0,:])): | ||||
| self.beamCodeDict.setdefault(i) | ||||
| self.beamRangeDict.setdefault(i) | ||||
| self.beamCodeDict[i] = self.radacHeaderObj.beamCode[0,i] | ||||
| just4record0 = self.radacHeaderObj.beamCodeByPulse[0,:] | ||||
| for i in range(len(self.beamCodeDict.values())): | ||||
| xx = numpy.where(just4record0==self.beamCodeDict.values()[i]) | ||||
| self.beamRangeDict[i] = xx[0] | ||||
| def __getExpParameters(self): | ||||
| if not(self.status): | ||||
| return None | ||||
| experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup') | ||||
| expFinder = glob.glob1(experimentCfgPath,'*.exp') | ||||
| if len(expFinder)== 0: | ||||
| self.status = 0 | ||||
| return None | ||||
| experimentFilename = os.path.join(experimentCfgPath,expFinder[0]) | ||||
| f = open(experimentFilename) | ||||
| lines = f.readlines() | ||||
| f.close() | ||||
| parmsList = ['npulsesint*','recordsperfile*','nbeamcodes*','ngates*'] | ||||
| filterList = [fnmatch.filter(lines, x) for x in parmsList] | ||||
| values = [re.sub(r'\D',"",x[0]) for x in filterList] | ||||
| self.npulsesint_fromfile = int(values[0]) | ||||
| self.recordsperfile_fromfile = int(values[1]) | ||||
| self.nbeamcodes_fromfile = int(values[2]) | ||||
| self.ngates_fromfile = int(values[3]) | ||||
| tufileFinder = fnmatch.filter(lines, 'tufile=*') | ||||
| tufile = tufileFinder[0].split('=')[1].split('\n')[0] | ||||
| tufilename = os.path.join(experimentCfgPath,tufile) | ||||
| f = open(tufilename) | ||||
| lines = f.readlines() | ||||
| f.close() | ||||
| self.ippSeconds_fromfile = float(lines[1].split()[2])/1E6 | ||||
| self.status = 1 | ||||
| def __setIdsAndArrays(self): | ||||
| self.dataByFrame = self.__setDataByFrame() | ||||
| self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :] | ||||
| self.readRanges() | ||||
| self.idpulse_range1, self.idpulse_range2 = self.radacHeaderObj.getIndexRangeToPulse(0) | ||||
| self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses) | ||||
| self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame) | ||||
| def __setNextFile(self): | ||||
| newFile = self.__setNextFileOffline() | ||||
| if not(newFile): | ||||
| return 0 | ||||
| self.__readHeader() | ||||
| if self.__firstFile: | ||||
| self.__setIdsAndArrays() | ||||
| self.__firstFile = False | ||||
| self.__getBeamCode() | ||||
| self.readDataBlock() | ||||
| def setup(self,path=None, | ||||
| startDate=None, | ||||
| endDate=None, | ||||
| startTime=datetime.time(0,0,0), | ||||
| endTime=datetime.time(23,59,59), | ||||
| walk=True): | ||||
| #Busqueda de archivos offline | ||||
| self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk) | ||||
| if not(self.filenameList): | ||||
| print "There is no files into the folder: %s"%(path) | ||||
| sys.exit(-1) | ||||
| self.__getExpParameters() | ||||
| self.fileIndex = -1 | ||||
| self.__setNextFile() | ||||
| def readRanges(self): | ||||
| dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range') | ||||
| #self.rangeFromFile = dataset.value | ||||
| self.rangeFromFile = numpy.reshape(dataset.value,(-1)) | ||||
| return range | ||||
| def readRadacTime(self,idrecord, range1, range2): | ||||
| self.radacTimeFromFile = self.radacHeaderObj.radacTime.value | ||||
| radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses)) | ||||
| #radacTimeByFrame = dataset[idrecord - 1,range1] | ||||
| #radacTimeByFrame = dataset[idrecord,range2] | ||||
| return radacTimeByFrame | ||||
| def readBeamCode(self, idrecord, range1, range2): | ||||
| dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode') | ||||
| beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses)) | ||||
| self.beamCodesFromFile = dataset.value | ||||
| #beamcodeByFrame[range1] = dataset[idrecord - 1, range1] | ||||
| #beamcodeByFrame[range2] = dataset[idrecord, range2] | ||||
| beamcodeByFrame[range1] = dataset[idrecord, range1] | ||||
| beamcodeByFrame[range2] = dataset[idrecord, range2] | ||||
| return beamcodeByFrame | ||||
| def __setDataByFrame(self): | ||||
| ndata = 2 # porque es complejo | ||||
| dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata)) | ||||
| return dataByFrame | ||||
| def __readDataSet(self): | ||||
| dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data') | ||||
| return dataset | ||||
| def __setDataBlock(self,): | ||||
| real = self.dataByFrame[:,:,0] #asumo que 0 es real | ||||
| imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario | ||||
| datablock = real + imag*1j #armo el complejo | ||||
| return datablock | ||||
| def readSamples_version1(self,idrecord): | ||||
| #estas tres primeras lineas solo se deben ejecutar una vez | ||||
| if self.flagIsNewFile: | ||||
| #reading dataset | ||||
| self.dataset = self.__readDataSet() | ||||
| self.flagIsNewFile = 0 | ||||
| if idrecord == 0: | ||||
| #if self.buffer_last_record == None: | ||||
| selectorById = self.radacHeaderObj.pulseCount[0,self.idpulse_range2] | ||||
| self.dataByFrame[selectorById,:,:] = self.dataset[0, self.idpulse_range2,:,:] | ||||
| self.radacTimeByFrame[selectorById] = self.radacHeaderObj.radacTime[0, self.idpulse_range2] | ||||
| selectorById = self.radacHeaderObj.pulseCount[0,self.idpulse_range1] | ||||
| self.radacTimeByFrame[selectorById] = self.buffer_radactime[selectorById] | ||||
| datablock = self.__setDataBlock() | ||||
| return datablock | ||||
| selectorById = self.radacHeaderObj.pulseCount[idrecord-1,self.idpulse_range1] | ||||
| self.dataByFrame[selectorById,:,:] = self.dataset[idrecord-1, self.idpulse_range1, :, :] | ||||
| self.radacTimeByFrame[selectorById] = self.radacHeaderObj.radacTime[idrecord-1, self.idpulse_range1] | ||||
| selectorById = self.radacHeaderObj.pulseCount[idrecord,self.idpulse_range2]#data incompleta ultimo archivo de carpeta, verifica el record real segun la dimension del arreglo de datos | ||||
| self.dataByFrame[selectorById,:,:] = self.dataset[idrecord, self.idpulse_range2, :, :] | ||||
| self.radacTimeByFrame[selectorById] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range2] | ||||
| datablock = self.__setDataBlock() | ||||
| selectorById = self.radacHeaderObj.pulseCount[idrecord,self.idpulse_range1] | ||||
| self.dataByFrame[selectorById,:,:] = self.dataset[idrecord, self.idpulse_range1, :, :] | ||||
| self.buffer_radactime[selectorById] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range1] | ||||
| return datablock | ||||
| def readSamples(self,idrecord): | ||||
| if self.flagIsNewFile: | ||||
| self.dataByFrame = self.__setDataByFrame() | ||||
| self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :] | ||||
| #reading ranges | ||||
| self.readRanges() | ||||
| #reading dataset | ||||
| self.dataset = self.__readDataSet() | ||||
| self.flagIsNewFile = 0 | ||||
| self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :] | ||||
| self.dataByFrame = self.dataset[idrecord, :, :, :] | ||||
| datablock = self.__setDataBlock() | ||||
| return datablock | ||||
| def readDataBlock(self): | ||||
| self.datablock = self.readSamples_version1(self.idrecord_count) | ||||
| #self.datablock = self.readSamples(self.idrecord_count) | ||||
| #print 'record:', self.idrecord_count | ||||
| self.idrecord_count += 1 | ||||
| self.profileIndex = 0 | ||||
| if self.idrecord_count >= self.radacHeaderObj.nrecords: | ||||
| self.idrecord_count = 0 | ||||
| self.flagIsNewFile = 1 | ||||
| def readNextBlock(self): | ||||
| self.readDataBlock() | ||||
| if self.flagIsNewFile: | ||||
| self.__setNextFile() | ||||
| pass | ||||
| def __hasNotDataInBuffer(self): | ||||
| #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record | ||||
| if self.profileIndex >= self.radacHeaderObj.npulses: | ||||
| return 1 | ||||
| return 0 | ||||
| def printUTC(self): | ||||
| print self.dataOut.utctime | ||||
| print '' | ||||
| def setObjProperties(self): | ||||
| self.dataOut.heightList = self.rangeFromFile/1000.0 #km | ||||
| self.dataOut.nProfiles = self.radacHeaderObj.npulses | ||||
| self.dataOut.nRecords = self.radacHeaderObj.nrecords | ||||
| self.dataOut.nBeams = self.radacHeaderObj.nbeams | ||||
| self.dataOut.ippSeconds = self.ippSeconds_fromfile | ||||
| self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt | ||||
| self.dataOut.frequency = self.frequency_h5file | ||||
| self.dataOut.nBaud = None | ||||
| self.dataOut.nCode = None | ||||
| self.dataOut.code = None | ||||
| self.dataOut.beamCodeDict = self.beamCodeDict | ||||
| self.dataOut.beamRangeDict = self.beamRangeDict | ||||
| def getData(self): | ||||
| if self.flagNoMoreFiles: | ||||
| self.dataOut.flagNoData = True | ||||
| print 'Process finished' | ||||
| return 0 | ||||
| if self.__hasNotDataInBuffer(): | ||||
| self.readNextBlock() | ||||
| if self.datablock == None: # setear esta condicion cuando no hayan datos por leers | ||||
| self.dataOut.flagNoData = True | ||||
| return 0 | ||||
| self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1)) | ||||
| self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex] | ||||
| self.dataOut.flagNoData = False | ||||
| self.profileIndex += 1 | ||||
| return self.dataOut.data | ||||
| def run(self, **kwargs): | ||||
| if not(self.isConfig): | ||||
| self.setup(**kwargs) | ||||
| self.setObjProperties() | ||||
| self.isConfig = True | ||||
| self.getData() | ||||