diff --git a/schainpy/model/jrodata.py b/schainpy/model/jrodata.py index 69a900b..1dd20ea 100644 --- a/schainpy/model/jrodata.py +++ b/schainpy/model/jrodata.py @@ -643,4 +643,15 @@ class Fits: datatime = property(getDatatime, "I'm the 'datatime' property") ltctime = property(getltctime, "I'm the 'ltctime' property") - ltctime = property(getltctime, "I'm the 'ltctime' property") \ No newline at end of file + ltctime = property(getltctime, "I'm the 'ltctime' property") + +class AMISR: + def __init__(self): + self.flagNoData = True + self.data = None + self.utctime = None + self.type = "AMISR" + + def isEmpty(self): + + return self.flagNoData \ No newline at end of file diff --git a/schainpy/model/jrodataIO.py b/schainpy/model/jrodataIO.py index 193fa24..9b5d477 100644 --- a/schainpy/model/jrodataIO.py +++ b/schainpy/model/jrodataIO.py @@ -10,6 +10,7 @@ import time import numpy import fnmatch import time, datetime +import h5py from xml.etree.ElementTree import Element, SubElement, ElementTree try: import pyfits @@ -3418,4 +3419,407 @@ class FitsReader(ProcessingUnit): self.setup(**kwargs) self.isConfig = True - self.getData() \ No newline at end of file + self.getData() + + +class RadacHeader(): + def __init__(self, fp): + header = 'Raw11/Data/RadacHeader' + self.beamCode = fp.get(header+'/BeamCode') + self.code = fp.get(header+'/Code') + self.frameCount = fp.get(header+'/FrameCount') + self.modeGroup = fp.get(header+'/ModeGroup') + self.nsamplesPulse = fp.get(header+'/NSamplesPulse') + self.pulseCount = fp.get(header+'/PulseCount') + self.radacTime = fp.get(header+'/RadacTime') + self.timeCount = fp.get(header+'/TimeCount') + self.timeStatus = fp.get(header+'/TimeStatus') + + self.nrecords = self.pulseCount.shape[0] #numero de bloques + self.npulses = self.pulseCount.shape[1] #numero de perfiles + self.nsamples = self.nsamplesPulse[0,0] #numero de alturas + + + def getIndexRangeToPulse(self, idrecord=0): + indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0) + startPulseCountId = indexToZero[0][0] + endPulseCountId = startPulseCountId - 1 + range1 = numpy.arange(startPulseCountId,self.npulses,1) + range2 = numpy.arange(0,startPulseCountId,1) + return range1, range2 + + +class AMISRReader(ProcessingUnit): + + path = None + startDate = None + endDate = None + startTime = None + endTime = None + walk = None + isConfig = False + + def __init__(self): + self.set = None + self.subset = None + self.extension_file = '.h5' + self.dtc_str = 'dtc' + self.dtc_id = 0 + self.status = True + self.isConfig = False + self.dirnameList = [] + self.filenameList = [] + self.fileIndex = None + self.flagNoMoreFiles = False + self.flagIsNewFile = 0 + self.filename = '' + self.amisrFilePointer = None + self.radacHeaderObj = None + self.dataOut = self.__createObjByDefault() + self.datablock = None + self.rest_datablock = None + self.range = None + self.idrecord_count = 0 + self.profileIndex = 0 + self.idpulse_range1 = None + self.idpulse_range2 = None + self.beamCodeByFrame = None + self.radacTimeByFrame = None + #atributos originales tal y como esta en el archivo de datos + self.beamCodesFromFile = None + self.radacTimeFromFile = None + self.rangeFromFile = None + self.dataByFrame = None + self.dataset = None + + + def __createObjByDefault(self): + + dataObj = AMISR() + + return dataObj + + def __setParameters(self,path,startDate,endDate,startTime,endTime,walk): + self.path = path + self.startDate = startDate + self.endDate = endDate + self.startTime = startTime + self.endTime = endTime + self.walk = walk + + def __checkPath(self): + if os.path.exists(self.path): + self.status = 1 + else: + self.status = 0 + print 'Path:%s does not exists'%self.path + + return + + def __selDates(self, amisr_dirname_format): + year = int(amisr_dirname_format[0:4]) + month = int(amisr_dirname_format[4:6]) + dom = int(amisr_dirname_format[6:8]) + thisDate = datetime.date(year,month,dom) + + if (thisDate>=self.startDate and thisDate <= self.endDate): + return amisr_dirname_format + + def __findDataForDates(self): + + import re + + if not(self.status): + return None + + pat = '\d+.\d+' + dirnameList = [re.search(pat,x).string for x in os.listdir(self.path)] + dirnameList = [self.__selDates(x) for x in dirnameList] + dirnameList = filter(lambda x:x!=None,dirnameList) + if len(dirnameList)>0: + self.status = 1 + self.dirnameList = dirnameList + self.dirnameList.sort() + else: + self.status = 0 + return None + + def __getTimeFromData(self): + pass + + def __filterByGlob1(self, dirName): + filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file) + filterDict = {} + filterDict.setdefault(dirName) + filterDict[dirName] = filter_files + return filterDict + + def __getFilenameList(self, fileListInKeys, dirList): + for value in fileListInKeys: + dirName = value.keys()[0] + for file in value[dirName]: + filename = os.path.join(dirName, file) + self.filenameList.append(filename) + + + def __selectDataForTimes(self): + #aun no esta implementado el filtro for tiempo + if not(self.status): + return None + + dirList = [os.path.join(self.path,x) for x in self.dirnameList] + + fileListInKeys = [self.__filterByGlob1(x) for x in dirList] + + self.__getFilenameList(fileListInKeys, dirList) + + if len(self.filenameList)>0: + self.status = 1 + self.filenameList.sort() + else: + self.status = 0 + return None + + + def __searchFilesOffline(self, + path, + startDate, + endDate, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + walk=True): + + self.__setParameters(path, startDate, endDate, startTime, endTime, walk) + + self.__checkPath() + + self.__findDataForDates() + + self.__selectDataForTimes() + + for i in range(len(self.filenameList)): + print "%s" %(self.filenameList[i]) + + return + + def __setNextFileOffline(self): + idFile = self.fileIndex + + while (True): + idFile += 1 + if not(idFile < len(self.filenameList)): + self.flagNoMoreFiles = 1 + print "No more Files" + return 0 + + filename = self.filenameList[idFile] + + amisrFilePointer = h5py.File(filename,'r') + + break + + self.flagIsNewFile = 1 + self.fileIndex = idFile + self.filename = filename + + self.amisrFilePointer = amisrFilePointer + + print "Setting the file: %s"%self.filename + + return 1 + + def __readHeader(self): + self.radacHeaderObj = RadacHeader(self.amisrFilePointer) + self.flagIsNewFile = 1 + + + + def __setNextFile(self): + + newFile = self.__setNextFileOffline() + + if not(newFile): + return 0 + + self.__readHeader() + + self.readDataBlock() + + + def setup(self,path=None, + startDate=None, + endDate=None, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + walk=True): + + #Busqueda de archivos offline + self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk) + + if not(self.filenameList): + print "There is no files into the folder: %s"%(path) + + sys.exit(-1) + + self.fileIndex = -1 + + self.__setNextFile() + + def readRanges(self): + dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range') + self.rangeFromFile = dataset.values() + return range + + + def readRadacTime(self,idrecord, range1, range2): + self.radacTimeFromFile = self.radacHeaderObj.radacTime.value + + radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses)) + #radacTimeByFrame = dataset[idrecord - 1,range1] + #radacTimeByFrame = dataset[idrecord,range2] + + return radacTimeByFrame + + def readBeamCode(self, idrecord, range1, range2): + dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode') + beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses)) + self.beamCodesFromFile = dataset.value + + #beamcodeByFrame[range1] = dataset[idrecord - 1, range1] + #beamcodeByFrame[range2] = dataset[idrecord, range2] + beamcodeByFrame[range1] = dataset[idrecord, range1] + beamcodeByFrame[range2] = dataset[idrecord, range2] + + return beamcodeByFrame + + + def __setDataByFrame(self): + ndata = 2 # porque es complejo + dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata)) + return dataByFrame + + def __readDataSet(self): + dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data') + return dataset + + def __setDataBlock(self,): + real = self.dataByFrame[:,:,0] #asumo que 0 es real + imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario + datablock = real + imag*1j #armo el complejo + return datablock + + def readSamples_version1(self,idrecord): + #estas tres primeras lineas solo se deben ejecutar una vez + if self.flagIsNewFile: + self.idpulse_range1, self.idpulse_range2 = self.radacHeaderObj.getIndexRangeToPulse(0) + self.dataByFrame = self.__setDataByFrame() + self.beamCodeByFrame = self.readBeamCode(idrecord, self.idpulse_range1, self.idpulse_range2) + self.radacTimeByFrame = self.readRadacTime(idrecord, self.idpulse_range1, self.idpulse_range2) + #reading dataset + self.dataset = self.__readDataSet() + + if idrecord == 0: + + if len(numpy.where(self.dataByFrame!=0.0)[0]) or len(numpy.where(self.dataByFrame!=0.0)[1]) or len(numpy.where(self.dataByFrame!=0.0)[2]): + #falta agregar una condicion para datos discontinuos + #por defecto une los datos del record anterior + self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :] + #timepulse + self.radacTimeByFrame[self.idpulse_range2] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range2] + else: + self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :] + + self.radacTimeByFrame[self.idpulse_range1] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range1] + + datablock = self.__setDataBlock() + + return datablock + + self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord - 1,self.idpulse_range1, :, :] + self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :] + datablock = self.__setDataBlock() + self.flagIsNewFile = 0 + + self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :] + + + return datablock + + + def readSamples(self,idrecord): + if self.flagIsNewFile: + self.dataByFrame = self.__setDataByFrame() + self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :] + + #reading dataset + self.dataset = self.__readDataSet() + + self.flagIsNewFile = 0 + self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :] + self.dataByFrame = self.dataset[idrecord, :, :, :] + datablock = self.__setDataBlock() + return datablock + + + def readDataBlock(self): + + #self.datablock = self.readSamples(self.idrecord_count) + self.datablock = self.readSamples(self.idrecord_count) + #print 'record:', self.idrecord_count + + self.idrecord_count += 1 + self.profileIndex = 0 + + if self.idrecord_count >= self.radacHeaderObj.nrecords: + self.idrecord_count = 0 + self.flagIsNewFile = 1 + + def readNextBlock(self): + + self.readDataBlock() + + if self.flagIsNewFile: + self.__setNextFile() + pass + + def __hasNotDataInBuffer(self): + #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record + if self.profileIndex >= self.radacHeaderObj.npulses: + return 1 + return 0 + + + def getData(self): + + if self.flagNoMoreFiles: + self.dataOut.flagNoData = True + print 'Process finished' + return 0 + + if self.__hasNotDataInBuffer(): + self.readNextBlock() +# if not( self.readNextBlock() ): +# return 0 +# self.getFirstHeader() + + if self.datablock == None: # setear esta condicion cuando no hayan datos por leers + self.dataOut.flagNoData = True + return 0 + + self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1)) + + self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex] + + self.dataOut.flagNoData = False + + self.profileIndex += 1 + + return self.dataOut.data + + + def run(self, **kwargs): + if not(self.isConfig): + self.setup(**kwargs) + self.isConfig = True + + self.getData()