schain Commit - r510:de96659ace58 · Jicamarca Repository

Bug Fixed: AMISR Setup File does not correspond with the experiment range dates...

Daniel Valdez -

r510:de96659ace58

parent child

Context file:

r510:de96659ace58

Collapse all files

schainpy/model/io/jroIO_amisr.py +10 -2

             '''
             @author: Daniel Suarez
             '''
             import os
             import sys
             import glob
             import fnmatch
             import datetime
             import time
             import re
             import h5py
             import numpy
             from model.proc.jroproc_base import ProcessingUnit, Operation
             from model.data.jroamisr import AMISR
             class RadacHeader():
                 def __init__(self, fp):
                     header = 'Raw11/Data/RadacHeader'
                     self.beamCodeByPulse = fp.get(header+'/BeamCode')
                     self.beamCode = fp.get('Raw11/Data/Beamcodes')
                     self.code = fp.get(header+'/Code')
                     self.frameCount = fp.get(header+'/FrameCount')
                     self.modeGroup = fp.get(header+'/ModeGroup')
                     self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
                     self.pulseCount = fp.get(header+'/PulseCount')
                     self.radacTime = fp.get(header+'/RadacTime')
                     self.timeCount = fp.get(header+'/TimeCount')
                     self.timeStatus = fp.get(header+'/TimeStatus')
                     self.nrecords = self.pulseCount.shape[0] #nblocks
                     self.npulses = self.pulseCount.shape[1] #nprofile
                     self.nsamples = self.nsamplesPulse[0,0] #ngates
                     self.nbeams = self.beamCode.shape[1]
                 def getIndexRangeToPulse(self, idrecord=0):
                     #indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
                     #startPulseCountId = indexToZero[0][0]
                     #endPulseCountId = startPulseCountId - 1
                     #range1 = numpy.arange(startPulseCountId,self.npulses,1)
                     #range2 = numpy.arange(0,startPulseCountId,1)
                     #return range1, range2
                     zero = 0
                     npulse = max(self.pulseCount[0,:]+1)-1
                     looking_index = numpy.where(self.pulseCount.value[idrecord,:]==npulse)[0]
                     getLastIndex = looking_index[-1]
                     index_data = numpy.arange(0,getLastIndex+1,1)
                     index_buffer = numpy.arange(getLastIndex+1,self.npulses,1)
                     return index_data, index_buffer
             class AMISRReader(ProcessingUnit):
                 path = None
                 startDate = None
                 endDate = None
                 startTime = None
                 endTime = None
                 walk = None
                 isConfig = False
                 def __init__(self):
                     self.set = None
                     self.subset = None
                     self.extension_file = '.h5'
                     self.dtc_str = 'dtc'
                     self.dtc_id = 0
                     self.status = True
                     self.isConfig = False
                     self.dirnameList = []
                     self.filenameList = []
                     self.fileIndex = None
                     self.flagNoMoreFiles = False
                     self.flagIsNewFile = 0
                     self.filename = ''
                     self.amisrFilePointer = None
                     self.radacHeaderObj = None
                     self.dataOut = self.__createObjByDefault()
                     self.datablock = None
                     self.rest_datablock = None
                     self.range = None
                     self.idrecord_count = 0
                     self.profileIndex = 0
                     self.index_amisr_sample = None
                     self.index_amisr_buffer = None
                     self.beamCodeByFrame = None
                     self.radacTimeByFrame = None
                     #atributos originales tal y como esta en el archivo de datos
                     self.beamCodesFromFile = None
                     self.radacTimeFromFile = None
                     self.rangeFromFile = None
                     self.dataByFrame = None
                     self.dataset = None
                     self.beamCodeDict = {}
                     self.beamRangeDict = {}
                     #experiment cgf file
                     self.npulsesint_fromfile = None
                     self.recordsperfile_fromfile = None
                     self.nbeamcodes_fromfile = None
                     self.ngates_fromfile = None
                     self.ippSeconds_fromfile = None
                     self.frequency_h5file = None
                     self.__firstFile = True
                     self.buffer_radactime = None
                     self.index4_schain_datablock = None
                     self.index4_buffer = None
                     self.schain_datablock = None
                     self.buffer = None
                     self.linear_pulseCount = None
                     self.npulseByFrame = None
                     self.profileIndex_offset = None
                     self.timezone = 'ut'
                     self.__waitForNewFile = 20
                     self.__filename_online = None
                 def __createObjByDefault(self):
                     dataObj = AMISR()
                     return dataObj
                 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.walk = walk
                 def __checkPath(self):
                     if os.path.exists(self.path):
                         self.status = 1
                     else:
                         self.status = 0
                         print 'Path:%s does not exists'%self.path
                     return
                 def __selDates(self, amisr_dirname_format):
                     try:
                         year = int(amisr_dirname_format[0:4])
                         month = int(amisr_dirname_format[4:6])
                         dom = int(amisr_dirname_format[6:8])
                         thisDate = datetime.date(year,month,dom)
                         if (thisDate>=self.startDate and thisDate <= self.endDate):
                             return amisr_dirname_format
                     except:
                         return None
                 def __findDataForDates(self,online=False):
                     if not(self.status):
                         return None
                     pat = '\d+.\d+'
                     dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
                     dirnameList = filter(lambda x:x!=None,dirnameList)
                     dirnameList = [x.string for x in dirnameList]
                     if not(online):
                         dirnameList = [self.__selDates(x) for x in dirnameList]
                         dirnameList = filter(lambda x:x!=None,dirnameList)
                     if len(dirnameList)>0:
                         self.status = 1
                         self.dirnameList = dirnameList
                         self.dirnameList.sort()
                     else:
                         self.status = 0
                         return None
                 def __getTimeFromData(self):
                     startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
                     endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                     print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
                     print '........................................'
                     filter_filenameList = []
                     self.filenameList.sort()
                     for i in range(len(self.filenameList)-1):
                         filename = self.filenameList[i]
                         fp = h5py.File(filename,'r')
                         time_str = fp.get('Time/RadacTimeString')
                         startDateTimeStr_File = time_str[0][0].split('.')[0]
                         junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                         endDateTimeStr_File = time_str[-1][-1].split('.')[0]
                         junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                         fp.close()
                         if self.timezone == 'lt':
                             startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                             endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
                         if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
                             #self.filenameList.remove(filename)
                             filter_filenameList.append(filename)
                     filter_filenameList.sort()
                     self.filenameList = filter_filenameList
                     return 1
                 def __filterByGlob1(self, dirName):
                     filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
                     filterDict = {}
                     filterDict.setdefault(dirName)
                     filterDict[dirName] = filter_files
                     return filterDict
                 def __getFilenameList(self, fileListInKeys, dirList):
                     for value in fileListInKeys:
                         dirName = value.keys()[0]
                         for file in value[dirName]:
                             filename = os.path.join(dirName, file)
                             self.filenameList.append(filename)
                 def __selectDataForTimes(self, online=False):
                     #aun no esta implementado el filtro for tiempo
                     if not(self.status):
                         return None
                     dirList = [os.path.join(self.path,x) for x in self.dirnameList]
                     fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
                     self.__getFilenameList(fileListInKeys, dirList)
                     if not(online):
                         #filtro por tiempo
                         if not(self.all):
                             self.__getTimeFromData()
                         if len(self.filenameList)>0:
                             self.status = 1
                             self.filenameList.sort()
                         else:
                             self.status = 0
                             return None
-                        return 1
                     else:
                         #get the last file - 1
                         self.filenameList = [self.filenameList[-2]]
-                        return 1
+                    new_dirnameList = []
+                    for dirname in self.dirnameList:
+                        junk = numpy.array([dirname in x for x in self.filenameList])
+                        junk_sum = junk.sum()
+                        if junk_sum > 0:
+                            new_dirnameList.append(dirname)
+                    self.dirnameList = new_dirnameList
+                    return 1
                 def __searchFilesOnline(self,
                                         path,
                                         walk=True):
                     startDate = datetime.datetime.utcnow().date()
                     endDate = datetime.datetime.utcnow().date()
                     self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk)
                     self.__checkPath()
                     self.__findDataForDates(online=True)
                     self.dirnameList = [self.dirnameList[-1]]
                     self.__selectDataForTimes(online=True)
                     return
                 def __searchFilesOffline(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         walk=True):
                     self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
                     self.__checkPath()
                     self.__findDataForDates()
                     self.__selectDataForTimes()
                     for i in range(len(self.filenameList)):
                         print "%s" %(self.filenameList[i])
                     return
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print "No more Files"
                             return 0
                         filename = self.filenameList[idFile]
                         amisrFilePointer = h5py.File(filename,'r')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.amisrFilePointer = amisrFilePointer
                     print "Setting the file: %s"%self.filename
                     return 1
                 def __setNextFileOnline(self):
                     filename = self.filenameList[0]
                     if self.__filename_online != None:
                         self.__selectDataForTimes(online=True)
                         filename = self.filenameList[0]
                         while self.__filename_online == filename:
                             print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
                             time.sleep(self.__waitForNewFile)
                             self.__selectDataForTimes(online=True)
                             filename = self.filenameList[0]
                     self.__filename_online = filename
                     self.amisrFilePointer = h5py.File(filename,'r')
                     self.flagIsNewFile = 1
                     self.filename = filename
                     print "Setting the file: %s"%self.filename
                     return 1
                 def __readHeader(self):
                     self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
                     #update values from experiment cfg file
                     if self.radacHeaderObj.nrecords == self.recordsperfile_fromfile:
                         self.radacHeaderObj.nrecords = self.recordsperfile_fromfile
                     self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile
                     self.radacHeaderObj.npulses = self.npulsesint_fromfile
                     self.radacHeaderObj.nsamples = self.ngates_fromfile
                     #looking index list for data
                     start_index = self.radacHeaderObj.pulseCount[0,:][0]
                     end_index = self.radacHeaderObj.npulses
                     range4data = range(start_index, end_index)
                     self.index4_schain_datablock = numpy.array(range4data)
                     buffer_start_index = 0
                     buffer_end_index = self.radacHeaderObj.pulseCount[0,:][0]
                     range4buffer = range(buffer_start_index, buffer_end_index)
                     self.index4_buffer = numpy.array(range4buffer)
                     self.linear_pulseCount = numpy.array(range4data + range4buffer)
                     self.npulseByFrame = max(self.radacHeaderObj.pulseCount[0,:]+1)
                     #get tuning frequency
                     frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency')
                     self.frequency_h5file = frequency_h5file_dataset[0,0]
                     self.flagIsNewFile = 1
                 def __getBeamCode(self):
                     self.beamCodeDict = {}
                     self.beamRangeDict = {}
                     beamCodeMap = self.amisrFilePointer.get('Setup/BeamcodeMap')
                     for i in range(len(self.radacHeaderObj.beamCode[0,:])):
                         self.beamCodeDict.setdefault(i)
                         self.beamRangeDict.setdefault(i)
                         beamcodeValue = self.radacHeaderObj.beamCode[0,i]
                         beamcodeIndex = numpy.where(beamCodeMap[:,0] == beamcodeValue)[0][0]
                         x = beamCodeMap[beamcodeIndex][1]
                         y = beamCodeMap[beamcodeIndex][2]
                         z = beamCodeMap[beamcodeIndex][3]
                         self.beamCodeDict[i] = [beamcodeValue, x, y, z]
                     just4record0 = self.radacHeaderObj.beamCodeByPulse[0,:]
                     for i in range(len(self.beamCodeDict.values())):
                         xx = numpy.where(just4record0==self.beamCodeDict.values()[i][0])
                         indexPulseByBeam = self.linear_pulseCount[xx[0]]
                         self.beamRangeDict[i] = indexPulseByBeam
                 def __getExpParameters(self):
                     if not(self.status):
                         return None
                     experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup')
                     expFinder = glob.glob1(experimentCfgPath,'*.exp')
                     if len(expFinder)== 0:
                         self.status = 0
                         return None
                     experimentFilename = os.path.join(experimentCfgPath,expFinder[0])
                     f = open(experimentFilename)
                     lines = f.readlines()
                     f.close()
                     parmsList = ['npulsesint*','recordsperfile*','nbeamcodes*','ngates*']
                     filterList = [fnmatch.filter(lines, x) for x in parmsList]
                     values = [re.sub(r'\D',"",x[0]) for x in filterList]
                     self.npulsesint_fromfile = int(values[0])
                     self.recordsperfile_fromfile = int(values[1])
                     self.nbeamcodes_fromfile = int(values[2])
                     self.ngates_fromfile = int(values[3])
                     tufileFinder = fnmatch.filter(lines, 'tufile=*')
                     tufile = tufileFinder[0].split('=')[1].split('\n')[0]
                     tufile = tufile.split('\r')[0]
                     tufilename = os.path.join(experimentCfgPath,tufile)
                     f = open(tufilename)
                     lines = f.readlines()
                     f.close()
                     self.ippSeconds_fromfile = float(lines[1].split()[2])/1E6
                     self.status = 1
                 def __setIdsAndArrays(self):
                     self.dataByFrame = self.__setDataByFrame()
                     self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :]
                     self.readRanges()
                     self.index_amisr_sample, self.index_amisr_buffer = self.radacHeaderObj.getIndexRangeToPulse(0)
                     self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses)
                     if len(self.index_amisr_buffer) > 0:
                         self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame)
                 def __setNextFile(self,online=False):
                     if not(online):
                         newFile = self.__setNextFileOffline()
                     else:
                         newFile = self.__setNextFileOnline()
                     if not(newFile):
                         return 0
                     self.__readHeader()
                     if self.__firstFile:
                         self.__setIdsAndArrays()
                         self.__firstFile = False
                     self.__getBeamCode()
                     self.readDataBlock()
                 def setup(self,path=None,
                                 startDate=None,
                                 endDate=None,
                                 startTime=datetime.time(0,0,0),
                                 endTime=datetime.time(23,59,59),
                                 walk=True,
                                 timezone='ut',
                                 all=0,
                                 online=False):
                     self.timezone = timezone
                     self.all = all
                     self.online = online
                     if not(online):
                         #Busqueda de archivos offline
                         self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
                     else:
                         self.__searchFilesOnline(path, walk)
                     if not(self.filenameList):
                         print "There is no files into the folder: %s"%(path)
                         sys.exit(-1)
                     self.__getExpParameters()
                     self.fileIndex = -1
                     self.__setNextFile(online)
             #         first_beamcode = self.radacHeaderObj.beamCodeByPulse[0,0]
             #         index = numpy.where(self.radacHeaderObj.beamCodeByPulse[0,:]!=first_beamcode)[0][0]
                     self.profileIndex_offset = self.radacHeaderObj.pulseCount[0,:][0]
                     self.profileIndex = self.profileIndex_offset
                 def readRanges(self):
                     dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
                     self.rangeFromFile = numpy.reshape(dataset.value,(-1))
                     return self.rangeFromFile
                 def readRadacTime(self,idrecord, range1, range2):
                     self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
                     radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
                     #radacTimeByFrame = dataset[idrecord - 1,range1]
                     #radacTimeByFrame = dataset[idrecord,range2]
                     return radacTimeByFrame
                 def readBeamCode(self, idrecord, range1, range2):
                     dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
                     beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
                     self.beamCodesFromFile = dataset.value
                     #beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
                     #beamcodeByFrame[range2] = dataset[idrecord, range2]
                     beamcodeByFrame[range1] = dataset[idrecord, range1]
                     beamcodeByFrame[range2] = dataset[idrecord, range2]
                     return beamcodeByFrame
                 def __setDataByFrame(self):
                     ndata = 2 # porque es complejo
                     dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
                     return dataByFrame
                 def __readDataSet(self):
                     dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
                     return dataset
                 def __setDataBlock(self,):
                     real = self.dataByFrame[:,:,0] #asumo que 0 es real
                     imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
                     datablock = real + imag*1j #armo el complejo
                     return datablock
                 def readSamples_version1(self,idrecord):
                     #estas tres primeras lineas solo se deben ejecutar una vez
                     if self.flagIsNewFile:
                         #reading dataset
                         self.dataset = self.__readDataSet()
                         self.flagIsNewFile = 0
                     if idrecord == 0:
                         self.dataByFrame[self.index4_schain_datablock, : ,:] = self.dataset[0, self.index_amisr_sample,:,:]
                         self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[0, self.index_amisr_sample]
                         datablock = self.__setDataBlock()
                         if len(self.index_amisr_buffer) > 0:
                             self.buffer = self.dataset[0, self.index_amisr_buffer,:,:]
                             self.buffer_radactime = self.radacHeaderObj.radacTime[0, self.index_amisr_buffer]
                         return datablock
                     if len(self.index_amisr_buffer) > 0:
                         self.dataByFrame[self.index4_buffer,:,:] = self.buffer.copy()
                         self.radacTimeByFrame[self.index4_buffer] = self.buffer_radactime.copy()
                     self.dataByFrame[self.index4_schain_datablock,:,:] = self.dataset[idrecord, self.index_amisr_sample,:,:]
                     self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_sample]
                     datablock = self.__setDataBlock()
                     if len(self.index_amisr_buffer) > 0:
                         self.buffer = self.dataset[idrecord, self.index_amisr_buffer, :, :]
                         self.buffer_radactime = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_buffer]
                     return datablock
                 def readSamples(self,idrecord):
                     if self.flagIsNewFile:
                         self.dataByFrame = self.__setDataByFrame()
                         self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
                         #reading ranges
                         self.readRanges()
                         #reading dataset
                         self.dataset = self.__readDataSet()
                     self.flagIsNewFile = 0
                     self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
                     self.dataByFrame = self.dataset[idrecord, :, :, :]
                     datablock = self.__setDataBlock()
                     return datablock
                 def readDataBlock(self):
                     self.datablock = self.readSamples_version1(self.idrecord_count)
                     #self.datablock = self.readSamples(self.idrecord_count)
                     #print 'record:', self.idrecord_count
                     self.idrecord_count += 1
                     self.profileIndex = 0
                     if self.idrecord_count >= self.radacHeaderObj.nrecords:
                         self.idrecord_count = 0
                         self.flagIsNewFile = 1
                 def readNextBlock(self):
                     self.readDataBlock()
                     if self.flagIsNewFile:
                         self.__setNextFile(self.online)
                     pass
                 def __hasNotDataInBuffer(self):
                     #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
                     if self.profileIndex >= self.radacHeaderObj.npulses:
                         return 1
                     return 0
                 def printUTC(self):
                     print self.dataOut.utctime
                     print ''
                 def setObjProperties(self):
                     self.dataOut.heightList = self.rangeFromFile/1000.0 #km
                     self.dataOut.nProfiles = self.radacHeaderObj.npulses
                     self.dataOut.nRecords = self.radacHeaderObj.nrecords
                     self.dataOut.nBeams = self.radacHeaderObj.nbeams
                     self.dataOut.ippSeconds = self.ippSeconds_fromfile
                     self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.frequency_h5file
                     self.dataOut.npulseByFrame = self.npulseByFrame
                     self.dataOut.nBaud = None
                     self.dataOut.nCode = None
                     self.dataOut.code = None
                     self.dataOut.beamCodeDict = self.beamCodeDict
                     self.dataOut.beamRangeDict = self.beamRangeDict
                     if self.timezone == 'lt':
                         self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
                     else:
                         self.dataOut.timeZone = 0 #by default time is UTC
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     if self.__hasNotDataInBuffer():
                         self.readNextBlock()
                     if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
                         self.dataOut.flagNoData = True
                         return 0
                     self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
                     self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
                     self.dataOut.profileIndex = self.profileIndex
                     self.dataOut.flagNoData = False
                     self.profileIndex += 1
                     return self.dataOut.data
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.setObjProperties()
                         self.isConfig = True
                     self.getData()

schainpy/model/proc/jroproc_voltage.py +1 -1

             import numpy
             from jroproc_base import ProcessingUnit, Operation
             from model.data.jrodata import Voltage
             class VoltageProc(ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
             #         self.objectDict = {}
                     self.dataOut = Voltage()
                     self.flip = 1
                 def run(self):
                     if self.dataIn.type == 'AMISR':
                         self.__updateObjFromAmisrInput()
                     if self.dataIn.type == 'Voltage':
                         self.dataOut.copy(self.dataIn)
             #         self.dataOut.copy(self.dataIn)
                 def __updateObjFromAmisrInput(self):
                     self.dataOut.timeZone = self.dataIn.timeZone
                     self.dataOut.dstFlag = self.dataIn.dstFlag
                     self.dataOut.errorCount = self.dataIn.errorCount
                     self.dataOut.useLocalTime = self.dataIn.useLocalTime
                     self.dataOut.flagNoData = self.dataIn.flagNoData
                     self.dataOut.data = self.dataIn.data
                     self.dataOut.utctime = self.dataIn.utctime
                     self.dataOut.channelList = self.dataIn.channelList
                     self.dataOut.timeInterval = self.dataIn.timeInterval
                     self.dataOut.heightList = self.dataIn.heightList
                     self.dataOut.nProfiles = self.dataIn.nProfiles
                     self.dataOut.nCohInt = self.dataIn.nCohInt
                     self.dataOut.ippSeconds = self.dataIn.ippSeconds
                     self.dataOut.frequency = self.dataIn.frequency
                     self.dataOut.azimuth = self.dataIn.azimuth
                     self.dataOut.zenith = self.dataIn.zenith
                     self.dataOut.beam.codeList = self.dataIn.beam.codeList
                     self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
                     self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
             #
             #        pass#
             #
             #    def init(self):
             #
             #
             #        if self.dataIn.type == 'AMISR':
             #            self.__updateObjFromAmisrInput()
             #
             #        if self.dataIn.type == 'Voltage':
             #            self.dataOut.copy(self.dataIn)
             #        # No necesita copiar en cada init() los atributos de dataIn
             #        # la copia deberia hacerse por cada nuevo bloque de datos
                 def selectChannels(self, channelList):
                     channelIndexList = []
                     for channel in channelList:
                         index = self.dataOut.channelList.index(channel)
                         channelIndexList.append(index)
                     self.selectChannelsByIndex(channelIndexList)
                 def selectChannelsByIndex(self, channelIndexList):
                     """
                     Selecciona un bloque de datos en base a canales segun el channelIndexList
                     Input:
                         channelIndexList    :    lista sencilla de canales a seleccionar por ej. [2,3,7]
                     Affected:
                         self.dataOut.data
                         self.dataOut.channelIndexList
                         self.dataOut.nChannels
                         self.dataOut.m_ProcessingHeader.totalSpectra
                         self.dataOut.systemHeaderObj.numChannels
                         self.dataOut.m_ProcessingHeader.blockSize
                     Return:
                         None
                     """
                     for channelIndex in channelIndexList:
                         if channelIndex not in self.dataOut.channelIndexList:
                             print channelIndexList
                             raise ValueError, "The value %d in channelIndexList is not valid" %channelIndex
             #         nChannels = len(channelIndexList)
                     data = self.dataOut.data[channelIndexList,:]
                     self.dataOut.data = data
                     self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
             #        self.dataOut.nChannels = nChannels
                     return 1
                 def selectHeights(self, minHei=None, maxHei=None):
                     """
                     Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
                     minHei <= height <= maxHei
                     Input:
                         minHei    :    valor minimo de altura a considerar
                         maxHei    :    valor maximo de altura a considerar
                     Affected:
                         Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if minHei == None:
                         minHei = self.dataOut.heightList[0]
                     if maxHei == None:
                         maxHei = self.dataOut.heightList[-1]
                     if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
                         raise ValueError, "some value in (%d,%d) is not valid" % (minHei, maxHei)
                     if (maxHei > self.dataOut.heightList[-1]):
                         maxHei = self.dataOut.heightList[-1]
             #            raise ValueError, "some value in (%d,%d) is not valid" % (minHei, maxHei)
                     minIndex = 0
                     maxIndex = 0
                     heights = self.dataOut.heightList
                     inda = numpy.where(heights >= minHei)
                     indb = numpy.where(heights <= maxHei)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(heights)
                     self.selectHeightsByIndex(minIndex, maxIndex)
                     return 1
                 def selectHeightsByIndex(self, minIndex, maxIndex):
                     """
                     Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
                     minIndex <= index <= maxIndex
                     Input:
                         minIndex    :    valor de indice minimo de altura a considerar
                         maxIndex    :    valor de indice maximo de altura a considerar
                     Affected:
                         self.dataOut.data
                         self.dataOut.heightList
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
                     if (maxIndex >= self.dataOut.nHeights):
                         maxIndex = self.dataOut.nHeights-1
             #            raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
             #         nHeights = maxIndex - minIndex + 1
                     #voltage
                     data = self.dataOut.data[:,minIndex:maxIndex+1]
             #         firstHeight = self.dataOut.heightList[minIndex]
                     self.dataOut.data = data
                     self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex+1]
                     return 1
                 def filterByHeights(self, window, axis=1):
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     if window == None:
                         window = (self.dataOut.radarControllerHeaderObj.txA/self.dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
                     newdelta = deltaHeight * window
                     r = self.dataOut.data.shape[axis] % window
                     if axis == 1:
                         buffer = self.dataOut.data[:,0:self.dataOut.data.shape[axis]-r]
                         buffer = buffer.reshape(self.dataOut.data.shape[0],self.dataOut.data.shape[axis]/window,window)
                         buffer = numpy.sum(buffer,axis+1)
                     elif axis == 2:
                         buffer = self.dataOut.data[:, :, 0:self.dataOut.data.shape[axis]-r]
                         buffer = buffer.reshape(self.dataOut.data.shape[0],self.dataOut.data.shape[1],self.dataOut.data.shape[axis]/window,window)
                         buffer = numpy.sum(buffer,axis+1)
                     else:
                         raise ValueError, "axis value should be 1 or 2, the input value %d is not valid" % (axis)
                     self.dataOut.data = buffer.copy()
                     self.dataOut.heightList = numpy.arange(self.dataOut.heightList[0],newdelta*(self.dataOut.nHeights-r)/window,newdelta)
                     self.dataOut.windowOfFilter = window
                     return 1
                 def deFlip(self):
                     self.dataOut.data *= self.flip
                     self.flip *= -1.
                 def setRadarFrequency(self, frequency=None):
                     if frequency != None:
                         self.dataOut.frequency = frequency
                     return 1
             class CohInt(Operation):
                 isConfig = False
                 __profIndex = 0
                 __withOverapping  = False
                 __byTime = False
                 __initime = None
                 __lastdatatime = None
                 __integrationtime = None
                 __buffer = None
                 __dataReady = False
                 n = None
                 def __init__(self):
                     Operation.__init__(self)
             #         self.isConfig = False
                 def setup(self, n=None, timeInterval=None, overlapping=False, byblock=False):
                     """
                     Set the parameters of the integration class.
                     Inputs:
                         n        :    Number of coherent integrations
                         timeInterval   :    Time of integration. If the parameter "n" is selected this one does not work
                         overlapping    :
                     """
                     self.__initime = None
                     self.__lastdatatime = 0
                     self.__buffer = None
                     self.__dataReady = False
                     self.byblock = byblock
                     if n == None and timeInterval == None:
                         raise ValueError, "n or timeInterval should be specified ..."
                     if n != None:
                         self.n = n
                         self.__byTime = False
                     else:
-                        self.__integrationtime = timeInterval * 60. #if (type(timeInterval)!=integer) -> change this line
+                        self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
                         self.n = 9999
                         self.__byTime = True
                     if overlapping:
                         self.__withOverapping = True
                         self.__buffer = None
                     else:
                         self.__withOverapping = False
                         self.__buffer = 0
                     self.__profIndex = 0
                 def putData(self, data):
                     """
                     Add a profile to the __buffer and increase in one the __profileIndex
                     """
                     if not self.__withOverapping:
                         self.__buffer += data.copy()
                         self.__profIndex += 1
                         return
                     #Overlapping data
                     nChannels, nHeis = data.shape
                     data = numpy.reshape(data, (1, nChannels, nHeis))
                     #If the buffer is empty then it takes the data value
                     if self.__buffer == None:
                         self.__buffer = data
                         self.__profIndex += 1
                         return
                     #If the buffer length is lower than n then stakcing the data value
                     if self.__profIndex < self.n:
                         self.__buffer = numpy.vstack((self.__buffer, data))
                         self.__profIndex += 1
                         return
                     #If the buffer length is equal to n then replacing the last buffer value with the data value
                     self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
                     self.__buffer[self.n-1] = data
                     self.__profIndex = self.n
                     return
                 def pushData(self):
                     """
                     Return the sum of the last profiles and the profiles used in the sum.
                     Affected:
                     self.__profileIndex
                     """
                     if not self.__withOverapping:
                         data = self.__buffer
                         n = self.__profIndex
                         self.__buffer = 0
                         self.__profIndex = 0
                         return data, n
                     #Integration with Overlapping
                     data = numpy.sum(self.__buffer, axis=0)
                     n = self.__profIndex
                     return data, n
                 def byProfiles(self, data):
                     self.__dataReady = False
                     avgdata = None
             #         n = None
                     self.putData(data)
                     if self.__profIndex == self.n:
                         avgdata, n = self.pushData()
                         self.__dataReady = True
                     return avgdata
                 def byTime(self, data, datatime):
                     self.__dataReady = False
                     avgdata = None
                     n = None
                     self.putData(data)
                     if (datatime - self.__initime) >= self.__integrationtime:
                         avgdata, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata
                 def integrate(self, data, datatime=None):
                     if self.__initime == None:
                         self.__initime = datatime
                     if self.__byTime:
                         avgdata = self.byTime(data, datatime)
                     else:
                         avgdata = self.byProfiles(data)
                     self.__lastdatatime = datatime
                     if avgdata == None:
                         return None, None
                     avgdatatime = self.__initime
                     deltatime = datatime -self.__lastdatatime
                     if not self.__withOverapping:
                         self.__initime = datatime
                     else:
                         self.__initime += deltatime
                     return avgdata, avgdatatime
                 def integrateByBlock(self, dataOut):
                     times = int(dataOut.data.shape[1]/self.n)
                     avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
                     id_min = 0
                     id_max = self.n
                     for i in range(times):
                         junk = dataOut.data[:,id_min:id_max,:]
                         avgdata[:,i,:] = junk.sum(axis=1)
                         id_min += self.n
                         id_max += self.n
                     timeInterval = dataOut.ippSeconds*self.n
                     avgdatatime = (times - 1) * timeInterval + dataOut.utctime
                     self.__dataReady = True
                     return avgdata, avgdatatime
                 def run(self, dataOut, **kwargs):
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     if self.byblock:
                         avgdata, avgdatatime = self.integrateByBlock(dataOut)
                     else:
                         avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
             #        dataOut.timeInterval *= n
                     dataOut.flagNoData = True
                     if self.__dataReady:
                         dataOut.data = avgdata
                         dataOut.nCohInt *= self.n
                         dataOut.utctime = avgdatatime
                         dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
                         dataOut.flagNoData = False
             class Decoder(Operation):
                 isConfig = False
                 __profIndex = 0
                 code = None
                 nCode = None
                 nBaud = None
                 def __init__(self):
                     Operation.__init__(self)
                     self.times = None
                     self.osamp = None
                     self.__setValues = False
             #         self.isConfig = False
                 def setup(self, code, shape, times, osamp):
                     self.__profIndex = 0
                     self.code = code
                     self.nCode = len(code)
                     self.nBaud = len(code[0])
                     if times != None:
                         self.times = times
                     if ((osamp != None) and (osamp >1)):
                         self.osamp = osamp
                         self.code = numpy.repeat(code, repeats=self.osamp,axis=1)
                         self.nBaud = self.nBaud*self.osamp
                     if len(shape) == 2:
                         self.__nChannels, self.__nHeis = shape
                         __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
                         __codeBuffer[:,0:self.nBaud] = self.code
                         self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
                         self.ndatadec = self.__nHeis - self.nBaud + 1
                         self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
                     else:
                         self.__nChannels, self.__nProfiles, self.__nHeis = shape
                         self.ndatadec = self.__nHeis - self.nBaud + 1
                         self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
                 def convolutionInFreq(self, data):
                     fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
                     fft_data = numpy.fft.fft(data, axis=1)
                     conv = fft_data*fft_code
                     data = numpy.fft.ifft(conv,axis=1)
                     datadec = data[:,:-self.nBaud+1]
                     return datadec
                 def convolutionInFreqOpt(self, data):
                     fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
                     data = cfunctions.decoder(fft_code, data)
                     datadec = data[:,:-self.nBaud+1]
                     return datadec
                 def convolutionInTime(self, data):
                     code = self.code[self.__profIndex]
                     for i in range(self.__nChannels):
                         self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='valid')
                     return self.datadecTime
                 def convolutionByBlockInTime(self, data):
                     junk = numpy.lib.stride_tricks.as_strided(self.code, (self.times, self.code.size), (0, self.code.itemsize))
                     junk = junk.flatten()
                     code_block = numpy.reshape(junk, (self.nCode*self.times,self.nBaud))
                     for i in range(self.__nChannels):
                         for j in range(self.__nProfiles):
                             self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='valid')
                     return self.datadecTime
                 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, times=None, osamp=None):
                     if code == None:
                         code = dataOut.code
                     else:
                         code = numpy.array(code).reshape(nCode,nBaud)
                     if not self.isConfig:
                         self.setup(code, dataOut.data.shape, times, osamp)
                         dataOut.code = code
                         dataOut.nCode = nCode
                         dataOut.nBaud = nBaud
                         dataOut.radarControllerHeaderObj.code = code
                         dataOut.radarControllerHeaderObj.nCode = nCode
                         dataOut.radarControllerHeaderObj.nBaud = nBaud
                         self.isConfig = True
                     if mode == 0:
                         datadec = self.convolutionInTime(dataOut.data)
                     if mode == 1:
                         datadec = self.convolutionInFreq(dataOut.data)
                     if mode == 2:
                         datadec = self.convolutionInFreqOpt(dataOut.data)
                     if mode == 3:
                         datadec = self.convolutionByBlockInTime(dataOut.data)
                     if not(self.__setValues):
                         dataOut.code = self.code
                         dataOut.nCode = self.nCode
                         dataOut.nBaud = self.nBaud
                         dataOut.radarControllerHeaderObj.code = self.code
                         dataOut.radarControllerHeaderObj.nCode = self.nCode
                         dataOut.radarControllerHeaderObj.nBaud = self.nBaud
                         self.__setValues = True
                     dataOut.data = datadec
                     dataOut.heightList = dataOut.heightList[0:self.ndatadec]
                     dataOut.flagDecodeData = True #asumo q la data no esta decodificada
                     if self.__profIndex == self.nCode-1:
                         self.__profIndex = 0
                         return 1
                     self.__profIndex += 1
                     return 1
             #        dataOut.flagDeflipData = True #asumo q la data no esta sin flip
             class ProfileConcat(Operation):
                 isConfig = False
                 buffer = None
                 def __init__(self):
                     Operation.__init__(self)
                     self.profileIndex = 0
                 def reset(self):
                     self.buffer = numpy.zeros_like(self.buffer)
                     self.start_index = 0
                     self.times = 1
                 def setup(self, data, m, n=1):
                     self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
                     self.profiles = data.shape[1]
                     self.start_index = 0
                     self.times = 1
                 def concat(self, data):
                     self.buffer[:,self.start_index:self.profiles*self.times] = data.copy()
                     self.start_index = self.start_index + self.profiles
                 def run(self, dataOut, m):
                     dataOut.flagNoData = True
                     if not self.isConfig:
                         self.setup(dataOut.data, m, 1)
                         self.isConfig = True
                     self.concat(dataOut.data)
                     self.times += 1
                     if self.times > m:
                         dataOut.data = self.buffer
                         self.reset()
                         dataOut.flagNoData = False
                         # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
                         deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                         xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * 5
                         dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
             class ProfileSelector(Operation):
                 profileIndex = None
                 # Tamanho total de los perfiles
                 nProfiles = None
                 def __init__(self):
                     Operation.__init__(self)
                     self.profileIndex = 0
                 def incIndex(self):
                     self.profileIndex += 1
                     if self.profileIndex >= self.nProfiles:
                         self.profileIndex = 0
                 def isProfileInRange(self, minIndex, maxIndex):
                     if self.profileIndex < minIndex:
                         return False
                     if self.profileIndex > maxIndex:
                         return False
                     return True
                 def isProfileInList(self, profileList):
                     if self.profileIndex not in profileList:
                         return False
                     return True
                 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False):
                     dataOut.flagNoData = True
                     self.nProfiles = dataOut.nProfiles
                     if byblock:
                         if profileList != None:
                             dataOut.data = dataOut.data[:,profileList,:]
                             pass
                         else:
                             pmin = profileRangeList[0]
                             pmax = profileRangeList[1]
                             dataOut.data = dataOut.data[:,pmin:pmax+1,:]
                         dataOut.flagNoData = False
                         self.profileIndex = 0
                         return 1
                     if profileList != None:
                         if self.isProfileInList(profileList):
                             dataOut.flagNoData = False
                         self.incIndex()
                         return 1
                     elif profileRangeList != None:
                         minIndex = profileRangeList[0]
                         maxIndex = profileRangeList[1]
                         if self.isProfileInRange(minIndex, maxIndex):
                             dataOut.flagNoData = False
                         self.incIndex()
                         return 1
                     elif beam != None: #beam is only for AMISR data
                         if self.isProfileInList(dataOut.beamRangeDict[beam]):
                             dataOut.flagNoData = False
                         self.incIndex()
                         return 1
                     else:
                         raise ValueError, "ProfileSelector needs profileList or profileRangeList"
                     return 0
             class Reshaper(Operation):
                 def __init__(self):
                     Operation.__init__(self)
                     self.updateNewHeights = False
                 def run(self, dataOut, shape):
                     shape_tuple = tuple(shape)
                     dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
                     dataOut.flagNoData = False
                     if not(self.updateNewHeights):
                         old_nheights = dataOut.nHeights
                         new_nheights = dataOut.data.shape[2]
                         factor = new_nheights / old_nheights
                         deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                         xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * factor
                         dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages