schain Commit - r497:85140003652c · Jicamarca Repository

Bug fixed: AMISR Reader filling with zeros at the begining of the processing....

Daniel Valdez -

r497:85140003652c

parent child

Context file:

r497:85140003652c

Collapse all files

schainpy/model/data/jroamisr.py +3 -1

                     self.type = "AMISR"
                     #propiedades para compatibilidad con Voltages
-                    self.timeZone = 300#timezone like jroheader, difference in minutes between UTC and localtime
+                    self.timeZone = 0#timezone like jroheader, difference in minutes between UTC and localtime
                     self.dstFlag = 0#self.dataIn.dstFlag
                     self.errorCount = 0#self.dataIn.errorCount
                     self.useLocalTime = True#self.dataIn.useLocalTime
                     self.beamCodeDict = None
                     self.beamRangeDict = None
+                    self.npulseByFrame = None
                 def copy(self, inputObj=None):
                     if inputObj == None:

schainpy/model/graphics/figure.py +1 -1

             import numpy
             import time, datetime
             import mpldriver
-            from customftp import *
             import Queue
             import threading

schainpy/model/io/jroIO_amisr.py +67 -36

             import glob
             import fnmatch
             import datetime
+            import time
             import re
             import h5py
             import numpy
                     self.npulses = self.pulseCount.shape[1] #nprofile
                     self.nsamples = self.nsamplesPulse[0,0] #ngates
                     self.nbeams = self.beamCode.shape[1]
                 def getIndexRangeToPulse(self, idrecord=0):
-                    indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
+                    #indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
-                    startPulseCountId = indexToZero[0][0]
+                    #startPulseCountId = indexToZero[0][0]
-                    endPulseCountId = startPulseCountId - 1
+                    #endPulseCountId = startPulseCountId - 1
-                    range1 = numpy.arange(startPulseCountId,self.npulses,1)
+                    #range1 = numpy.arange(startPulseCountId,self.npulses,1)
-                    range2 = numpy.arange(0,startPulseCountId,1)
+                    #range2 = numpy.arange(0,startPulseCountId,1)
-                    return range1, range2
+                    #return range1, range2
+                    looking_zeros_index = numpy.where(self.pulseCount.value[idrecord,:]==0)[0]
+                    getLastIndexZero = looking_zeros_index[-1]
+                    index_data = numpy.arange(0,getLastIndexZero,1)
+                    index_buffer = numpy.arange(getLastIndexZero,self.npulses,1)
+                    return index_data, index_buffer
             class AMISRReader(ProcessingUnit):
                     self.range = None
                     self.idrecord_count = 0
                     self.profileIndex = 0
-                    self.idpulse_range1 = None
+                    self.index_amisr_sample = None
-                    self.idpulse_range2 = None
+                    self.index_amisr_buffer = None
                     self.beamCodeByFrame = None
                     self.radacTimeByFrame = None
                     #atributos originales tal y como esta en el archivo de datos
                     self.__firstFile = True
                     self.buffer_radactime = None
+                    self.index4_schain_datablock = None
+                    self.index4_buffer = None
+                    self.schain_datablock = None
+                    self.buffer = None
+                    self.linear_pulseCount = None
+                    self.npulseByFrame = None
+                    self.profileIndex_offset = None
+                    self.timezone = 'ut'
                 def __createObjByDefault(self):
                     dataObj = AMISR()
                     self.radacHeaderObj.npulses = self.npulsesint_fromfile
                     self.radacHeaderObj.nsamples = self.ngates_fromfile
+                    #looking index list for data
+                    start_index = self.radacHeaderObj.pulseCount[0,:][0]
+                    end_index = self.radacHeaderObj.npulses
+                    range4data = range(start_index, end_index)
+                    self.index4_schain_datablock = numpy.array(range4data)
+                    buffer_start_index = 0
+                    buffer_end_index = self.radacHeaderObj.pulseCount[0,:][0]
+                    range4buffer = range(buffer_start_index, buffer_end_index)
+                    self.index4_buffer = numpy.array(range4buffer)
+                    self.linear_pulseCount = numpy.array(range4data + range4buffer)
+                    self.npulseByFrame = max(self.radacHeaderObj.pulseCount[0,:]+1)
                     #get tuning frequency
                     frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency')
                     self.frequency_h5file = frequency_h5file_dataset[0,0]
                     self.dataByFrame = self.__setDataByFrame()
                     self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :]
                     self.readRanges()
-                    self.idpulse_range1, self.idpulse_range2 = self.radacHeaderObj.getIndexRangeToPulse(0)
+                    self.index_amisr_sample, self.index_amisr_buffer = self.radacHeaderObj.getIndexRangeToPulse(0)
                     self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses)
                     self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame)
                                 endDate=None,
                                 startTime=datetime.time(0,0,0),
                                 endTime=datetime.time(23,59,59),
-                                walk=True):
+                                walk=True,
+                                timezone='ut',):
+                    self.timezone = timezone
                     #Busqueda de archivos offline
                     self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
                     self.fileIndex = -1
                     self.__setNextFile()
+                    first_beamcode = self.radacHeaderObj.beamCodeByPulse[0,0]
+                    index = numpy.where(self.radacHeaderObj.beamCodeByPulse[0,:]!=first_beamcode)[0][0]
+                    self.profileIndex_offset = self.radacHeaderObj.pulseCount[0,:][index]
+                    self.profileIndex = self.profileIndex_offset
                 def readRanges(self):
                     dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
                         self.flagIsNewFile = 0
                     if idrecord == 0:
-                        #if self.buffer_last_record == None:
+                        self.dataByFrame[self.index4_schain_datablock, : ,:] = self.dataset[0, self.index_amisr_sample,:,:]
-                        selectorById = self.radacHeaderObj.pulseCount[0,self.idpulse_range2]
+                        self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[0, self.index_amisr_sample]
-                        self.dataByFrame[selectorById,:,:] = self.dataset[0, self.idpulse_range2,:,:]
-                        self.radacTimeByFrame[selectorById] = self.radacHeaderObj.radacTime[0, self.idpulse_range2]
-                        selectorById = self.radacHeaderObj.pulseCount[0,self.idpulse_range1]
-                        self.radacTimeByFrame[selectorById] = self.buffer_radactime[selectorById]
                         datablock = self.__setDataBlock()
+                        self.buffer = self.dataset[0, self.index_amisr_buffer,:,:]
+                        self.buffer_radactime = self.radacHeaderObj.radacTime[0, self.index_amisr_buffer]
                         return datablock
-                    selectorById = self.radacHeaderObj.pulseCount[idrecord-1,self.idpulse_range1]
+                    self.dataByFrame[self.index4_buffer,:,:] = self.buffer.copy()
-                    self.dataByFrame[selectorById,:,:] = self.dataset[idrecord-1, self.idpulse_range1, :, :]
+                    self.radacTimeByFrame[self.index4_buffer] = self.buffer_radactime.copy()
-                    self.radacTimeByFrame[selectorById] = self.radacHeaderObj.radacTime[idrecord-1, self.idpulse_range1]
+                    self.dataByFrame[self.index4_schain_datablock,:,:] = self.dataset[idrecord, self.index_amisr_sample,:,:]
+                    self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_sample]
-                    selectorById = self.radacHeaderObj.pulseCount[idrecord,self.idpulse_range2]#data incompleta ultimo archivo de carpeta, verifica el record real segun la dimension del arreglo de datos
+                    datablock = self.__setDataBlock()
-                    self.dataByFrame[selectorById,:,:] = self.dataset[idrecord, self.idpulse_range2, :, :]
-                    self.radacTimeByFrame[selectorById] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range2]
-                    datablock = self.__setDataBlock()
-                    selectorById = self.radacHeaderObj.pulseCount[idrecord,self.idpulse_range1]
+                    self.buffer = self.dataset[idrecord, self.index_amisr_buffer, :, :]
-                    self.dataByFrame[selectorById,:,:] = self.dataset[idrecord, self.idpulse_range1, :, :]
+                    self.buffer_radactime = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_buffer]
-                    self.buffer_radactime[selectorById] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range1]
                     return datablock
                     self.dataOut.ippSeconds = self.ippSeconds_fromfile
                     self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.frequency_h5file
+                    self.dataOut.npulseByFrame = self.npulseByFrame
                     self.dataOut.nBaud = None
                     self.dataOut.nCode = None
                     self.dataOut.code = None
                     self.dataOut.beamCodeDict = self.beamCodeDict
                     self.dataOut.beamRangeDict = self.beamRangeDict
+                    if self.timezone == 'lt':
+                        self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
+                    else:
+                        self.dataOut.timeZone = 0 #by default time is UTC
                 def getData(self):
                     if self.flagNoMoreFiles:

schainpy/model/proc/jroproc_amisr.py +15 -10

@@ -18,19 +18,24 class AMISRProc(ProcessingUnit):
18		18
19	class PrintInfo(Operation):	19	class PrintInfo(Operation):
20	def __init__(self):	20	def __init__(self):
21	pass	21	self.__isPrinted = False
22		22
23	def run(self, dataOut):	23	def run(self, dataOut):
24		24
25	print 'Number of Records by File: %d'%dataOut.nRecords	25	if not self.__isPrinted:
26	print 'Number of ~~Pulses~~: %d'%dataOut.n~~Profile~~s	26	print 'Number of Records by File: %d'%dataOut.nRecords
27	print 'Number of ~~Samples by~~ Pulse: %d'%~~len~~(dataOut.~~heightList~~)	27	print 'Number of Pulses: %d'%dataOut.nProfiles
28	print 'Ipp Seconds: %f'%dataOut.ippSeconds	28	print 'Number of Pulses by Frame: %d'%dataOut.npulseByFrame
29	print 'Number of ~~Beams~~: %d'%dataOut.~~nBeams~~	29	print 'Number of Samples by Pulse: %d'%len(dataOut.heightList)
30	print 'BeamCodes:'	30	print 'Ipp Seconds: %f'%dataOut.ippSeconds
31	beamStrList = ['Beam %d -> Code %d'%(k,v) for k,v in dataOut.beamCodeDict.items()]	31	print 'Number of Beams: %d'%dataOut.nBeams
32	for b in beamStrList:	32	print 'BeamCodes:'
33	print b	33	beamStrList = ['Beam %d -> Code %d'%(k,v) for k,v in dataOut.beamCodeDict.items()]
		34	for b in beamStrList:
		35	print b
		36	self.__isPrinted = True
		37
		38	return
34		39
35		40
36	class BeamSelector(Operation):	41	class BeamSelector(Operation):

schainpy/test/amisr_proc.py +15 -11

             controllerObj.setup(id = '191', name='test01', description=desc)
-            path = os.path.join(os.environ['HOME'],'Documents/amisr')
+            path = os.path.join(os.environ['HOME'],'Development/amisr/data')
+            #path = os.path.join(os.environ['HOME'],'Documents/amisr')
             figpath = os.path.join(os.environ['HOME'],'Pictures/amisr')
             readUnitConfObj = controllerObj.addReadUnit(datatype='AMISRReader',
                                                         path=path,
-                                                        startDate='2014/08/18',
+                                                        startDate='2014/01/06',
-                                                        endDate='2014/08/18',
+                                                        endDate='2014/12/06',
                                                         startTime='00:00:00',
                                                         endTime='23:59:59',
-                                                        walk=1)
+                                                        walk=1,
+                                                        timezone='lt')
             #AMISR Processing Unit
             procUnitAMISRBeam0 = controllerObj.addProcUnit(datatype='AMISRProc', inputId=readUnitConfObj.getId())
+            opObj11 = procUnitAMISRBeam0.addOperation(name='PrintInfo', optype='other')
             #Beam Selector
             opObj11 = procUnitAMISRBeam0.addOperation(name='BeamSelector', optype='other')
             opObj11.addParameter(name='beam', value='0', format='int')
             #Noise Estimation
             opObj11 = procUnitConfObjSpectraBeam0.addOperation(name='getNoise')
             opObj11.addParameter(name='minHei', value='100', format='float')
-            opObj11.addParameter(name='maxHei', value='450', format='float')
+            opObj11.addParameter(name='maxHei', value='300', format='float')
             #SpectraPlot
             opObj11 = procUnitConfObjSpectraBeam0.addOperation(name='SpectraPlot', optype='other')
             opObj11.addParameter(name='wintitle', value=title0, format='str')
             opObj11.addParameter(name='showprofile', value='0', format='int')
             #Setting RTI time using xmin,xmax
-            opObj11.addParameter(name='xmin', value='0', format='int')
+            opObj11.addParameter(name='xmin', value='15', format='int')
-            opObj11.addParameter(name='xmax', value='18', format='int')
+            opObj11.addParameter(name='xmax', value='23', format='int')
             #Setting dB range with zmin, zmax
-            opObj11.addParameter(name='zmin', value='45', format='int')
+            #opObj11.addParameter(name='zmin', value='45', format='int')
-            opObj11.addParameter(name='zmax', value='70', format='int')
+            #opObj11.addParameter(name='zmax', value='70', format='int')
             #Save RTI
-            figfile0 = 'amisr_rti_beam0.png'
+            #figfile0 = 'amisr_rti_beam0.png'
-            opObj11.addParameter(name='figpath', value=figpath, format='str')
+            #opObj11.addParameter(name='figpath', value=figpath, format='str')
             #opObj11.addParameter(name='figfile', value=figfile0, format='str')

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages