From b1b398996ebbb1d51385b3d5b1a5bf03e86af84f 2021-12-17 19:11:00 From: joabAM Date: 2021-12-17 19:11:00 Subject: [PATCH] Lectura y procesamiento de spectra desde HDF5 --- diff --git a/schainpy/model/data/jrodata.py b/schainpy/model/data/jrodata.py index 2ccf15c..39ae51f 100644 --- a/schainpy/model/data/jrodata.py +++ b/schainpy/model/data/jrodata.py @@ -465,6 +465,9 @@ class Spectra(JROData): self.ippFactor = 1 self.beacon_heiIndexList = [] self.noise_estimation = None + self.codeList = [] + self.azimuthList = [] + self.elevationList = [] self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt', 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp','nIncohInt', 'nFFTPoints', 'nProfiles'] @@ -866,6 +869,7 @@ class Parameters(Spectra): Constructor ''' self.radarControllerHeaderObj = RadarControllerHeader() + self.radarControllerHeaderObj.set_ippSeconds(0) self.systemHeaderObj = SystemHeader() self.type = "Parameters" self.timeZone = 0 diff --git a/schainpy/model/graphics/jroplot_base.py b/schainpy/model/graphics/jroplot_base.py index 809e655..fb3444c 100644 --- a/schainpy/model/graphics/jroplot_base.py +++ b/schainpy/model/graphics/jroplot_base.py @@ -497,7 +497,6 @@ class Plot(Operation): 'interval': dataOut.timeInterval, 'channels': dataOut.channelList } - data, meta = self.update(dataOut) metadata.update(meta) self.data.update(data, timestamp, metadata) diff --git a/schainpy/model/graphics/jroplot_parameters.py b/schainpy/model/graphics/jroplot_parameters.py index 4c46a24..732060d 100644 --- a/schainpy/model/graphics/jroplot_parameters.py +++ b/schainpy/model/graphics/jroplot_parameters.py @@ -84,7 +84,6 @@ class PowerPlot(RTIPlot): data = { 'pow': 10*numpy.log10(dataOut.data_pow) } - #print("data",data) return data, {} class SpectralWidthPlot(RTIPlot): diff --git a/schainpy/model/graphics/jroplot_spectra.py b/schainpy/model/graphics/jroplot_spectra.py index d537a10..dc830df 100644 --- a/schainpy/model/graphics/jroplot_spectra.py +++ b/schainpy/model/graphics/jroplot_spectra.py @@ -12,6 +12,7 @@ import numpy from schainpy.model.graphics.jroplot_base import Plot, plt, log from itertools import combinations + class SpectraPlot(Plot): ''' Plot for Spectra data @@ -36,10 +37,12 @@ class SpectraPlot(Plot): self.width = 3.5 * self.ncols self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08}) self.ylabel = 'Range [km]' + def update_list(self,dataOut): + if len(self.channelList) == 0: + self.channelList = dataOut.channelList def update(self, dataOut): - if self.channelList == None: - self.channelList = dataOut.channelList + self.update_list(dataOut) data = {} meta = {} spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor) @@ -178,9 +181,9 @@ class CrossSpectraPlot(Plot): data = self.data[-1] cspc = data['cspc'] - #print(self.crossPairs) + for n in range(len(self.data.pairs)): - #pair = self.data.pairs[n] + pair = self.crossPairs[n] coh = cspc[n*2] @@ -238,6 +241,7 @@ class RTIPlot(Plot): if len(self.channelList) == 0: self.channelList = dataOut.channelList + def update(self, dataOut): self.update_list(dataOut) data = {} diff --git a/schainpy/model/io/jroIO_base.py b/schainpy/model/io/jroIO_base.py index b9d8ebc..10cee2c 100644 --- a/schainpy/model/io/jroIO_base.py +++ b/schainpy/model/io/jroIO_base.py @@ -691,7 +691,7 @@ class Reader(object): """Check if the given datetime is in range""" startDateTime= datetime.datetime.combine(startDate,startTime) endDateTime = datetime.datetime.combine(endDate,endTime) - #print("dt eval: ", dt, startDateTime,endDateTime) + if startDateTime <= dt <= endDateTime: return True return False diff --git a/schainpy/model/io/jroIO_kamisr.py b/schainpy/model/io/jroIO_kamisr.py index 9199145..bd3a28b 100644 --- a/schainpy/model/io/jroIO_kamisr.py +++ b/schainpy/model/io/jroIO_kamisr.py @@ -644,7 +644,7 @@ class AMISRReader(ProcessingUnit): self.profileIndex += 1 - return self.dataOut.data + return self.dataOut.data #retorno necesario?? def run(self, **kwargs): diff --git a/schainpy/model/io/jroIO_param.py b/schainpy/model/io/jroIO_param.py index b26b7be..cd93488 100644 --- a/schainpy/model/io/jroIO_param.py +++ b/schainpy/model/io/jroIO_param.py @@ -82,7 +82,7 @@ class HDFReader(Reader, ProcessingUnit): def __init__(self): ProcessingUnit.__init__(self) - self.dataOut = Parameters() + self.ext = ".hdf5" self.optchar = "D" self.meta = {} @@ -95,6 +95,10 @@ class HDFReader(Reader, ProcessingUnit): self.folderfmt = "*%Y%j" self.utcoffset = 0 + self.dataOut = Parameters() + self.dataOut.error=False ## NOTE: Importante definir esto antes inicio + self.dataOut.flagNoData = True + def setup(self, **kwargs): self.set_kwargs(**kwargs) @@ -109,7 +113,7 @@ class HDFReader(Reader, ProcessingUnit): self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt) pathname, filename = os.path.split(fullpath) - #print(pathname,filename) + try: fullpath = next(fullpath) @@ -140,7 +144,7 @@ class HDFReader(Reader, ProcessingUnit): self.setNextFile() - return + def readFirstHeader(self): @@ -150,16 +154,12 @@ class HDFReader(Reader, ProcessingUnit): self.__readData() self.__setBlockList() - if 'type' in self.meta: - ##print("Creting dataOut...") - self.dataOut = eval(self.meta['type'])() - ##print(vars(self.dataOut)) for attr in self.meta: - ##print("attr: ", attr) - ##print(type(self.dataOut).__name__) - setattr(self.dataOut, attr, self.meta[attr]) + setattr(self.dataOut, attr, self.meta[attr]) + self.dataOut.error=False + self.dataOut.flagNoData = False self.blockIndex = 0 return @@ -276,10 +276,10 @@ class HDFReader(Reader, ProcessingUnit): if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime): self.dataOut.flagNoData = True self.blockIndex = self.blocksPerFile - #self.dataOut.error = True TERMINA EL PROGRAMA, removido + self.dataOut.error = True # TERMINA EL PROGRAMA return for attr in self.data: - #print("attr ",attr) + if self.data[attr].ndim == 1: setattr(self.dataOut, attr, self.data[attr][self.blockIndex]) else: @@ -298,9 +298,11 @@ class HDFReader(Reader, ProcessingUnit): self.blockIndex, self.blocksPerFile),self.name) + if self.blockIndex == self.blocksPerFile: + self.setNextFile() + self.dataOut.flagNoData = False - self.dataOut.error = False - return + def run(self, **kwargs): @@ -308,13 +310,8 @@ class HDFReader(Reader, ProcessingUnit): self.setup(**kwargs) self.isConfig = True - if self.blockIndex == self.blocksPerFile: - self.setNextFile() - self.getData() - return - @MPDecorator class HDFWriter(Operation): """Operation to write HDF5 files. @@ -645,7 +642,6 @@ class HDFWriter(Operation): if (self.blockIndex == self.blocksPerFile) or self.timeFlag(): self.closeFile() self.setNextFile() - print("breaking file") for i, ds in enumerate(self.ds): attr, ch = self.data[i] diff --git a/schainpy/model/proc/jroproc_base.py b/schainpy/model/proc/jroproc_base.py index 053dd92..0402dad 100644 --- a/schainpy/model/proc/jroproc_base.py +++ b/schainpy/model/proc/jroproc_base.py @@ -32,6 +32,7 @@ class ProcessingUnit(object): self.dataIn = unit.dataOut + def getAllowedArgs(self): if hasattr(self, '__attrs__'): return self.__attrs__ @@ -64,6 +65,7 @@ class ProcessingUnit(object): self.dataOut.error = self.dataIn.error self.dataOut.flagNoData = True except: + err = traceback.format_exc() if 'SchainWarning' in err: log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), self.name) diff --git a/schainpy/model/proc/jroproc_parameters.py b/schainpy/model/proc/jroproc_parameters.py index abc4bb6..6611521 100755 --- a/schainpy/model/proc/jroproc_parameters.py +++ b/schainpy/model/proc/jroproc_parameters.py @@ -142,6 +142,7 @@ class ParametersProc(ProcessingUnit): # self.dataOut.normFactor = self.dataIn.normFactor self.dataOut.pairsList = self.dataIn.pairsList self.dataOut.groupList = self.dataIn.pairsList + self.dataOut.flagNoData = False if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels @@ -183,7 +184,6 @@ class ParametersProc(ProcessingUnit): if self.dataIn.type == "Parameters": self.dataOut.copy(self.dataIn) self.dataOut.flagNoData = False - self.prin("DAta In") return True self.__updateObjFromInput() diff --git a/schainpy/model/proc/jroproc_spectra.py b/schainpy/model/proc/jroproc_spectra.py index ae0c9cd..a2cf3c6 100644 --- a/schainpy/model/proc/jroproc_spectra.py +++ b/schainpy/model/proc/jroproc_spectra.py @@ -20,7 +20,7 @@ from schainpy.model.data.jrodata import hildebrand_sekhon from schainpy.utils import log from scipy.optimize import curve_fit - +SPEED_OF_LIGHT = 299792458 class SpectraProc(ProcessingUnit): @@ -68,6 +68,8 @@ class SpectraProc(ProcessingUnit): self.dataOut.azimuthList = self.dataIn.azimuthList self.dataOut.elevationList = self.dataIn.elevationList + + def __getFft(self): """ Convierte valores de Voltaje a Spectra @@ -121,7 +123,7 @@ class SpectraProc(ProcessingUnit): self.dataOut.flagShiftFFT = False def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False): - #print("spectra run") + if self.dataIn.type == "Spectra": self.dataOut.copy(self.dataIn) if shift_fft: @@ -135,6 +137,7 @@ class SpectraProc(ProcessingUnit): if pairsList: self.__selectPairs(pairsList) + elif self.dataIn.type == "Voltage": self.dataOut.flagNoData = True @@ -546,7 +549,6 @@ class CleanRayleigh(Operation): self.currentTime = self.__initime self.pairsArray = numpy.array(dataOut.pairsList) self.factor_stdv = factor_stdv - #print("CHANNELS: ",[x for x in self.channels]) if n != None : self.byProfiles = True @@ -560,11 +562,11 @@ class CleanRayleigh(Operation): def run(self, dataOut,min_hei=None,max_hei=None, n=None, timeInterval=10,factor_stdv=2.5): - #print (dataOut.utctime) + if not self.isConfig : - #print("Setting config") + self.setup(dataOut, min_hei,max_hei,n,timeInterval,factor_stdv) - #print("Config Done") + tini=dataOut.utctime if self.byProfiles: @@ -572,14 +574,14 @@ class CleanRayleigh(Operation): self.__dataReady = True else: if (tini - self.__initime) >= self.__integrationtime: - #print(tini - self.__initime,self.__profIndex) + self.__dataReady = True self.__initime = tini #if (tini.tm_min % 2) == 0 and (tini.tm_sec < 5 and self.fint==0): if self.__dataReady: - #print("Data ready",self.__profIndex) + self.__profIndex = 0 jspc = self.buffer jcspc = self.buffer2 @@ -1075,7 +1077,6 @@ class IntegrationFaradaySpectra(Operation): buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k]) outliers_IDs_cspc=[] cspc_outliers_exist=False - #print("AQUIII") for i in range(self.nChannels):#dataOut.nChannels): buffer1=numpy.copy(self.__buffer_spc[:,i,:,k])