diff --git a/schainpy/model/data/jrodata.py b/schainpy/model/data/jrodata.py index 6a59848..daa22c1 100644 --- a/schainpy/model/data/jrodata.py +++ b/schainpy/model/data/jrodata.py @@ -65,21 +65,38 @@ def hildebrand_sekhon(data, navg): mean : noise's level """ - sorted_spectrum = numpy.sort(data, axis=None) - nnoise = len(sorted_spectrum) # default to all points in the spectrum as noise - for npts in range(1, len(sorted_spectrum)+1): - partial = sorted_spectrum[:npts] - mean = partial.mean() - var = partial.var() - if var * navg < mean**2.: - nnoise = npts - else: - # partial spectrum no longer has characteristics of white noise - break + sortdata = numpy.sort(data, axis=None) + lenOfData = len(sortdata) + nums_min = lenOfData*0.2 + + if nums_min <= 5: + + nums_min = 5 + + sump = 0. + sumq = 0. + + j = 0 + cont = 1 + + while((cont==1)and(j nums_min: + rtest = float(j)/(j-1) + 1.0/navg + if ((sumq*j) > (rtest*sump**2)): + j = j - 1 + sump = sump - sortdata[j] + sumq = sumq - sortdata[j]**2 + cont = 0 + + j += 1 + + lnoise = sump /j - noise_spectrum = sorted_spectrum[:nnoise] - mean = noise_spectrum.mean() - return mean + return lnoise class Beam: diff --git a/schainpy/model/graphics/jroplot_parameters.py b/schainpy/model/graphics/jroplot_parameters.py index dbdfb52..f3259ac 100644 --- a/schainpy/model/graphics/jroplot_parameters.py +++ b/schainpy/model/graphics/jroplot_parameters.py @@ -4,7 +4,8 @@ import numpy import inspect from .figure import Figure, isRealtime, isTimeInHourRange from .plotting_codes import * - +from schainpy.model.proc.jroproc_base import MPDecorator +from schainpy.utils import log class FitGauPlot(Figure): @@ -225,8 +226,8 @@ class MomentsPlot(Figure): WIDTHPROF = None HEIGHTPROF = None PREFIX = 'prm' - def __init__(self, **kwargs): - Figure.__init__(self, **kwargs) + def __init__(self): + Figure.__init__(self) self.isConfig = False self.__nsubplots = 1 @@ -404,7 +405,6 @@ class MomentsPlot(Figure): thisDatetime=thisDatetime) - class SkyMapPlot(Figure): __isConfig = None @@ -773,7 +773,7 @@ class WindProfilerPlot(Figure): self.isConfig = False update_figfile = True - +@MPDecorator class ParametersPlot(Figure): __isConfig = None @@ -786,8 +786,8 @@ class ParametersPlot(Figure): nplots = None nchan = None - def __init__(self, **kwargs): - Figure.__init__(self, **kwargs) + def __init__(self):#, **kwargs): + Figure.__init__(self)#, **kwargs) self.timerange = None self.isConfig = False self.__nsubplots = 1 @@ -866,6 +866,9 @@ class ParametersPlot(Figure): zmin : None, zmax : None """ + if dataOut.flagNoData: + return dataOut + if HEIGHT is not None: self.HEIGHT = HEIGHT @@ -981,8 +984,8 @@ class ParametersPlot(Figure): thisDatetime=thisDatetime, update_figfile=update_figfile) - - + return dataOut +@MPDecorator class Parameters1Plot(Figure): __isConfig = None @@ -992,8 +995,8 @@ class Parameters1Plot(Figure): HEIGHTPROF = None PREFIX = 'prm' - def __init__(self, **kwargs): - Figure.__init__(self, **kwargs) + def __init__(self): + Figure.__init__(self) self.timerange = 2*60*60 self.isConfig = False self.__nsubplots = 1 @@ -1080,6 +1083,8 @@ class Parameters1Plot(Figure): zmin : None, zmax : None """ + if dataOut.flagNoData: + return dataOut data_param = getattr(dataOut, parameterObject) @@ -1230,6 +1235,7 @@ class Parameters1Plot(Figure): wr_period=wr_period, thisDatetime=thisDatetime, update_figfile=False) + return dataOut class SpectralFittingPlot(Figure): @@ -2148,4 +2154,5 @@ class NSMeteorDetection2Plot(Figure): save=save, ftp=ftp, wr_period=wr_period, - thisDatetime=thisDatetime) \ No newline at end of file + thisDatetime=thisDatetime) + \ No newline at end of file diff --git a/schainpy/model/graphics/jroplot_spectra.py b/schainpy/model/graphics/jroplot_spectra.py index 24f8702..d6ea2b6 100644 --- a/schainpy/model/graphics/jroplot_spectra.py +++ b/schainpy/model/graphics/jroplot_spectra.py @@ -23,8 +23,8 @@ class SpectraPlot(Figure): HEIGHTPROF = None PREFIX = 'spc' - def __init__(self):#, **kwargs): - Figure.__init__(self)#, **kwargs) + def __init__(self): + Figure.__init__(self) self.isConfig = False self.__nsubplots = 1 self.WIDTH = 250 @@ -237,8 +237,8 @@ class CrossSpectraPlot(Figure): HEIGHTPROF = None PREFIX = 'cspc' - def __init__(self):#, **kwargs): - Figure.__init__(self)#, **kwargs) + def __init__(self): + Figure.__init__(self) self.isConfig = False self.__nsubplots = 4 self.counter_imagwr = 0 @@ -462,9 +462,9 @@ class RTIPlot(Figure): HEIGHTPROF = None PREFIX = 'rti' - def __init__(self):#, **kwargs): + def __init__(self): - Figure.__init__(self)#, **kwargs) + Figure.__init__(self) self.timerange = None self.isConfig = False self.__nsubplots = 1 @@ -576,7 +576,7 @@ class RTIPlot(Figure): else: factor = normFactor -# factor = dataOut.normFactor + #factor = dataOut.normFactor x = dataOut.getTimeRange() y = dataOut.getHeiRange() @@ -588,7 +588,7 @@ class RTIPlot(Figure): thisDatetime = dataOut.datatime -# thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0]) + #thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0]) title = wintitle + " RTI" #: %s" %(thisDatetime.strftime("%d-%b-%Y")) xlabel = "" ylabel = "Range (Km)" @@ -675,8 +675,8 @@ class CoherenceMap(Figure): HEIGHTPROF = None PREFIX = 'cmap' - def __init__(self):#, **kwargs): - Figure.__init__(self)#, **kwargs) + def __init__(self): + Figure.__init__(self) self.timerange = 2*60*60 self.isConfig = False self.__nsubplots = 1 @@ -887,8 +887,8 @@ class PowerProfilePlot(Figure): HEIGHTPROF = None PREFIX = 'spcprofile' - def __init__(self):#, **kwargs): - Figure.__init__(self)#, **kwargs) + def __init__(self): + Figure.__init__(self) self.isConfig = False self.__nsubplots = 1 @@ -1017,8 +1017,8 @@ class SpectraCutPlot(Figure): HEIGHTPROF = None PREFIX = 'spc_cut' - def __init__(self):#, **kwargs): - Figure.__init__(self)#, **kwargs) + def __init__(self): + Figure.__init__(self) self.isConfig = False self.__nsubplots = 1 @@ -1153,8 +1153,8 @@ class Noise(Figure): PREFIX = 'noise' - def __init__(self):#, **kwargs): - Figure.__init__(self)#, **kwargs) + def __init__(self): + Figure.__init__(self) self.timerange = 24*60*60 self.isConfig = False self.__nsubplots = 1 @@ -1359,8 +1359,8 @@ class BeaconPhase(Figure): PREFIX = 'beacon_phase' - def __init__(self):#, **kwargs): - Figure.__init__(self)#, **kwargs) + def __init__(self): + Figure.__init__(self) self.timerange = 24*60*60 self.isConfig = False self.__nsubplots = 1 @@ -1584,4 +1584,4 @@ class BeaconPhase(Figure): thisDatetime=thisDatetime, update_figfile=update_figfile) - return dataOut #Yong \ No newline at end of file + return dataOut \ No newline at end of file diff --git a/schainpy/model/graphics/jroplot_voltage.py b/schainpy/model/graphics/jroplot_voltage.py index 0a1218b..5544143 100644 --- a/schainpy/model/graphics/jroplot_voltage.py +++ b/schainpy/model/graphics/jroplot_voltage.py @@ -229,4 +229,4 @@ class Scope(Figure): wr_period=wr_period, thisDatetime=thisDatetime) - return dataOut \ No newline at end of file + return dataOut \ No newline at end of file diff --git a/schainpy/model/io/jroIO_base.py b/schainpy/model/io/jroIO_base.py index c250f82..2a6c9f4 100644 --- a/schainpy/model/io/jroIO_base.py +++ b/schainpy/model/io/jroIO_base.py @@ -1823,4 +1823,6 @@ class JRODataWriter(JRODataIO): set=set, ext=ext, datatype=datatype, **kwargs) self.isConfig = True - self.putData() \ No newline at end of file + self.dataOut = dataOut + self.putData() + return self.dataOut \ No newline at end of file diff --git a/schainpy/model/io/jroIO_param.py b/schainpy/model/io/jroIO_param.py index e8e5965..2845d8a 100644 --- a/schainpy/model/io/jroIO_param.py +++ b/schainpy/model/io/jroIO_param.py @@ -6,13 +6,14 @@ import re import datetime from schainpy.model.data.jrodata import * -from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation +from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator # from .jroIO_base import * from schainpy.model.io.jroIO_base import * import schainpy +from schainpy.utils import log - -class ParamReader(ProcessingUnit): +@MPDecorator +class ParamReader(JRODataReader,ProcessingUnit): ''' Reads HDF5 format files @@ -74,8 +75,8 @@ class ParamReader(ProcessingUnit): dataOut = None - def __init__(self, **kwargs): - ProcessingUnit.__init__(self, **kwargs) + def __init__(self):#, **kwargs): + ProcessingUnit.__init__(self) #, **kwargs) self.dataOut = Parameters() return @@ -534,7 +535,7 @@ class ParamReader(ProcessingUnit): self.getData() return - +@MPDecorator class ParamWriter(Operation): ''' HDF5 Writer, stores parameters data in HDF5 format files @@ -602,9 +603,9 @@ class ParamWriter(Operation): lastTime = None - def __init__(self, **kwargs): - Operation.__init__(self, **kwargs) - self.isConfig = False + def __init__(self):#, **kwargs): + Operation.__init__(self)#, **kwargs) + #self.isConfig = False return def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs): @@ -614,11 +615,12 @@ class ParamWriter(Operation): self.dataList = dataList self.dataOut = dataOut self.mode = mode - if self.mode is not None: self.mode = numpy.zeros(len(self.dataList)) + mode else: + #self.mode = numpy.ones(len(self.dataList),int) self.mode = numpy.ones(len(self.dataList)) + log.error(self.mode)#yong arrayDim = numpy.zeros((len(self.dataList),5)) @@ -640,6 +642,9 @@ class ParamWriter(Operation): #Not array, just a number #Mode 0 + #log.error(mode)#yong + #log.error(len(mode))#yong + #log.error(type(mode))#yong if type(dataAux)==float or type(dataAux)==int: dsDict['mode'] = 0 dsDict['nDim'] = 0 @@ -647,7 +652,7 @@ class ParamWriter(Operation): dsList.append(dsDict) #Mode 2: meteors - elif mode[i] == 2: + elif self.mode[i] == 2: # dsDict['nDim'] = 0 dsDict['dsName'] = 'table0' dsDict['mode'] = 2 # Mode meteors @@ -656,7 +661,7 @@ class ParamWriter(Operation): dsDict['dsNumber'] = 1 arrayDim[i,3] = dataAux.shape[-1] - arrayDim[i,4] = mode[i] #Mode the data was stored + arrayDim[i,4] = self.mode[i] #Mode the data was stored dsList.append(dsDict) @@ -664,7 +669,7 @@ class ParamWriter(Operation): else: arrayDim0 = dataAux.shape #Data dimensions arrayDim[i,0] = len(arrayDim0) #Number of array dimensions - arrayDim[i,4] = mode[i] #Mode the data was stored + arrayDim[i,4] = self.mode[i] #Mode the data was stored strtable = 'table' dsDict['mode'] = 1 # Mode parameters @@ -846,7 +851,7 @@ class ParamWriter(Operation): os.makedirs(fullpath) setFile = -1 #inicializo mi contador de seteo - if self.setType is None: + if None is None: setFile += 1 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar, timeTuple.tm_year, @@ -902,11 +907,13 @@ class ParamWriter(Operation): for j in range(dsInfo['dsNumber']): dsInfo = dsList[i] tableName = dsInfo['dsName'] - shape = int(dsInfo['shape']) + if dsInfo['nDim'] == 3: + shape = dsInfo['shape'].astype(int) ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True) else: + shape = int(dsInfo['shape']) ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True) ds.append(ds0) @@ -1078,7 +1085,7 @@ class ParamWriter(Operation): self.fp.close() return - def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs): + def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs): if not(self.isConfig): flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile, @@ -1092,4 +1099,5 @@ class ParamWriter(Operation): self.setNextFile() self.putData() - return \ No newline at end of file + return + \ No newline at end of file diff --git a/schainpy/model/io/jroIO_spectra.py b/schainpy/model/io/jroIO_spectra.py index b6fc5cf..ea4b4af 100644 --- a/schainpy/model/io/jroIO_spectra.py +++ b/schainpy/model/io/jroIO_spectra.py @@ -9,6 +9,7 @@ from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader from schainpy.model.data.jrodata import Spectra +from schainpy.utils import log @MPDecorator class SpectraReader(JRODataReader, ProcessingUnit): @@ -388,7 +389,7 @@ class SpectraReader(JRODataReader, ProcessingUnit): self.dataOut.realtime = self.online return self.dataOut.data_spc - +@MPDecorator class SpectraWriter(JRODataWriter, Operation): """ @@ -414,7 +415,7 @@ class SpectraWriter(JRODataWriter, Operation): # dataOut = None - def __init__(self, **kwargs): + def __init__(self):#, **kwargs): """ Inicializador de la clase SpectraWriter para la escritura de datos de espectros. @@ -428,9 +429,9 @@ class SpectraWriter(JRODataWriter, Operation): Return: None """ - Operation.__init__(self, **kwargs) + Operation.__init__(self)#, **kwargs) - self.isConfig = False + #self.isConfig = False self.nTotalBlocks = 0 @@ -594,8 +595,6 @@ class SpectraWriter(JRODataWriter, Operation): # self.setFirstHeader() self.writeNextBlock() - return 1 - def __getBlockSize(self): ''' Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra diff --git a/schainpy/model/io/jroIO_voltage.py b/schainpy/model/io/jroIO_voltage.py index afdfc5f..2293e05 100644 --- a/schainpy/model/io/jroIO_voltage.py +++ b/schainpy/model/io/jroIO_voltage.py @@ -556,7 +556,7 @@ class VoltageReader(JRODataReader, ProcessingUnit): return self.dataOut.data - +@MPDecorator class VoltageWriter(JRODataWriter, Operation): """ Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura @@ -569,7 +569,7 @@ class VoltageWriter(JRODataWriter, Operation): shapeBuffer = None - def __init__(self, **kwargs): + def __init__(self):#, **kwargs): """ Inicializador de la clase VoltageWriter para la escritura de datos de espectros. @@ -578,7 +578,7 @@ class VoltageWriter(JRODataWriter, Operation): Return: None """ - Operation.__init__(self, **kwargs) + Operation.__init__(self)#, **kwargs) self.nTotalBlocks = 0 diff --git a/schainpy/model/proc/jroproc_base.py b/schainpy/model/proc/jroproc_base.py index 78e4451..6e2a4e9 100644 --- a/schainpy/model/proc/jroproc_base.py +++ b/schainpy/model/proc/jroproc_base.py @@ -326,7 +326,7 @@ def MPDecorator(BaseClass): while True: dataOut = self.listen() - + if dataOut == 'end': break diff --git a/schainpy/model/proc/jroproc_parameters.py b/schainpy/model/proc/jroproc_parameters.py index 9328a44..737ed9e 100644 --- a/schainpy/model/proc/jroproc_parameters.py +++ b/schainpy/model/proc/jroproc_parameters.py @@ -17,11 +17,11 @@ import time from scipy.optimize import fmin_l_bfgs_b #optimize with bounds on state papameters -from .jroproc_base import ProcessingUnit, Operation +from .jroproc_base import ProcessingUnit, Operation, MPDecorator from schainpy.model.data.jrodata import Parameters, hildebrand_sekhon from scipy import asarray as ar,exp from scipy.optimize import curve_fit - +from schainpy.utils import log import warnings from numpy import NaN from scipy.optimize.optimize import OptimizeWarning @@ -49,8 +49,10 @@ def _unpickle_method(func_name, obj, cls): break return func.__get__(obj, cls) +@MPDecorator class ParametersProc(ProcessingUnit): + METHODS = {} nSeconds = None def __init__(self): @@ -61,7 +63,8 @@ class ParametersProc(ProcessingUnit): self.firstdatatime = None self.profIndex = 0 self.dataOut = Parameters() - + self.setupReq = False #Agregar a todas las unidades de proc + def __updateObjFromInput(self): self.dataOut.inputUnit = self.dataIn.type @@ -97,7 +100,9 @@ class ParametersProc(ProcessingUnit): # self.dataOut.noise = self.dataIn.noise def run(self): - + + + #---------------------- Voltage Data --------------------------- if self.dataIn.type == "Voltage": @@ -1354,7 +1359,7 @@ class FullSpectralAnalysis(Operation): Vver=xFrec[Vpos] print('vzon y vmer', Vzon, Vmer) return Vzon, Vmer, Vver, GaussCenter - + class SpectralMoments(Operation): ''' @@ -1401,7 +1406,7 @@ class SpectralMoments(Operation): dataOut.data_DOP = data_param[:,1] dataOut.data_MEAN = data_param[:,2] dataOut.data_STD = data_param[:,3] - return + return dataOut def __calculateMoments(self, oldspec, oldfreq, n0, nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None): diff --git a/schainpy/model/proc/jroproc_spectra.py b/schainpy/model/proc/jroproc_spectra.py index 5a7246c..4a2435c 100644 --- a/schainpy/model/proc/jroproc_spectra.py +++ b/schainpy/model/proc/jroproc_spectra.py @@ -10,16 +10,10 @@ from schainpy.utils import log @MPDecorator class SpectraProc(ProcessingUnit): - METHODS = {'selectHeights' : ['minHei', 'maxHei'], - 'selectChannels' : 'channelList', - 'selectChannelsByIndex': 'channelIndexList', - 'getBeaconSignal' : ['tauindex', 'channelindex', 'hei_ref'], - 'selectHeightsByIndex' : ['minIndex', 'maxIndex'] - } - def __init__(self):#, **kwargs): + def __init__(self): - ProcessingUnit.__init__(self)#, **kwargs) + ProcessingUnit.__init__(self) self.buffer = None self.firstdatatime = None @@ -129,10 +123,6 @@ class SpectraProc(ProcessingUnit): if self.dataIn.type == "Spectra": self.dataOut.copy(self.dataIn) - # if not pairsList: - # pairsList = itertools.combinations(self.dataOut.channelList, 2) - # if self.dataOut.data_cspc is not None: - # self.__selectPairs(pairsList) if shift_fft: #desplaza a la derecha en el eje 2 determinadas posiciones shift = int(self.dataOut.nFFTPoints/2) @@ -802,12 +792,9 @@ class IncohInt(Operation): n = None - def __init__(self):#, **kwargs): + def __init__(self): - Operation.__init__(self)#, **kwargs) - - -# self.isConfig = False + Operation.__init__(self) def setup(self, n=None, timeInterval=None, overlapping=False): """ @@ -838,7 +825,7 @@ class IncohInt(Operation): if n is not None: self.n = int(n) else: - # if (type(timeInterval)!=integer) -> change this line + self.__integrationtime = int(timeInterval) self.n = None self.__byTime = True diff --git a/schainpy/model/proc/jroproc_voltage.py b/schainpy/model/proc/jroproc_voltage.py index af0ca38..3757f25 100644 --- a/schainpy/model/proc/jroproc_voltage.py +++ b/schainpy/model/proc/jroproc_voltage.py @@ -39,7 +39,7 @@ class VoltageProc(ProcessingUnit): self.dataOut.data = self.dataIn.data self.dataOut.utctime = self.dataIn.utctime self.dataOut.channelList = self.dataIn.channelList - # self.dataOut.timeInterval = self.dataIn.timeInterval + #self.dataOut.timeInterval = self.dataIn.timeInterval self.dataOut.heightList = self.dataIn.heightList self.dataOut.nProfiles = self.dataIn.nProfiles @@ -332,9 +332,9 @@ class CohInt(Operation): __dataToPutStride = False n = None - def __init__(self):#, **kwargs): + def __init__(self, **kwargs): - Operation.__init__(self)#, **kwargs) + Operation.__init__(self, **kwargs) # self.isConfig = False @@ -589,14 +589,14 @@ class Decoder(Operation): nCode = None nBaud = None - def __init__(self):#, **kwargs): + def __init__(self, **kwargs): - Operation.__init__(self)#, **kwargs) + Operation.__init__(self, **kwargs) self.times = None self.osamp = None # self.__setValues = False - # self.isConfig = False + self.isConfig = False self.setupReq = False def setup(self, code, osamp, dataOut): @@ -777,9 +777,9 @@ class ProfileConcat(Operation): isConfig = False buffer = None - def __init__(self):#, **kwargs): + def __init__(self, **kwargs): - Operation.__init__(self)#, **kwargs) + Operation.__init__(self, **kwargs) self.profileIndex = 0 def reset(self): @@ -829,9 +829,9 @@ class ProfileSelector(Operation): # Tamanho total de los perfiles nProfiles = None - def __init__(self):#, **kwargs): + def __init__(self, **kwargs): - Operation.__init__(self)#, **kwargs) + Operation.__init__(self, **kwargs) self.profileIndex = 0 def incProfileIndex(self): @@ -986,9 +986,9 @@ class ProfileSelector(Operation): class Reshaper(Operation): - def __init__(self):#, **kwargs): + def __init__(self, **kwargs): - Operation.__init__(self)#, **kwargs) + Operation.__init__(self, **kwargs) self.__buffer = None self.__nitems = 0 @@ -1091,9 +1091,9 @@ class Reshaper(Operation): class SplitProfiles(Operation): - def __init__(self):#, **kwargs): + def __init__(self, **kwargs): - Operation.__init__(self)#, **kwargs) + Operation.__init__(self, **kwargs) def run(self, dataOut, n): @@ -1132,9 +1132,9 @@ class SplitProfiles(Operation): return dataOut class CombineProfiles(Operation): - def __init__(self):#, **kwargs): + def __init__(self, **kwargs): - Operation.__init__(self)#, **kwargs) + Operation.__init__(self, **kwargs) self.__remData = None self.__profileIndex = 0