schain Commit - r1751:2d2f96339643 · Jicamarca Repository

test 2

joabAM -

r1751:2d2f96339643

parent child

Context file:

r1751:2d2f96339643

Collapse all files

schainpy/model/data/jrodata.py +6 -2

             # Copyright (c) 2012-2020 Jicamarca Radio Observatory
             # All rights reserved.
             #
             # Distributed under the terms of the BSD 3-clause license.
             """Definition of diferent Data objects for different types of data
             Here you will find the diferent data objects for the different types
             of data, this data objects must be used as dataIn or dataOut objects in
             processing units and operations. Currently the supported data objects are:
             Voltage, Spectra, SpectraHeis, Fits, Correlation and Parameters
             """
             import copy
             import numpy
             import datetime
             import json
             import schainpy.admin
             from schainpy.utils import log
             from .jroheaderIO import SystemHeader, RadarControllerHeader,ProcessingHeader
             from schainpy.model.data import _noise
             SPEED_OF_LIGHT = 3e8
             def getNumpyDtype(dataTypeCode):
                 if dataTypeCode == 0:
                     numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
                 elif dataTypeCode == 1:
                     numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
                 elif dataTypeCode == 2:
                     numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
                 elif dataTypeCode == 3:
                     numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
                 elif dataTypeCode == 4:
                     numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                 elif dataTypeCode == 5:
                     numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
                 else:
                     raise ValueError('dataTypeCode was not defined')
                 return numpyDtype
             def getDataTypeCode(numpyDtype):
                 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
                     datatype = 0
                 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
                     datatype = 1
                 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
                     datatype = 2
                 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
                     datatype = 3
                 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
                     datatype = 4
                 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
                     datatype = 5
                 else:
                     datatype = None
                 return datatype
             def hildebrand_sekhon(data, navg):
                 """
                 This method is for the objective determination of the noise level in Doppler spectra. This
                 implementation technique is based on the fact that the standard deviation of the spectral
                 densities is equal to the mean spectral density for white Gaussian noise
                 Inputs:
                     Data    :    heights
                     navg    :    numbers of averages
                 Return:
                     mean    :    noise's level
                 """
                 sortdata = numpy.sort(data, axis=None)
                 '''
                 lenOfData = len(sortdata)
                 nums_min = lenOfData*0.2
                 if nums_min <= 5:
                     nums_min = 5
                 sump = 0.
                 sumq = 0.
                 j = 0
                 cont = 1
                 while((cont == 1)and(j < lenOfData)):
                     sump += sortdata[j]
                     sumq += sortdata[j]**2
                     if j > nums_min:
                         rtest = float(j)/(j-1) + 1.0/navg
                         if ((sumq*j) > (rtest*sump**2)):
                             j = j - 1
                             sump = sump - sortdata[j]
                             sumq = sumq - sortdata[j]**2
                             cont = 0
                     j += 1
                 lnoise = sump / j
                 '''
                 return _noise.hildebrand_sekhon(sortdata, navg)
             class Beam:
                 def __init__(self):
                     self.codeList = []
                     self.azimuthList = []
                     self.zenithList = []
             class GenericData(object):
                 flagNoData = True
                 def copy(self, inputObj=None):
                     if inputObj == None:
                         return copy.deepcopy(self)
                     for key in list(inputObj.__dict__.keys()):
                         attribute = inputObj.__dict__[key]
                         # If this attribute is a tuple or list
                         if type(inputObj.__dict__[key]) in (tuple, list):
                             self.__dict__[key] = attribute[:]
                             continue
                         # If this attribute is another object or instance
                         if hasattr(attribute, '__dict__'):
                             self.__dict__[key] = attribute.copy()
                             continue
                         self.__dict__[key] = inputObj.__dict__[key]
                 def deepcopy(self):
                     return copy.deepcopy(self)
                 def isEmpty(self):
                     return self.flagNoData
                 def isReady(self):
                     return not self.flagNoData
             class JROData(GenericData):
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
                 type = None
                 datatype = None  # dtype but in string
                 nProfiles = None
                 heightList = None
                 channelList = None
                 flagDiscontinuousBlock = False
                 useLocalTime = False
                 utctime = None
                 timeZone = None
                 dstFlag = None
                 errorCount = None
                 blocksize = None
                 flagDecodeData = False  # asumo q la data no esta decodificada
                 flagDeflipData = False  # asumo q la data no esta sin flip
                 flagShiftFFT = False
                 nCohInt = None
                 windowOfFilter = 1
                 C = 3e8
                 frequency = 49.92e6
                 realtime = False
                 beacon_heiIndexList = None
                 last_block = None
                 blocknow = None
                 azimuth = None
                 zenith = None
                 beam = Beam()
                 profileIndex = None
                 error = None
                 data = None
                 nmodes = None
                 metadata_list = ['heightList', 'timeZone', 'type']
                 ippFactor = 1 #Added to correct the freq and vel range for AMISR data
                 useInputBuffer = False
                 buffer_empty = True
                 codeList = []
                 azimuthList = []
                 elevationList = []
                 last_noise = None
                 __ipp = None
                 __ippSeconds = None
                 sampled_heightsFFT = None
                 pulseLength_TxA = None
                 deltaHeight = None
                 __code = None
                 __nCode = None
                 __nBaud = None
                 unitsDescription = "The units of the parameters are according to the International System of units (Seconds, Meter, Hertz, ...), except \
             the parameters related to distances such as heightList, or heightResolution wich are in Km"
                 def __str__(self):
                     return '{} - {}'.format(self.type, self.datatime())
                 def getNoise(self):
                     raise NotImplementedError
                 @property
                 def nChannels(self):
                     return len(self.channelList)
                 @property
                 def channelIndexList(self):
                     return list(range(self.nChannels))
                 @property
                 def nHeights(self):
                     return len(self.heightList)
                 def getDeltaH(self):
                     return self.heightList[1] - self.heightList[0]
                 @property
                 def ltctime(self):
+                    try:
+                        self.timeZone = self.timeZone.decode("utf-8")
+                    except Exception as e:
+                        pass
                     if self.useLocalTime:
                         if self.timeZone =='lt':
                             return self.utctime - 300 * 60
                         elif self.timeZone =='ut':
                             return self.utctime
                         else:
-                            log.error("No valid timeZone detected")
+                            log.error("No valid timeZone detected:{}".format(self.timeZone))
                     return self.utctime
                 @property
                 def datatime(self):
                     datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
                     return datatimeValue
                 def getTimeRange(self):
                     datatime = []
                     datatime.append(self.ltctime)
                     datatime.append(self.ltctime + self.timeInterval + 1)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getFmaxTimeResponse(self):
                     period = (10**-6) * self.getDeltaH() / (0.15)
                     PRF = 1. / (period * self.nCohInt)
                     fmax = PRF
                     return fmax
                 def getFmax(self):
                     PRF = 1. / (self.__ippSeconds * self.nCohInt)
                     fmax = PRF
                     return fmax
                 def getVmax(self):
                     _lambda = self.C / self.frequency
                     vmax = self.getFmax() * _lambda / 2
                     return vmax
                 ## Radar Controller Header must be immutable
                 @property
                 def ippSeconds(self):
                     '''
                     '''
                     #return self.radarControllerHeaderObj.ippSeconds
                     return self.__ippSeconds
                 @ippSeconds.setter
                 def ippSeconds(self, ippSeconds):
                     '''
                     '''
                     #self.radarControllerHeaderObj.ippSeconds = ippSeconds
                     self.__ippSeconds = ippSeconds
                     self.__ipp = ippSeconds*SPEED_OF_LIGHT/2000.0
                 @property
                 def code(self):
                     '''
                     '''
                     # return self.radarControllerHeaderObj.code
                     return self.__code
                 @code.setter
                 def code(self, code):
                     '''
                     '''
                     # self.radarControllerHeaderObj.code = code
                     self.__code = code
                 @property
                 def nCode(self):
                     '''
                     '''
                     # return self.radarControllerHeaderObj.nCode
                     return self.__nCode
                 @nCode.setter
                 def nCode(self, ncode):
                     '''
                     '''
                     # self.radarControllerHeaderObj.nCode = ncode
                     self.__nCode = ncode
                 @property
                 def nBaud(self):
                     '''
                     '''
                     # return self.radarControllerHeaderObj.nBaud
                     return self.__nBaud
                 @nBaud.setter
                 def nBaud(self, nbaud):
                     '''
                     '''
                     # self.radarControllerHeaderObj.nBaud = nbaud
                     self.__nBaud = nbaud
                 @property
                 def ipp(self):
                     '''
                     '''
                     # return self.radarControllerHeaderObj.ipp
                     return self.__ipp
                 @ipp.setter
                 def ipp(self, ipp):
                     '''
                     '''
                     # self.radarControllerHeaderObj.ipp = ipp
                     self.__ipp = ipp
                 @property
                 def metadata(self):
                     '''
                     '''
                     return {attr: getattr(self, attr) for attr in self.metadata_list}
             class Voltage(JROData):
                 dataPP_POW   = None
                 dataPP_DOP   = None
                 dataPP_WIDTH = None
                 dataPP_SNR   = None
                 # To use oper
                 flagProfilesByRange = False
                 nProfilesByRange = None
                 max_nIncohInt = 1
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.useLocalTime = True
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.type = "Voltage"
                     self.data = None
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.timeZone = 0
                     self.dstFlag = None
                     self.errorCount = None
                     self.nCohInt = None
                     self.blocksize = None
                     self.flagCohInt = False
                     self.flagDecodeData = False  # asumo q la data no esta decodificada
                     self.flagDeflipData = False  # asumo q la data no esta sin flip
                     self.flagShiftFFT = False
                     self.flagDataAsBlock = False  # Asumo que la data es leida perfil a perfil
                     self.profileIndex = 0
                     self.ippFactor=1
                     self.metadata_list = ['type', 'heightList', 'timeZone', 'nProfiles', 'channelList', 'nCohInt',
                         'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp']
                 def getNoisebyHildebrand(self, channel=None, ymin_index=None, ymax_index=None):
                     """
                     Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                     Return:
                         noiselevel
                     """
                     if channel != None:
                         data = self.data[channel,ymin_index:ymax_index]
                         nChannels = 1
                     else:
                         data = self.data[:,ymin_index:ymax_index]
                         nChannels = self.nChannels
                     noise = numpy.zeros(nChannels)
                     power = data * numpy.conjugate(data)
                     for thisChannel in range(nChannels):
                         if nChannels == 1:
                             daux = power[:].real
                         else:
                             daux = power[thisChannel, :].real
                         noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
                     return noise
                 def getNoise(self, type=1, channel=None,ymin_index=None, ymax_index=None):
                     if type == 1:
                         noise = self.getNoisebyHildebrand(channel,ymin_index, ymax_index)
                     return noise
                 def getPower(self, channel=None):
                     if channel != None:
                         data = self.data[channel]
                     else:
                         data = self.data
                     power = data * numpy.conjugate(data)
                     powerdB = 10 * numpy.log10(power.real)
                     powerdB = numpy.squeeze(powerdB)
                     return powerdB
                 @property
                 def data_pow(self):
                     return self.getPower()
                 @property
                 def timeInterval(self):
                     return self.ippSeconds * self.nCohInt
                 noise = property(getNoise, "I'm the 'nHeights' property.")
             class Spectra(JROData):
                 data_outlier = None
                 flagProfilesByRange = False
                 nProfilesByRange = None
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.data_dc = None
                     self.data_spc = None
                     self.data_cspc = None
                     self.useLocalTime = True
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.type = "Spectra"
                     self.timeZone = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
                     self.pairsList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.nCohInt = None
                     self.nIncohInt = None
                     self.blocksize = None
                     self.nFFTPoints = None
                     self.wavelength = None
                     self.flagDecodeData = False  # asumo q la data no esta decodificada
                     self.flagDeflipData = False  # asumo q la data no esta sin flip
                     self.flagShiftFFT = False
                     self.ippFactor = 1
                     self.beacon_heiIndexList = []
                     self.noise_estimation = None
                     self.codeList = []
                     self.azimuthList = []
                     self.elevationList = []
                     self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt',
                         'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp','nIncohInt', 'nFFTPoints', 'nProfiles']
                 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                     """
                     Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                     Return:
                         noiselevel
                     """
                     noise = numpy.zeros(self.nChannels)
                     for channel in range(self.nChannels):
                         daux = self.data_spc[channel,
                                              xmin_index:xmax_index, ymin_index:ymax_index]
                         # noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
                         noise[channel] = hildebrand_sekhon(daux, self.max_nIncohInt[channel])
                     return noise
                 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                     if self.noise_estimation is not None:
                         # this was estimated by getNoise Operation defined in jroproc_spectra.py
                         return self.noise_estimation
                     else:
                         noise = self.getNoisebyHildebrand(
                             xmin_index, xmax_index, ymin_index, ymax_index)
                         return noise
                 def getFreqRangeTimeResponse(self, extrapoints=0):
                     deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
                     freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
                     return freqrange
                 def getAcfRange(self, extrapoints=0):
                     deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
                     freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
                     return freqrange
                 def getFreqRange(self, extrapoints=0):
                     deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
                     freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
                     return freqrange
                 def getVelRange(self, extrapoints=0):
                     deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
                     velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
                     if self.nmodes:
                         return velrange/self.nmodes
                     else:
                         return velrange
                 @property
                 def nPairs(self):
                     return len(self.pairsList)
                 @property
                 def pairsIndexList(self):
                     return list(range(self.nPairs))
                 @property
                 def normFactor(self):
                     pwcode = 1
                     if self.flagDecodeData:
                         try:
                             pwcode = numpy.sum(self.code[0]**2)
                         except Exception as e:
                             log.warning("Failed pwcode read, setting to 1")
                             pwcode = 1
                     #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
                     normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
                     if self.flagProfilesByRange:
                         normFactor *= (self.nProfilesByRange/self.nProfilesByRange.max())
                     return normFactor
                 @property
                 def flag_cspc(self):
                     if self.data_cspc is None:
                         return True
                     return False
                 @property
                 def flag_dc(self):
                     if self.data_dc is None:
                         return True
                     return False
                 @property
                 def timeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
                     if self.nmodes:
                         return self.nmodes*timeInterval
                     else:
                         return timeInterval
                 def getPower(self):
                     factor = self.normFactor
                     power = numpy.zeros( (self.nChannels,self.nHeights) )
                     for ch in range(self.nChannels):
                         z = None
                         if hasattr(factor,'shape'):
                             if factor.ndim > 1:
                                 z = self.data_spc[ch]/factor[ch]
                             else:
                                 z = self.data_spc[ch]/factor
                         else:
                             z = self.data_spc[ch]/factor
                         z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                         avg = numpy.average(z, axis=0)
                         power[ch] = 10 * numpy.log10(avg)
                     return power
                 @property
                 def max_nIncohInt(self):
                     ints = numpy.zeros(self.nChannels)
                     for ch in range(self.nChannels):
                         if hasattr(self.nIncohInt,'shape'):
                             if self.nIncohInt.ndim > 1:
                                 ints[ch,] = self.nIncohInt[ch].max()
                             else:
                                 ints[ch,] = self.nIncohInt
                                 self.nIncohInt = int(self.nIncohInt)
                         else:
                             ints[ch,] = self.nIncohInt
                     return ints
                 def getCoherence(self, pairsList=None, phase=False):
                     z = []
                     if pairsList is None:
                         pairsIndexList = self.pairsIndexList
                     else:
                         pairsIndexList = []
                         for pair in pairsList:
                             if pair not in self.pairsList:
                                 raise ValueError("Pair %s is not in dataOut.pairsList" % (
                                     pair))
                             pairsIndexList.append(self.pairsList.index(pair))
                     for i in range(len(pairsIndexList)):
                         pair = self.pairsList[pairsIndexList[i]]
                         ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
                         powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
                         powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
                         avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
                         if phase:
                             data = numpy.arctan2(avgcoherenceComplex.imag,
                                                  avgcoherenceComplex.real) * 180 / numpy.pi
                         else:
                             data = numpy.abs(avgcoherenceComplex)
                         z.append(data)
                     return numpy.array(z)
                 def setValue(self, value):
                     print("This property should not be initialized", value)
                     return
                 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
             class SpectraHeis(Spectra):
                 def __init__(self):
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "SpectraHeis"
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.blocksize = None
                     self.profileIndex = 0
                     self.nCohInt = 1
                     self.nIncohInt = 1
                 @property
                 def normFactor(self):
                     pwcode = 1
                     if self.flagDecodeData:
                         pwcode = numpy.sum(self.code[0]**2)
                     normFactor = self.nIncohInt * self.nCohInt * pwcode
                     return normFactor
                 @property
                 def timeInterval(self):
                     return self.ippSeconds * self.nCohInt * self.nIncohInt
             class Fits(JROData):
                 def __init__(self):
                     self.type = "Fits"
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
                     self.flagNoData = True
                     self.utctime = None
                     self.nCohInt = 1
                     self.nIncohInt = 1
                     self.useLocalTime = True
                     self.profileIndex = 0
                     self.timeZone = 0
                 def getTimeRange(self):
                     datatime = []
                     datatime.append(self.ltctime)
                     datatime.append(self.ltctime + self.timeInterval)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getChannelIndexList(self):
                     return list(range(self.nChannels))
                 def getNoise(self, type=1):
                     if type == 1:
                         noise = self.getNoisebyHildebrand()
                     if type == 2:
                         noise = self.getNoisebySort()
                     if type == 3:
                         noise = self.getNoisebyWindow()
                     return noise
                 @property
                 def timeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     return timeInterval
                 @property
                 def ippSeconds(self):
                     '''
                     '''
                     return self.ipp_sec
                 noise = property(getNoise, "I'm the 'nHeights' property.")
             class Correlation(JROData):
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Correlation"
                     self.data = None
                     self.dtype = None
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.timeZone = 0
                     self.dstFlag = None
                     self.errorCount = None
                     self.blocksize = None
                     self.flagDecodeData = False  # asumo q la data no esta decodificada
                     self.flagDeflipData = False  # asumo q la data no esta sin flip
                     self.pairsList = None
                     self.nPoints = None
                 def getPairsList(self):
                     return self.pairsList
                 def getNoise(self, mode=2):
                     indR = numpy.where(self.lagR == 0)[0][0]
                     indT = numpy.where(self.lagT == 0)[0][0]
                     jspectra0 = self.data_corr[:, :, indR, :]
                     jspectra = copy.copy(jspectra0)
                     num_chan = jspectra.shape[0]
                     num_hei = jspectra.shape[2]
                     freq_dc = jspectra.shape[1] / 2
                     ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
                     if ind_vel[0] < 0:
                         ind_vel[list(range(0, 1))] = ind_vel[list(
                             range(0, 1))] + self.num_prof
                     if mode == 1:
                         jspectra[:, freq_dc, :] = (
                             jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2  # CORRECCION
                     if mode == 2:
                         vel = numpy.array([-2, -1, 1, 2])
                         xx = numpy.zeros([4, 4])
                         for fil in range(4):
                             xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
                         xx_inv = numpy.linalg.inv(xx)
                         xx_aux = xx_inv[0, :]
                         for ich in range(num_chan):
                             yy = jspectra[ich, ind_vel, :]
                             jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
                             junkid = jspectra[ich, freq_dc, :] <= 0
                             cjunkid = sum(junkid)
                             if cjunkid.any():
                                 jspectra[ich, freq_dc, junkid.nonzero()] = (
                                     jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
                     noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
                     return noise
                 @property
                 def timeInterval(self):
                     return self.ippSeconds * self.nCohInt * self.nProfiles
                 def splitFunctions(self):
                     pairsList = self.pairsList
                     ccf_pairs = []
                     acf_pairs = []
                     ccf_ind = []
                     acf_ind = []
                     for l in range(len(pairsList)):
                         chan0 = pairsList[l][0]
                         chan1 = pairsList[l][1]
                         # Obteniendo pares de Autocorrelacion
                         if chan0 == chan1:
                             acf_pairs.append(chan0)
                             acf_ind.append(l)
                         else:
                             ccf_pairs.append(pairsList[l])
                             ccf_ind.append(l)
                     data_acf = self.data_cf[acf_ind]
                     data_ccf = self.data_cf[ccf_ind]
                     return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
                 @property
                 def normFactor(self):
                     acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
                     acf_pairs = numpy.array(acf_pairs)
                     normFactor = numpy.zeros((self.nPairs, self.nHeights))
                     for p in range(self.nPairs):
                         pair = self.pairsList[p]
                         ch0 = pair[0]
                         ch1 = pair[1]
                         ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
                         ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
                         normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
                     return normFactor
             class Parameters(Spectra):
                 groupList = None  # List of Pairs, Groups, etc
                 data_param = None  # Parameters obtained
                 data_pre = None  # Data Pre Parametrization
                 data_SNR = None  # Signal to Noise Ratio
                 abscissaList = None  # Abscissa, can be velocities, lags or time
                 utctimeInit = None  # Initial UTC time
                 paramInterval = None  # Time interval to calculate Parameters in seconds
                 useLocalTime = True
                 # Fitting
                 data_error = None  # Error of the estimation
                 constants = None
                 library = None
                 # Output signal
                 outputInterval = None  # Time interval to calculate output signal in seconds
                 data_output = None  # Out signal
                 nAvg = None
                 noise_estimation = None
                 GauSPC = None  # Fit gaussian SPC
                 data_outlier = None
                 data_vdrift = None
                 radarControllerHeaderTxt=None #header Controller like text
                 txPower = None
                 flagProfilesByRange = False
                 nProfilesByRange = None
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.type = "Parameters"
                     self.timeZone = 0
                 def getTimeRange1(self, interval):
                     datatime = []
                     if self.useLocalTime:
                         time1 = self.utctimeInit - self.timeZone * 60
                     else:
                         time1 = self.utctimeInit
                     datatime.append(time1)
                     datatime.append(time1 + interval)
                     datatime = numpy.array(datatime)
                     return datatime
                 @property
                 def timeInterval(self):
                     if hasattr(self, 'timeInterval1'):
                         return self.timeInterval1
                     else:
                         return self.paramInterval
                 def setValue(self, value):
                     print("This property should not be initialized")
                     return
                 def getNoise(self):
                     return self.spc_noise
                 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
             class PlotterData(object):
                 '''
                 Object to hold data to be plotted
                 '''
                 MAXNUMX = 200
                 MAXNUMY = 200
                 def __init__(self, code, exp_code, localtime=True):
                     self.key = code
                     self.exp_code = exp_code
                     self.ready = False
                     self.flagNoData = False
                     self.localtime = localtime
                     self.data = {}
                     self.meta = {}
                     self.__heights = []
                 def __str__(self):
                     dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
                     return 'Data[{}][{}]'.format(';'.join(dum), len(self.times))
                 def __len__(self):
                     return len(self.data)
                 def __getitem__(self, key):
                     if isinstance(key, int):
                         return self.data[self.times[key]]
                     elif isinstance(key, str):
                         ret = numpy.array([self.data[x][key] for x in self.times])
                         if ret.ndim > 1:
                             ret = numpy.swapaxes(ret, 0, 1)
                         return ret
                 def __contains__(self, key):
                     return key in self.data[self.min_time]
                 def setup(self):
                     '''
                     Configure object
                     '''
                     self.type = ''
                     self.ready = False
                     del self.data
                     self.data = {}
                     self.__heights = []
                     self.__all_heights = set()
                 def shape(self, key):
                     '''
                     Get the shape of the one-element data for the given key
                     '''
                     if len(self.data[self.min_time][key]):
                         return self.data[self.min_time][key].shape
                     return (0,)
                 def update(self, data, tm, meta={}):
                     '''
                     Update data object with new dataOut
                     '''
                     self.data[tm] = data
                     for key, value in meta.items():
                         setattr(self, key, value)
                 def normalize_heights(self):
                     '''
                     Ensure same-dimension of the data for different heighList
                     '''
                     H = numpy.array(list(self.__all_heights))
                     H.sort()
                     for key in self.data:
                         shape = self.shape(key)[:-1] + H.shape
                         for tm, obj in list(self.data[key].items()):
                             h = self.__heights[self.times.tolist().index(tm)]
                             if H.size == h.size:
                                 continue
                             index = numpy.where(numpy.in1d(H, h))[0]
                             dummy = numpy.zeros(shape) + numpy.nan
                             if len(shape) == 2:
                                 dummy[:, index] = obj
                             else:
                                 dummy[index] = obj
                             self.data[key][tm] = dummy
                     self.__heights = [H for tm in self.times]
                 def jsonify(self, tm, plot_name, plot_type, decimate=False):
                     '''
                     Convert data to json
                     '''
                     meta = {}
                     meta['xrange'] = []
                     dy = int(len(self.yrange)/self.MAXNUMY) + 1
                     tmp = self.data[tm][self.key]
                     shape = tmp.shape
                     if len(shape) == 2:
                         data = self.roundFloats(self.data[tm][self.key][::, ::dy].tolist())
                     elif len(shape) == 3:
                         dx = int(self.data[tm][self.key].shape[1]/self.MAXNUMX) + 1
                         data = self.roundFloats(
                             self.data[tm][self.key][::, ::dx, ::dy].tolist())
                         meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
                     else:
                         data = self.roundFloats(self.data[tm][self.key].tolist())
                     ret = {
                         'plot': plot_name,
                         'code': self.exp_code,
                         'time': float(tm),
                         'data': data,
                         }
                     meta['type'] = plot_type
                     meta['interval'] = float(self.interval)
                     meta['localtime'] = self.localtime
                     meta['yrange'] = self.roundFloats(self.yrange[::dy].tolist())
                     meta.update(self.meta)
                     ret['metadata'] = meta
                     return json.dumps(ret)
                 @property
                 def times(self):
                     '''
                     Return the list of times of the current data
                     '''
                     ret = [t for t in self.data]
                     ret.sort()
                     return numpy.array(ret)
                 @property
                 def min_time(self):
                     '''
                     Return the minimun time value
                     '''
                     return self.times[0]
                 @property
                 def max_time(self):
                     '''
                     Return the maximun time value
                     '''
                     return self.times[-1]
                 # @property
                 # def heights(self):
                 #     '''
                 #     Return the list of heights of the current data
                 #     '''
                 #     return numpy.array(self.__heights[-1])
                 @staticmethod
                 def roundFloats(obj):
                     if isinstance(obj, list):
                         return list(map(PlotterData.roundFloats, obj))
                     elif isinstance(obj, float):
                         return round(obj, 2)

schainpy/model/graphics/jroplot_parameters.py +1 -1

             import os
             import datetime
             import numpy
             from schainpy.model.graphics.jroplot_base import Plot, plt
             from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot, SpectraCutPlot
             from schainpy.utils import log
             EARTH_RADIUS = 6.3710e3
             def ll2xy(lat1, lon1, lat2, lon2):
                 p = 0.017453292519943295
                 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
                     numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
                 r = 12742 * numpy.arcsin(numpy.sqrt(a))
                 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
                                       * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
                 theta = -theta + numpy.pi/2
                 return r*numpy.cos(theta), r*numpy.sin(theta)
             def km2deg(km):
                 '''
                 Convert distance in km to degrees
                 '''
                 return numpy.rad2deg(km/EARTH_RADIUS)
             class SpectralMomentsPlot(SpectraPlot):
                 '''
                 Plot for Spectral Moments
                 '''
                 CODE = 'spc_moments'
                 # colormap = 'jet'
                 # plot_type = 'pcolor'
             class DobleGaussianPlot(SpectraPlot):
                 '''
                 Plot for Double Gaussian Plot
                 '''
                 CODE = 'gaussian_fit'
                 # colormap = 'jet'
                 # plot_type = 'pcolor'
             class DoubleGaussianSpectraCutPlot(SpectraCutPlot):
                 '''
                 Plot SpectraCut with Double Gaussian Fit
                 '''
                 CODE = 'cut_gaussian_fit'
             class SpectralFitObliquePlot(SpectraPlot):
                 '''
                 Plot for Spectral Oblique
                 '''
                 CODE = 'spc_moments'
                 colormap = 'jet'
                 plot_type = 'pcolor'
             class SnrPlot(RTIPlot):
                 '''
                 Plot for SNR Data
                 '''
                 CODE = 'snr'
                 colormap = 'jet'
                 def update(self, dataOut):
                     if len(self.channelList) == 0:
                         self.update_list(dataOut)
                     meta = {}
                     data = {
                         'snr': 10 * numpy.log10(dataOut.data_snr)
                     }
                     return data, meta
             class DopplerPlot(RTIPlot):
                 '''
                 Plot for DOPPLER Data (1st moment)
                 '''
                 CODE = 'dop'
                 colormap = 'RdBu_r'
                 def update(self, dataOut):
                     self.update_list(dataOut)
                     data = {
                         'dop': dataOut.data_dop
                     }
                     return data, {}
             class PowerPlot(RTIPlot):
                 '''
                 Plot for Power Data (0 moment)
                 '''
                 CODE = 'pow'
                 colormap = 'jet'
                 def update(self, dataOut):
                     self.update_list(dataOut)
                     data = {
-                        'pow': 10*numpy.log10(dataOut.data_pow/dataOut.normFactor)
+                        'pow': 10*numpy.log10(dataOut.data_pow)
                     }
                     try:
                         data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
                     except:
                         pass
                     return data, {}
             class SpectralWidthPlot(RTIPlot):
                 '''
                 Plot for Spectral Width Data (2nd moment)
                 '''
                 CODE = 'width'
                 colormap = 'jet'
                 def update(self, dataOut):
                     self.update_list(dataOut)
                     data = {
                         'width': dataOut.data_width
                     }
                     data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
                     return data, {}
             class SkyMapPlot(Plot):
                 '''
                 Plot for meteors detection data
                 '''
                 CODE = 'param'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = 1
                     self.width = 7.2
                     self.height = 7.2
                     self.nplots = 1
                     self.xlabel = 'Zonal Zenith Angle (deg)'
                     self.ylabel = 'Meridional Zenith Angle (deg)'
                     self.polar = True
                     self.ymin = -180
                     self.ymax = 180
                     self.colorbar = False
                 def plot(self):
                     arrayParameters = numpy.concatenate(self.data['param'])
                     error = arrayParameters[:, -1]
                     indValid = numpy.where(error == 0)[0]
                     finalMeteor = arrayParameters[indValid, :]
                     finalAzimuth = finalMeteor[:, 3]
                     finalZenith = finalMeteor[:, 4]
                     x = finalAzimuth * numpy.pi / 180
                     y = finalZenith
                     ax = self.axes[0]
                     if ax.firsttime:
                         ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
                     else:
                         ax.plot.set_data(x, y)
                     dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
                     dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
                     title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
                                                                                                  dt2,
                                                                                                  len(x))
                     self.titles[0] = title
             class GenericRTIPlot(Plot):
                 '''
                 Plot for data_xxxx object
                 '''
                 CODE = 'param'
                 colormap = 'viridis'
                 plot_type = 'pcolorbuffer'
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     self.nrows = self.data.shape('param')[0]
                     self.nplots = self.nrows
                     self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
                     if not self.xlabel:
                         self.xlabel = 'Time'
                     self.ylabel = 'Range [km]'
                     if not self.titles:
                         self.titles = ['Param {}'.format(x) for x in range(self.nrows)]
                 def update(self, dataOut):
                     data = {
                         'param' : numpy.concatenate([getattr(dataOut, attr) for attr in self.attr_data], axis=0)
                     }
                     meta = {}
                     return data, meta
                 def plot(self):
                     # self.data.normalize_heights()
                     self.x = self.data.times
                     self.y = self.data.yrange
                     self.z = self.data['param']
                     self.z = numpy.ma.masked_invalid(self.z)
                     if self.decimation is None:
                         x, y, z = self.fill_gaps(self.x, self.y, self.z)
                     else:
                         x, y, z = self.fill_gaps(*self.decimate())
                     for n, ax in enumerate(self.axes):
                         self.zmax = self.zmax if self.zmax is not None else numpy.max(
                             self.z[n])
                         self.zmin = self.zmin if self.zmin is not None else numpy.min(
                             self.z[n])
                         if ax.firsttime:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=self.cmaps[n]
                                                    )
                         else:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             try:
                                ax.collections.remove(ax.collections[0])
                             except:
                                pass
                             ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=self.cmaps[n]
                                                    )
             class PolarMapPlot(Plot):
                 '''
                 Plot for weather radar
                 '''
                 CODE = 'param'
                 colormap = 'seismic'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = 1
                     self.width = 9
                     self.height = 8
                     self.mode = self.data.meta['mode']
                     if self.channels is not None:
                         self.nplots = len(self.channels)
                         self.nrows = len(self.channels)
                     else:
                         self.nplots = self.data.shape(self.CODE)[0]
                         self.nrows = self.nplots
                         self.channels = list(range(self.nplots))
                     if self.mode == 'E':
                         self.xlabel = 'Longitude'
                         self.ylabel = 'Latitude'
                     else:
                         self.xlabel = 'Range (km)'
                         self.ylabel = 'Height (km)'
                     self.bgcolor = 'white'
                     self.cb_labels = self.data.meta['units']
                     self.lat = self.data.meta['latitude']
                     self.lon = self.data.meta['longitude']
                     self.xmin, self.xmax = float(
                         km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
                     self.ymin, self.ymax = float(
                         km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
                     #  self.polar = True
                 def plot(self):
                     for n, ax in enumerate(self.axes):
                         data = self.data['param'][self.channels[n]]
                         zeniths = numpy.linspace(
 , self.data.meta['max_range'], data.shape[1])
                         if self.mode == 'E':
                             azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
                             r, theta = numpy.meshgrid(zeniths, azimuths)
                             x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
                                 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
                             x = km2deg(x) + self.lon
                             y = km2deg(y) + self.lat
                         else:
                             azimuths = numpy.radians(self.data.yrange)
                             r, theta = numpy.meshgrid(zeniths, azimuths)
                             x, y = r*numpy.cos(theta), r*numpy.sin(theta)
                         self.y = zeniths
                         if ax.firsttime:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.plt = ax.pcolormesh(# r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
                                 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
                                 vmin=self.zmin,
                                 vmax=self.zmax,
                                 cmap=self.cmaps[n])
                         else:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.collections.remove(ax.collections[0])
                             ax.plt = ax.pcolormesh(# r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
                                 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
                                 vmin=self.zmin,
                                 vmax=self.zmax,
                                 cmap=self.cmaps[n])
                         if self.mode == 'A':
                             continue
                         # plot district names
                         f = open('/data/workspace/schain_scripts/distrito.csv')
                         for line in f:
                             label, lon, lat = [s.strip() for s in line.split(',') if s]
                             lat = float(lat)
                             lon = float(lon)
                             # ax.plot(lon, lat, '.b', ms=2)
                             ax.text(lon, lat, label.decode('utf8'), ha='center',
                                     va='bottom', size='8', color='black')
                         # plot limites
                         limites = []
                         tmp = []
                         for line in open('/data/workspace/schain_scripts/lima.csv'):
                             if '#' in line:
                                 if tmp:
                                     limites.append(tmp)
                                 tmp = []
                                 continue
                             values = line.strip().split(',')
                             tmp.append((float(values[0]), float(values[1])))
                         for points in limites:
                             ax.add_patch(
                                 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
                         # plot Cuencas
                         for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
                             f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
                             values = [line.strip().split(',') for line in f]
                             points = [(float(s[0]), float(s[1])) for s in values]
                             ax.add_patch(Polygon(points, ec='b', fc='none'))
                         # plot grid
                         for r in (15, 30, 45, 60):
                             ax.add_artist(plt.Circle((self.lon, self.lat),
                                                      km2deg(r), color='0.6', fill=False, lw=0.2))
                             ax.text(
                                 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
                                 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
                                 '{}km'.format(r),
                                 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
                     if self.mode == 'E':
                         title = 'El={}\N{DEGREE SIGN}'.format(self.data.meta['elevation'])
                         label = 'E{:02d}'.format(int(self.data.meta['elevation']))
                     else:
                         title = 'Az={}\N{DEGREE SIGN}'.format(self.data.meta['azimuth'])
                         label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
                     self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
                     self.titles = ['{} {}'.format(
                         self.data.parameters[x], title) for x in self.channels]
             class TxPowerPlot(Plot):
                 '''
                 Plot for TX Power from external file
                 '''
                 CODE = 'tx_power'
                 plot_type = 'scatterbuffer'
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     self.nrows = 1
                     self.nplots = 1
                     self.ylabel = 'Power [kW]'
                     self.xlabel = 'Time'
                     self.titles = ['TX power']
                     self.colorbar = False
                     self.plots_adjust.update({'right': 0.85 })
                     #if not self.titles:
                     self.titles = ['TX Power Plot']
                 def update(self, dataOut):
                     data = {}
                     meta = {}
                     data['tx_power'] = dataOut.txPower/1000
                     meta['yrange'] = numpy.array([])
                     #print(dataOut.txPower/1000)
                     return data, meta
                 def plot(self):
                     x = self.data.times
                     xmin = self.data.min_time
                     xmax = xmin + self.xrange * 60 * 60
                     Y = self.data['tx_power']
                     if self.axes[0].firsttime:
                         if self.ymin is None: self.ymin = 0
                         if self.ymax is None: self.ymax = numpy.nanmax(Y) + 5
                         if self.ymax == 5:
                             self.ymax = 250
                             self.ymin = 100
                         self.axes[0].plot(x, Y, lw=1, label='Power')
                         plt.legend(bbox_to_anchor=(1.18, 1.0))
                     else:
                         self.axes[0].lines[0].set_data(x, Y)

schainpy/model/graphics/jroplot_spectra.py +8 -9

             # Copyright (c) 2012-2021 Jicamarca Radio Observatory
             # All rights reserved.
             #
             # Distributed under the terms of the BSD 3-clause license.
             """Classes to plot Spectra data
             """
             import os
             import numpy
             import datetime
             from schainpy.model.graphics.jroplot_base import Plot, plt, log
             from itertools import combinations
             from matplotlib.ticker import LinearLocator
             from schainpy.model.utils.BField import BField
             from scipy.interpolate import splrep
             from scipy.interpolate import splev
             from matplotlib import __version__ as plt_version
             if plt_version >='3.3.4':
                 EXTRA_POINTS = 0
             else:
                 EXTRA_POINTS = 1
             class SpectraPlot(Plot):
                 '''
                 Plot for Spectra data
                 '''
                 CODE = 'spc'
                 colormap = 'jet'
                 plot_type = 'pcolor'
                 buffering = False
                 channelList = []
                 elevationList = []
                 azimuthList = []
                 def setup(self):
                     self.nplots = len(self.data.channels)
                     self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
                     self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
                     self.height = 3.4 * self.nrows
                     self.cb_label = 'dB'
                     if self.showprofile:
                         self.width = 5.2 * self.ncols
                     else:
                         self.width = 4.2* self.ncols
                     self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.12})
                     self.ylabel = 'Range [km]'
                 def update_list(self,dataOut):
                     if len(self.channelList) == 0:
                         self.channelList = dataOut.channelList
                     if len(self.elevationList) == 0:
                         self.elevationList = dataOut.elevationList
                     if len(self.azimuthList) == 0:
                         self.azimuthList = dataOut.azimuthList
                 def update(self, dataOut):
                     self.update_list(dataOut)
                     data = {}
                     meta = {}
                     norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt  * dataOut.windowOfFilter
                     if dataOut.type == "Parameters":
                         noise = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
                         spc = 10*numpy.log10(dataOut.data_spc/(dataOut.nProfiles))
                     else:
                         noise = 10*numpy.log10(dataOut.getNoise()/norm)
                         z = numpy.zeros((dataOut.nChannels, dataOut.nFFTPoints, dataOut.nHeights))
                         for ch in range(dataOut.nChannels):
                             if hasattr(dataOut.normFactor,'ndim'):
                                 if dataOut.normFactor.ndim > 1:
                                     z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
                                 else:
                                     z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
                             else:
                                 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
                         z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                         spc = 10*numpy.log10(z)
                     data['spc'] = spc
                     data['rti'] = spc.mean(axis=1)
                     data['noise'] = noise
                     meta['xrange'] = (dataOut.getFreqRange(EXTRA_POINTS)/1000., dataOut.getAcfRange(EXTRA_POINTS), dataOut.getVelRange(EXTRA_POINTS))
                     if self.CODE == 'spc_moments':
                         data['moments'] = dataOut.moments
                     return data, meta
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.data.xrange[0]
                         self.xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.data.xrange[1]
                         self.xlabel = "Time (ms)"
                     else:
                         x = self.data.xrange[2]
                         self.xlabel = "Velocity (m/s)"
                     if (self.CODE == 'spc_moments') | (self.CODE == 'gaussian_fit'):
                         x = self.data.xrange[2]
                         self.xlabel = "Velocity (m/s)"
                     self.titles = []
                     y = self.data.yrange
                     self.y = y
                     data = self.data[-1]
                     z = data['spc']
                     for n, ax in enumerate(self.axes):
                         noise = self.data['noise'][n][0]
                         # noise = data['noise'][n]
                         if self.CODE == 'spc_moments':
                             mean = data['moments'][n, 1]
                         if self.CODE == 'gaussian_fit':
                             gau0 = data['gaussfit'][n][2,:,0]
                             gau1 = data['gaussfit'][n][2,:,1]
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
                             self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                             if self.showprofile:
                                 ax.plt_profile = self.pf_axes[n].plot(
                                     data['rti'][n], y)[0]
                                 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
                                                                     color="k", linestyle="dashed", lw=1)[0]
                             if self.CODE == 'spc_moments':
                                 ax.plt_mean = ax.plot(mean, y, color='k', lw=1)[0]
                             if self.CODE == 'gaussian_fit':
                                 ax.plt_gau0 = ax.plot(gau0, y, color='r', lw=1)[0]
                                 ax.plt_gau1 = ax.plot(gau1, y, color='y', lw=1)[0]
                         else:
                             ax.plt.set_array(z[n].T.ravel())
                             if self.showprofile:
                                 ax.plt_profile.set_data(data['rti'][n], y)
                                 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
                             if self.CODE == 'spc_moments':
                                 ax.plt_mean.set_data(mean, y)
                             if self.CODE == 'gaussian_fit':
                                 ax.plt_gau0.set_data(gau0, y)
                                 ax.plt_gau1.set_data(gau1, y)
                         if len(self.azimuthList) > 0 and len(self.elevationList) > 0:
                             self.titles.append('CH {}: {:2.1f}elv {:2.1f}az {:3.2f}dB'.format(self.channelList[n], noise, self.elevationList[n], self.azimuthList[n]))
                         else:
                             self.titles.append('CH {}:  {:3.2f}dB'.format(self.channelList[n], noise))
             class SpectraObliquePlot(Plot):
                 '''
                 Plot for Spectra data
                 '''
                 CODE = 'spc_oblique'
                 colormap = 'jet'
                 plot_type = 'pcolor'
                 def setup(self):
                     self.xaxis = "oblique"
                     self.nplots = len(self.data.channels)
                     self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
                     self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
                     self.height = 2.6 * self.nrows
                     self.cb_label = 'dB'
                     if self.showprofile:
                         self.width = 4 * self.ncols
                     else:
                         self.width = 3.5 * self.ncols
                     self.plots_adjust.update({'wspace': 0.8, 'hspace':0.2, 'left': 0.2, 'right': 0.9, 'bottom': 0.18})
                     self.ylabel = 'Range [km]'
                 def update(self, dataOut):
                     data = {}
                     meta = {}
                     spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
                     data['spc'] = spc
                     data['rti'] = dataOut.getPower()
                     data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
                     meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
                     data['shift1'] = dataOut.Dop_EEJ_T1[0]
                     data['shift2'] = dataOut.Dop_EEJ_T2[0]
                     data['max_val_2'] = dataOut.Oblique_params[0,-1,:]
                     data['shift1_error'] = dataOut.Err_Dop_EEJ_T1[0]
                     data['shift2_error'] = dataOut.Err_Dop_EEJ_T2[0]
                     return data, meta
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.data.xrange[0]
                         self.xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.data.xrange[1]
                         self.xlabel = "Time (ms)"
                     else:
                         x = self.data.xrange[2]
                         self.xlabel = "Velocity (m/s)"
                     self.titles = []
                     y = self.data.yrange
                     self.y = y
                     data = self.data[-1]
                     z = data['spc']
                     for n, ax in enumerate(self.axes):
                         noise = self.data['noise'][n][-1]
                         shift1 = data['shift1']
                         shift2 = data['shift2']
                         max_val_2 = data['max_val_2']
                         err1 = data['shift1_error']
                         err2 = data['shift2_error']
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
                             self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                             if self.showprofile:
                                 ax.plt_profile = self.pf_axes[n].plot(
                                     self.data['rti'][n][-1], y)[0]
                                 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
                                                                     color="k", linestyle="dashed", lw=1)[0]
                             self.ploterr1 = ax.errorbar(shift1, y, xerr=err1, fmt='k^', elinewidth=2.2, marker='o', linestyle='None',markersize=2.5,capsize=0.3,markeredgewidth=0.2)
                             self.ploterr2 = ax.errorbar(shift2, y, xerr=err2, fmt='m^',elinewidth=2.2,marker='o',linestyle='None',markersize=2.5,capsize=0.3,markeredgewidth=0.2)
                             self.ploterr3 = ax.errorbar(max_val_2, y, xerr=0, fmt='g^',elinewidth=2.2,marker='o',linestyle='None',markersize=2.5,capsize=0.3,markeredgewidth=0.2)
                         else:
                             self.ploterr1.remove()
                             self.ploterr2.remove()
                             self.ploterr3.remove()
                             ax.plt.set_array(z[n].T.ravel())
                             if self.showprofile:
                                 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
                                 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
                             self.ploterr1 = ax.errorbar(shift1, y, xerr=err1, fmt='k^', elinewidth=2.2, marker='o', linestyle='None',markersize=2.5,capsize=0.3,markeredgewidth=0.2)
                             self.ploterr2 = ax.errorbar(shift2, y, xerr=err2, fmt='m^',elinewidth=2.2,marker='o',linestyle='None',markersize=2.5,capsize=0.3,markeredgewidth=0.2)
                             self.ploterr3 = ax.errorbar(max_val_2, y, xerr=0, fmt='g^',elinewidth=2.2,marker='o',linestyle='None',markersize=2.5,capsize=0.3,markeredgewidth=0.2)
                         self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
             class CrossSpectraPlot(Plot):
                 CODE = 'cspc'
                 colormap = 'jet'
                 plot_type = 'pcolor'
                 zmin_coh = None
                 zmax_coh = None
                 zmin_phase = None
                 zmax_phase = None
                 realChannels = None
                 crossPairs = None
                 def setup(self):
                     self.ncols = 4
                     self.nplots = len(self.data.pairs) * 2
                     self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
                     self.width = 3.1 * self.ncols
                     self.height = 2.6 * self.nrows
                     self.ylabel = 'Range [km]'
                     self.showprofile = False
                     self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
                 def update(self, dataOut):
                     data = {}
                     meta = {}
                     spc = dataOut.data_spc
                     cspc = dataOut.data_cspc
                     meta['xrange'] = (dataOut.getFreqRange(EXTRA_POINTS)/1000., dataOut.getAcfRange(EXTRA_POINTS), dataOut.getVelRange(EXTRA_POINTS))
                     rawPairs = list(combinations(list(range(dataOut.nChannels)), 2))
                     meta['pairs'] = rawPairs
                     if self.crossPairs == None:
                         self.crossPairs = dataOut.pairsList
                     tmp = []
                     for n, pair in enumerate(meta['pairs']):
                         out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
                         coh = numpy.abs(out)
                         phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
                         tmp.append(coh)
                         tmp.append(phase)
                     data['cspc'] = numpy.array(tmp)
                     return data, meta
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.data.xrange[0]
                         self.xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.data.xrange[1]
                         self.xlabel = "Time (ms)"
                     else:
                         x = self.data.xrange[2]
                         self.xlabel = "Velocity (m/s)"
                     self.titles = []
                     y = self.data.yrange
                     self.y = y
                     data = self.data[-1]
                     cspc = data['cspc']
                     for n in range(len(self.data.pairs)):
                         pair = self.crossPairs[n]
                         coh = cspc[n*2]
                         phase = cspc[n*2+1]
                         ax = self.axes[2 * n]
                         if ax.firsttime:
                             ax.plt = ax.pcolormesh(x, y, coh.T,
                                                    vmin=self.zmin_coh,
                                                    vmax=self.zmax_coh,
                                                    cmap=plt.get_cmap(self.colormap_coh)
                                                    )
                         else:
                             ax.plt.set_array(coh.T.ravel())
                         self.titles.append(
                             'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
                         ax = self.axes[2 * n + 1]
                         if ax.firsttime:
                             ax.plt = ax.pcolormesh(x, y, phase.T,
                                                    vmin=-180,
                                                    vmax=180,
                                                    cmap=plt.get_cmap(self.colormap_phase)
                                                    )
                         else:
                             ax.plt.set_array(phase.T.ravel())
                         self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
             class CrossSpectra4Plot(Plot):
                 CODE = 'cspc'
                 colormap = 'jet'
                 plot_type = 'pcolor'
                 zmin_coh = None
                 zmax_coh = None
                 zmin_phase = None
                 zmax_phase = None
                 def setup(self):
                     self.ncols = 4
                     self.nrows = len(self.data.pairs)
                     self.nplots = self.nrows * 4
                     self.width = 3.1 * self.ncols
                     self.height = 5 * self.nrows
                     self.ylabel = 'Range [km]'
                     self.showprofile = False
                     self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.data.xrange[0]
                         self.xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.data.xrange[1]
                         self.xlabel = "Time (ms)"
                     else:
                         x = self.data.xrange[2]
                         self.xlabel = "Velocity (m/s)"
                     self.titles = []
                     y = self.data.heights
                     self.y = y
                     nspc = self.data['spc']
                     spc = self.data['cspc'][0]
                     cspc = self.data['cspc'][1]
                     for n in range(self.nrows):
                         noise = self.data['noise'][:,-1]
                         pair = self.data.pairs[n]
                         ax = self.axes[4 * n]
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.zmin = self.zmin if self.zmin else numpy.nanmin(nspc)
                             self.zmax = self.zmax if self.zmax else numpy.nanmax(nspc)
                             ax.plt = ax.pcolormesh(x , y , nspc[pair[0]].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                         else:
                             ax.plt.set_array(nspc[pair[0]].T.ravel())
                         self.titles.append('CH {}: {:3.2f}dB'.format(pair[0], noise[pair[0]]))
                         ax = self.axes[4 * n + 1]
                         if ax.firsttime:
                             ax.plt = ax.pcolormesh(x , y, numpy.flip(nspc[pair[1]],axis=0).T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                         else:
                             ax.plt.set_array(numpy.flip(nspc[pair[1]],axis=0).T.ravel())
                         self.titles.append('CH {}: {:3.2f}dB'.format(pair[1], noise[pair[1]]))
                         out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
                         coh = numpy.abs(out)
                         phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
                         ax = self.axes[4 * n + 2]
                         if ax.firsttime:
                             ax.plt = ax.pcolormesh(x, y, numpy.flip(coh,axis=0).T,
                                                    vmin=0,
                                                    vmax=1,
                                                    cmap=plt.get_cmap(self.colormap_coh)
                                                    )
                         else:
                             ax.plt.set_array(numpy.flip(coh,axis=0).T.ravel())
                         self.titles.append(
                             'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
                         ax = self.axes[4 * n + 3]
                         if ax.firsttime:
                             ax.plt = ax.pcolormesh(x, y, numpy.flip(phase,axis=0).T,
                                                    vmin=-180,
                                                    vmax=180,
                                                    cmap=plt.get_cmap(self.colormap_phase)
                                                    )
                         else:
                             ax.plt.set_array(numpy.flip(phase,axis=0).T.ravel())
                         self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
             class CrossSpectra2Plot(Plot):
                 CODE = 'cspc'
                 colormap = 'jet'
                 plot_type = 'pcolor'
                 zmin_coh = None
                 zmax_coh = None
                 zmin_phase = None
                 zmax_phase = None
                 def setup(self):
                     self.ncols = 1
                     self.nrows = len(self.data.pairs)
                     self.nplots = self.nrows * 1
                     self.width = 3.1 * self.ncols
                     self.height = 5 * self.nrows
                     self.ylabel = 'Range [km]'
                     self.showprofile = False
                     self.plots_adjust.update({'left': 0.22, 'right': .90, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.data.xrange[0]
                         self.xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.data.xrange[1]
                         self.xlabel = "Time (ms)"
                     else:
                         x = self.data.xrange[2]
                         self.xlabel = "Velocity (m/s)"
                     self.titles = []
                     y = self.data.heights
                     self.y = y
                     cspc = self.data['cspc'][1]
                     for n in range(self.nrows):
                         noise = self.data['noise'][:,-1]
                         pair = self.data.pairs[n]
                         out = cspc[n]
                         cross = numpy.abs(out)
                         z = cross/self.data.nFactor
                         cross = 10*numpy.log10(z)
                         ax = self.axes[1 * n]
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.zmin = self.zmin if self.zmin else numpy.nanmin(cross)
                             self.zmax = self.zmax if self.zmax else numpy.nanmax(cross)
                             ax.plt = ax.pcolormesh(x, y, cross.T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                         else:
                             ax.plt.set_array(cross.T.ravel())
                         self.titles.append(
                             'Cross Spectra Power Ch{} * Ch{}'.format(pair[0], pair[1]))
             class CrossSpectra3Plot(Plot):
                 CODE = 'cspc'
                 colormap = 'jet'
                 plot_type = 'pcolor'
                 zmin_coh = None
                 zmax_coh = None
                 zmin_phase = None
                 zmax_phase = None
                 def setup(self):
                     self.ncols = 3
                     self.nrows = len(self.data.pairs)
                     self.nplots = self.nrows * 3
                     self.width = 3.1 * self.ncols
                     self.height = 5 * self.nrows
                     self.ylabel = 'Range [km]'
                     self.showprofile = False
                     self.plots_adjust.update({'left': 0.22, 'right': .90, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.data.xrange[0]
                         self.xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.data.xrange[1]
                         self.xlabel = "Time (ms)"
                     else:
                         x = self.data.xrange[2]
                         self.xlabel = "Velocity (m/s)"
                     self.titles = []
                     y = self.data.heights
                     self.y = y
                     cspc = self.data['cspc'][1]
                     for n in range(self.nrows):
                         noise = self.data['noise'][:,-1]
                         pair = self.data.pairs[n]
                         out = cspc[n]
                         cross = numpy.abs(out)
                         z = cross/self.data.nFactor
                         cross = 10*numpy.log10(z)
                         out_r= out.real/self.data.nFactor
                         out_i= out.imag/self.data.nFactor
                         ax = self.axes[3 * n]
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.zmin = self.zmin if self.zmin else numpy.nanmin(cross)
                             self.zmax = self.zmax if self.zmax else numpy.nanmax(cross)
                             ax.plt = ax.pcolormesh(x, y, cross.T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                         else:
                             ax.plt.set_array(cross.T.ravel())
                         self.titles.append(
                             'Cross Spectra Power Ch{} * Ch{}'.format(pair[0], pair[1]))
                         ax = self.axes[3 * n + 1]
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.zmin = self.zmin if self.zmin else numpy.nanmin(cross)
                             self.zmax = self.zmax if self.zmax else numpy.nanmax(cross)
                             ax.plt = ax.pcolormesh(x, y, out_r.T,
                                                    vmin=-1.e6,
                                                    vmax=0,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                         else:
                             ax.plt.set_array(out_r.T.ravel())
                         self.titles.append(
                             'Cross Spectra Real Ch{} * Ch{}'.format(pair[0], pair[1]))
                         ax = self.axes[3 * n + 2]
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.zmin = self.zmin if self.zmin else numpy.nanmin(cross)
                             self.zmax = self.zmax if self.zmax else numpy.nanmax(cross)
                             ax.plt = ax.pcolormesh(x, y, out_i.T,
                                                    vmin=-1.e6,
                                                    vmax=1.e6,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                         else:
                             ax.plt.set_array(out_i.T.ravel())
                         self.titles.append(
                             'Cross Spectra Imag Ch{} * Ch{}'.format(pair[0], pair[1]))
             class RTIPlot(Plot):
                 '''
                 Plot for RTI data
                 '''
                 CODE = 'rti'
                 colormap = 'jet'
                 plot_type = 'pcolorbuffer'
                 titles = None
                 channelList = []
                 elevationList = []
                 azimuthList = []
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     self.nrows = len(self.data.channels)
                     self.nplots = len(self.data.channels)
                     self.ylabel = 'Range [km]'
                     #self.xlabel = 'Time'
                     self.cb_label = 'dB'
                     self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.1, 'right':0.95})
                     self.titles = ['{} Channel {}'.format(
                         self.CODE.upper(), x) for x in range(self.nplots)]
                 def update_list(self,dataOut):
                     if len(self.channelList) == 0:
                         self.channelList = dataOut.channelList
                     if len(self.elevationList) == 0:
                         self.elevationList = dataOut.elevationList
                     if len(self.azimuthList) == 0:
                         self.azimuthList = dataOut.azimuthList
                 def update(self, dataOut):
                     if len(self.channelList) == 0:
                         self.update_list(dataOut)
                     data = {}
                     meta = {}
                     data['rti'] = dataOut.getPower()
                     norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt  * dataOut.windowOfFilter
                     noise = 10*numpy.log10(dataOut.getNoise()/norm)
                     data['noise'] = noise
                     return data, meta
                 def plot(self):
                     self.x = self.data.times
                     self.y = self.data.yrange
                     self.z = self.data[self.CODE]
                     self.z = numpy.array(self.z, dtype=float)
                     self.z = numpy.ma.masked_invalid(self.z)
                     try:
                         if self.channelList != None:
                             if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
                                 self.titles = ['{}  Channel {} ({:2.1f} Elev,  {:2.1f} Azth)'.format(
                                     self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
                             else:
                                 self.titles = ['{}  Channel {}'.format(
                                     self.CODE.upper(), x) for x in self.channelList]
                     except:
                         if self.channelList.any() != None:
                             if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
                                 self.titles = ['{}  Channel {} ({:2.1f} Elev,  {:2.1f} Azth)'.format(
                                     self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
                             else:
                                 self.titles = ['{} Channel {}'.format(
                                     self.CODE.upper(), x) for x in self.channelList]
                     if self.decimation is None:
                         x, y, z = self.fill_gaps(self.x, self.y, self.z)
                     else:
                         x, y, z = self.fill_gaps(*self.decimate())
                     for n, ax in enumerate(self.axes):
                         self.zmin = self.zmin if self.zmin else numpy.min(self.z)
                         self.zmax = self.zmax if self.zmax else numpy.max(self.z)
                         data = self.data[-1]
                         if ax.firsttime:
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                             if self.showprofile:
-                                ax.plot_profile = self.pf_axes[n].plot(
+                                ax.plot_profile = self.pf_axes[n].plot(data[self.CODE][n], self.y)[0]
-                                    data[self.CODE][n], self.y)[0]
                                 if "noise" in self.data:
                                     ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
                                                                      color="k", linestyle="dashed", lw=1)[0]
                         else:
-                            ax.collections.remove(ax.collections[0]) #  error while running in 3.12
+                            ax.collections.remove(ax.collections[0])
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                             if self.showprofile:
                                 ax.plot_profile.set_data(data[self.CODE][n], self.y)
                                 if "noise" in self.data:
-                                    ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
+                                    ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
-                                                                     color="k", linestyle="dashed", lw=1)[0]
             class SpectrogramPlot(Plot):
                 '''
                 Plot for Spectrogram data
                 '''
                 CODE = 'Spectrogram_Profile'
                 colormap = 'binary'
                 plot_type = 'pcolorbuffer'
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     self.nrows = len(self.data.channels)
                     self.nplots = len(self.data.channels)
                     self.xlabel = 'Time'
                     self.plots_adjust.update({'hspace':1.2, 'left': 0.1, 'bottom': 0.12, 'right':0.95})
                     self.titles = []
                     self.titles = ['{} Channel {}'.format(
                         self.CODE.upper(), x) for x in range(self.nrows)]
                 def update(self, dataOut):
                     data = {}
                     meta = {}
                     maxHei = 1620#+12000
                     indb = numpy.where(dataOut.heightList <= maxHei)
                     hei = indb[0][-1]
                     factor = dataOut.nIncohInt
                     z = dataOut.data_spc[:,:,hei] / factor
                     z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                     meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
                     data['Spectrogram_Profile'] = 10 * numpy.log10(z)
                     data['hei'] = hei
                     data['DH'] = (dataOut.heightList[1] - dataOut.heightList[0])/dataOut.step
                     data['nProfiles'] = dataOut.nProfiles
                     return data, meta
                 def plot(self):
                     self.x = self.data.times
                     self.z = self.data[self.CODE]
                     self.y = self.data.xrange[0]
                     hei = self.data['hei'][-1]
                     DH = self.data['DH'][-1]
                     nProfiles = self.data['nProfiles'][-1]
                     self.ylabel = "Frequency (kHz)"
                     self.z = numpy.ma.masked_invalid(self.z)
                     if self.decimation is None:
                         x, y, z = self.fill_gaps(self.x, self.y, self.z)
                     else:
                         x, y, z = self.fill_gaps(*self.decimate())
                     for n, ax in enumerate(self.axes):
                         self.zmin = self.zmin if self.zmin else numpy.min(self.z)
                         self.zmax = self.zmax if self.zmax else numpy.max(self.z)
                         data = self.data[-1]
                         if ax.firsttime:
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                         else:
-                            # ax.collections.remove(ax.collections[0]) #  error while running
+                            ax.collections.remove(ax.collections[0]) #  error while running
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
             class CoherencePlot(RTIPlot):
                 '''
                 Plot for Coherence data
                 '''
                 CODE = 'coh'
                 titles = None
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     self.nrows = len(self.data.pairs)
                     self.nplots = len(self.data.pairs)
                     self.ylabel = 'Range [km]'
                     self.xlabel = 'Time'
                     self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1,'right':0.95})
                     if self.CODE == 'coh':
                         self.cb_label = ''
                         self.titles = [
                             'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
                     else:
                         self.cb_label = 'Degrees'
                         self.titles = [
                             'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
                 def update(self, dataOut):
                     data = {}
                     meta = {}
                     data['coh'] = dataOut.getCoherence()
                     meta['pairs'] = dataOut.pairsList
                     return data, meta
             class PhasePlot(CoherencePlot):
                 '''
                 Plot for Phase map data
                 '''
                 CODE = 'phase'
                 colormap = 'seismic'
                 def update(self, dataOut):
                     data = {}
                     meta = {}
                     data['phase'] = dataOut.getCoherence(phase=True)
                     meta['pairs'] = dataOut.pairsList
                     return data, meta
             class NoisePlot(Plot):
                 '''
                 Plot for noise
                 '''
                 CODE = 'noise'
                 plot_type = 'scatterbuffer'
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     self.nrows = 1
                     self.nplots = 1
                     self.ylabel = 'Intensity [dB]'
                     self.xlabel = 'Time'
                     self.titles = ['Noise']
                     self.colorbar = False
                     self.plots_adjust.update({'right': 0.85 })
                     self.titles = ['Noise Plot']
                 def update(self, dataOut):
                     data = {}
                     meta = {}
                     noise =  10*numpy.log10(dataOut.getNoise())
                     noise = noise.reshape(dataOut.nChannels, 1)
                     data['noise'] = noise
                     meta['yrange'] = numpy.array([])
                     return data, meta
                 def plot(self):
                     x = self.data.times
                     xmin = self.data.min_time
                     xmax = xmin + self.xrange * 60 * 60
                     Y = self.data['noise']
                     if self.axes[0].firsttime:
                         self.ymin = numpy.nanmin(Y) - 5
                         self.ymax = numpy.nanmax(Y) + 5
                         for ch in self.data.channels:
                             y = Y[ch]
                             self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
                         plt.legend(bbox_to_anchor=(1.18, 1.0))
                     else:
                         for ch in self.data.channels:
                             y = Y[ch]
                             self.axes[0].lines[ch].set_data(x, y)
             class PowerProfilePlot(Plot):
                 CODE = 'pow_profile'
                 plot_type = 'scatter'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = 1
                     self.nplots = 1
                     self.height = 4
                     self.width = 3
                     self.ylabel = 'Range [km]'
                     self.xlabel = 'Intensity [dB]'
                     self.titles = ['Power Profile']
                     self.colorbar = False
                 def update(self, dataOut):
                     data = {}
                     meta = {}
                     data[self.CODE] = dataOut.getPower()
                     return data, meta
                 def plot(self):
                     y = self.data.yrange
                     self.y = y
                     x = self.data[-1][self.CODE]
                     if self.xmin is None: self.xmin = numpy.nanmin(x)*0.9
                     if self.xmax is None: self.xmax = numpy.nanmax(x)*1.1
                     if self.axes[0].firsttime:
                         for ch in self.data.channels:
                             self.axes[0].plot(x[ch], y, lw=1, label='Ch{}'.format(ch))
                         plt.legend()
                     else:
                         for ch in self.data.channels:
                             self.axes[0].lines[ch].set_data(x[ch], y)
             class SpectraCutPlot(Plot):
                 CODE = 'spc_cut'
                 plot_type = 'scatter'
                 buffering = False
                 heights = []
                 channelList = []
                 maintitle = "Spectra Cuts"
                 flag_setIndex = False
                 def setup(self):
                     self.nplots = len(self.data.channels)
                     self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
                     self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
                     self.width = 4.5 * self.ncols + 2.5
                     self.height = 4.8 * self.nrows
                     self.ylabel = 'Power [dB]'
                     self.colorbar = False
                     self.plots_adjust.update({'left':0.1, 'hspace':0.3, 'right': 0.9, 'bottom':0.08})
                     if len(self.selectedHeightsList) > 0:
                         self.maintitle = "Spectra Cut"# for %d km " %(int(self.selectedHeight))
                 def update(self, dataOut):
                     if len(self.channelList) == 0:
                         self.channelList = dataOut.channelList
                     self.heights = dataOut.heightList
                     #print("sels: ",self.selectedHeightsList)
                     if len(self.selectedHeightsList)>0 and not self.flag_setIndex:
                         for sel_height in self.selectedHeightsList:
                             index_list = numpy.where(self.heights >= sel_height)
                             index_list = index_list[0]
                             self.height_index.append(index_list[0])
                         #print("sels i:"", self.height_index)
                         self.flag_setIndex = True
                         #print(self.height_index)
                     data = {}
                     meta = {}
                     norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt  * dataOut.windowOfFilter#*dataOut.nFFTPoints
                     n0 = 10*numpy.log10(dataOut.getNoise()/norm)
                     noise = numpy.repeat(n0,(dataOut.nFFTPoints*dataOut.nHeights)).reshape(dataOut.nChannels,dataOut.nFFTPoints,dataOut.nHeights)
                     z = []
                     for ch in range(dataOut.nChannels):
                         if hasattr(dataOut.normFactor,'shape'):
                             z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
                         else:
                             z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
                     z = numpy.asarray(z)
                     z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                     spc = 10*numpy.log10(z)
                     data['spc'] = spc - noise
                     meta['xrange'] = (dataOut.getFreqRange(EXTRA_POINTS)/1000., dataOut.getAcfRange(EXTRA_POINTS), dataOut.getVelRange(EXTRA_POINTS))
                     return data, meta
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.data.xrange[0][0:]
                         self.xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.data.xrange[1]
                         self.xlabel = "Time (ms)"
                     else:
                         x = self.data.xrange[2]
                         self.xlabel = "Velocity (m/s)"
                     self.titles = []
                     y = self.data.yrange
                     z = self.data[-1]['spc']
                     #print(z.shape)
                     if len(self.height_index) > 0:
                         index = self.height_index
                     else:
                         index = numpy.arange(0, len(y), int((len(y))/9))
                     #print("inde x ", index, self.axes)
                     for n, ax in enumerate(self.axes):
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.ymin = self.ymin if self.ymin else numpy.nanmin(z)
                             self.ymax = self.ymax if self.ymax else numpy.nanmax(z)
                             ax.plt = ax.plot(x, z[n, :, index].T)
                             labels = ['Range = {:2.1f}km'.format(y[i]) for i in index]
                             self.figures[0].legend(ax.plt, labels, loc='center right', prop={'size': 8})
                             ax.minorticks_on()
                             ax.grid(which='major', axis='both')
                             ax.grid(which='minor', axis='x')
                         else:
                             for i, line in enumerate(ax.plt):
                                 line.set_data(x, z[n, :, index[i]])
                         self.titles.append('CH {}'.format(self.channelList[n]))
                     plt.suptitle(self.maintitle,  fontsize=10)
             class BeaconPhase(Plot):
                 __isConfig = None
                 __nsubplots = None
                 PREFIX = 'beacon_phase'
                 def __init__(self):
                     Plot.__init__(self)
                     self.timerange = 24*60*60
                     self.isConfig = False
                     self.__nsubplots = 1
                     self.counter_imagwr = 0
                     self.WIDTH = 800
                     self.HEIGHT = 400
                     self.WIDTHPROF = 120
                     self.HEIGHTPROF = 0
                     self.xdata = None
                     self.ydata = None
                     self.PLOT_CODE = BEACON_CODE
                     self.FTP_WEI = None
                     self.EXP_CODE = None
                     self.SUB_EXP_CODE = None
                     self.PLOT_POS = None
                     self.filename_phase = None
                     self.figfile = None
                     self.xmin = None
                     self.xmax = None
                 def getSubplots(self):
                     ncol = 1
                     nrow = 1
                     return nrow, ncol
                 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
                     self.__showprofile = showprofile
                     self.nplots = nplots
                     ncolspan = 7
                     colspan = 6
                     self.__nsubplots = 2
                     self.createFigure(id = id,
                                       wintitle = wintitle,
                                       widthplot = self.WIDTH+self.WIDTHPROF,
                                       heightplot = self.HEIGHT+self.HEIGHTPROF,
                                       show=show)
                     nrow, ncol = self.getSubplots()
                     self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
                 def save_phase(self, filename_phase):
                     f = open(filename_phase,'w+')
                     f.write('\n\n')
                     f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
                     f.write('DD MM YYYY  HH MM SS   pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
                     f.close()
                 def save_data(self, filename_phase, data, data_datetime):
                     f=open(filename_phase,'a')
                     timetuple_data = data_datetime.timetuple()
                     day = str(timetuple_data.tm_mday)
                     month = str(timetuple_data.tm_mon)
                     year = str(timetuple_data.tm_year)
                     hour = str(timetuple_data.tm_hour)
                     minute = str(timetuple_data.tm_min)
                     second = str(timetuple_data.tm_sec)
                     f.write(day+' '+month+' '+year+'  '+hour+' '+minute+' '+second+'   '+str(data[0])+'   '+str(data[1])+'   '+str(data[2])+'   '+str(data[3])+'\n')
                     f.close()
                 def plot(self):
                     log.warning('TODO: Not yet implemented...')
                 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
                         xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
                         timerange=None,
                         save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                         server=None, folder=None, username=None, password=None,
                         ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                     if dataOut.flagNoData:
                         return dataOut
                     if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
                         return
                     if pairsList == None:
                         pairsIndexList = dataOut.pairsIndexList[:10]
                     else:
                         pairsIndexList = []
                         for pair in pairsList:
                             if pair not in dataOut.pairsList:
                                 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
                             pairsIndexList.append(dataOut.pairsList.index(pair))
                     if pairsIndexList == []:
                         return
              #         if len(pairsIndexList) > 4:
              #             pairsIndexList = pairsIndexList[0:4]
                     hmin_index = None
                     hmax_index = None
                     if hmin != None and hmax != None:
                         indexes = numpy.arange(dataOut.nHeights)
                         hmin_list = indexes[dataOut.heightList >= hmin]
                         hmax_list = indexes[dataOut.heightList <= hmax]
                         if hmin_list.any():
                             hmin_index = hmin_list[0]
                         if hmax_list.any():
                             hmax_index = hmax_list[-1]+1
                     x = dataOut.getTimeRange()
                     thisDatetime = dataOut.datatime
                     title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
                     xlabel = "Local Time"
                     ylabel = "Phase (degrees)"
                     update_figfile = False
                     nplots = len(pairsIndexList)
                     phase_beacon = numpy.zeros(len(pairsIndexList))
                     for i in range(nplots):
                         pair = dataOut.pairsList[pairsIndexList[i]]
                         ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
                         powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
                         powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
                         avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
                         phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
                         if dataOut.beacon_heiIndexList:
                             phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
                         else:
                             phase_beacon[i] = numpy.average(phase)
                     if not self.isConfig:
                         nplots = len(pairsIndexList)
                         self.setup(id=id,
                                    nplots=nplots,
                                    wintitle=wintitle,
                                    showprofile=showprofile,
                                    show=show)
                         if timerange != None:
                             self.timerange = timerange
                         self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
                         if ymin == None: ymin = 0
                         if ymax == None: ymax = 360
                         self.FTP_WEI = ftp_wei
                         self.EXP_CODE = exp_code
                         self.SUB_EXP_CODE = sub_exp_code
                         self.PLOT_POS = plot_pos
                         self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
                         self.isConfig = True
                         self.figfile = figfile
                         self.xdata = numpy.array([])
                         self.ydata = numpy.array([])
                         update_figfile = True
                         #open file beacon phase
                         path = '%s%03d' %(self.PREFIX, self.id)
                         beacon_file = os.path.join(path,'%s.txt'%self.name)
                         self.filename_phase = os.path.join(figpath,beacon_file)
                     self.setWinTitle(title)
                     title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
                     legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
                     axes = self.axesList[0]
                     self.xdata = numpy.hstack((self.xdata, x[0:1]))
                     if len(self.ydata)==0:
                         self.ydata = phase_beacon.reshape(-1,1)
                     else:
                         self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
                     axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
                                 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
                                 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
                                 XAxisAsTime=True, grid='both'
                                 )
                     self.draw()
                     if dataOut.ltctime >= self.xmax:
                         self.counter_imagwr = wr_period
                         self.isConfig = False
                         update_figfile = True
                     self.save(figpath=figpath,
                               figfile=figfile,
                               save=save,
                               ftp=ftp,
                               wr_period=wr_period,
                               thisDatetime=thisDatetime,
                               update_figfile=update_figfile)
                     return dataOut
             #####################################
             class NoiselessSpectraPlot(Plot):
                 '''
                 Plot for Spectra data, subtracting
                 the noise in all channels, using for
                 amisr-14 data
                 '''
                 CODE = 'noiseless_spc'
                 colormap = 'jet'
                 plot_type = 'pcolor'
                 buffering = False
                 channelList = []
                 last_noise = None
                 def setup(self):
                     self.nplots = len(self.data.channels)
                     self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
                     self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
                     self.height = 3.5 * self.nrows
                     self.cb_label = 'dB'
                     if self.showprofile:
                         self.width = 5.8 * self.ncols
                     else:
                         self.width = 4.8* self.ncols
                     self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.92, 'bottom': 0.12})
                     self.ylabel = 'Range [km]'
                 def update_list(self,dataOut):
                     if len(self.channelList) == 0:
                         self.channelList = dataOut.channelList
                 def update(self, dataOut):
                     self.update_list(dataOut)
                     data = {}
                     meta = {}
                     norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt  * dataOut.windowOfFilter
                     n0 = (dataOut.getNoise()/norm)
                     noise = numpy.repeat(n0,(dataOut.nFFTPoints*dataOut.nHeights)).reshape(dataOut.nChannels,dataOut.nFFTPoints,dataOut.nHeights)
                     noise = 10*numpy.log10(noise)
                     z = numpy.zeros((dataOut.nChannels, dataOut.nFFTPoints, dataOut.nHeights))
                     for ch in range(dataOut.nChannels):
                         if hasattr(dataOut.normFactor,'ndim'):
                             if dataOut.normFactor.ndim > 1:
                                 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
                             else:
                                 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
                         else:
                             z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
                     z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                     spc = 10*numpy.log10(z)
                     data['spc'] = spc - noise
                     #print(spc.shape)
                     data['rti'] = spc.mean(axis=1)
                     data['noise'] = noise
                     # data['noise'] = noise
                     meta['xrange'] = (dataOut.getFreqRange(EXTRA_POINTS)/1000., dataOut.getAcfRange(EXTRA_POINTS), dataOut.getVelRange(EXTRA_POINTS))
                     return data, meta
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.data.xrange[0]
                         self.xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.data.xrange[1]
                         self.xlabel = "Time (ms)"
                     else:
                         x = self.data.xrange[2]
                         self.xlabel = "Velocity (m/s)"
                     self.titles = []
                     y = self.data.yrange
                     self.y = y
                     data = self.data[-1]
                     z = data['spc']
                     for n, ax in enumerate(self.axes):
                         #noise = data['noise'][n]
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
                             self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                             if self.showprofile:
                                 ax.plt_profile = self.pf_axes[n].plot(
                                     data['rti'][n], y)[0]
                         else:
                             ax.plt.set_array(z[n].T.ravel())
                             if self.showprofile:
                                 ax.plt_profile.set_data(data['rti'][n], y)
                         self.titles.append('CH {}'.format(self.channelList[n]))
             class NoiselessRTIPlot(RTIPlot):
                 '''
                 Plot for RTI data
                 '''
                 CODE = 'noiseless_rti'
                 colormap = 'jet'
                 plot_type = 'pcolorbuffer'
                 titles = None
                 channelList = []
                 elevationList = []
                 azimuthList = []
                 last_noise = None
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     #print("dataChannels ",self.data.channels)
                     self.nrows = len(self.data.channels)
                     self.nplots = len(self.data.channels)
                     self.ylabel = 'Range [km]'
                     #self.xlabel = 'Time'
                     self.cb_label = 'dB'
                     self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
                     self.titles = ['{} Channel {}'.format(
                         self.CODE.upper(), x) for x in range(self.nplots)]
                 def update_list(self,dataOut):
                     if len(self.channelList) == 0:
                         self.channelList = dataOut.channelList
                     if len(self.elevationList) == 0:
                         self.elevationList = dataOut.elevationList
                     if len(self.azimuthList) == 0:
                         self.azimuthList = dataOut.azimuthList
                 def update(self, dataOut):
                     if len(self.channelList) == 0:
                         self.update_list(dataOut)
                     data = {}
                     meta = {}
                     #print(dataOut.max_nIncohInt, dataOut.nIncohInt)
                     #print(dataOut.windowOfFilter,dataOut.nCohInt,dataOut.nProfiles,dataOut.max_nIncohInt,dataOut.nIncohInt
                     norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt  * dataOut.windowOfFilter
                     n0 = 10*numpy.log10(dataOut.getNoise()/norm)
                     data['noise'] = n0
                     noise = numpy.repeat(n0,dataOut.nHeights).reshape(dataOut.nChannels,dataOut.nHeights)
                     noiseless_data = dataOut.getPower() - noise
                     #print("power, noise:", dataOut.getPower(), n0)
                     #print(noise)
                     #print(noiseless_data)
                     data['noiseless_rti'] = noiseless_data
                     return data, meta
                 def plot(self):
                     from matplotlib import pyplot as plt
                     self.x = self.data.times
                     self.y = self.data.yrange
                     self.z = self.data['noiseless_rti']
                     self.z = numpy.array(self.z, dtype=float)
                     self.z = numpy.ma.masked_invalid(self.z)
                     try:
                         if self.channelList != None:
                             if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
                                 self.titles = ['{}  Channel {} ({:2.1f} Elev,  {:2.1f} Azth)'.format(
                                     self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
                             else:
                                 self.titles = ['{}  Channel {}'.format(
                                     self.CODE.upper(), x) for x in self.channelList]
                     except:
                         if self.channelList.any() != None:
                             if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
                                 self.titles = ['{}  Channel {} ({:2.1f} Elev,  {:2.1f} Azth)'.format(
                                     self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
                             else:
                                 self.titles = ['{} Channel {}'.format(
                                     self.CODE.upper(), x) for x in self.channelList]
                     if self.decimation is None:
                         x, y, z = self.fill_gaps(self.x, self.y, self.z)
                     else:
                         x, y, z = self.fill_gaps(*self.decimate())
                     dummy_var = self.axes #Extrañamente esto actualiza el valor axes
                     #print("plot shapes ", z.shape, x.shape, y.shape)
                     #print(self.axes)
                     for n, ax in enumerate(self.axes):
                         self.zmin = self.zmin if self.zmin else numpy.min(self.z)
                         self.zmax = self.zmax if self.zmax else numpy.max(self.z)
                         data = self.data[-1]
                         if ax.firsttime:
                             if (n+1) == len(self.channelList):
                                 ax.set_xlabel('Time')
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                             if self.showprofile:
                                 ax.plot_profile = self.pf_axes[n].plot(data['noiseless_rti'][n], self.y)[0]
                         else:
-                            # ax.collections.remove(ax.collections[0]) #  error while running
+                            ax.collections.remove(ax.collections[0]) #  error while running
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                             if self.showprofile:
                                 ax.plot_profile.set_data(data['noiseless_rti'][n], self.y)
                                 # if "noise" in self.data:
                                 #     #ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
                                 #     ax.plot_noise.set_data(data['noise'][n], self.y)
             class OutliersRTIPlot(Plot):
                 '''
                 Plot for data_xxxx object
                 '''
                 CODE = 'outlier_rtc' # Range Time Counts
                 colormap = 'cool'
                 plot_type = 'pcolorbuffer'
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     self.nrows = self.data.shape('outlier_rtc')[0]
                     self.nplots = self.nrows
                     self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
                     if not self.xlabel:
                         self.xlabel = 'Time'
                     self.ylabel = 'Height [km]'
                     if not self.titles:
                         self.titles = ['Outliers Ch:{}'.format(x) for x in range(self.nrows)]
                 def update(self, dataOut):
                     data = {}
                     data['outlier_rtc'] = dataOut.data_outlier
                     meta = {}
                     return data, meta
                 def plot(self):
                     # self.data.normalize_heights()
                     self.x = self.data.times
                     self.y = self.data.yrange
                     self.z = self.data['outlier_rtc']
                     #self.z = numpy.ma.masked_invalid(self.z)
                     if self.decimation is None:
                         x, y, z = self.fill_gaps(self.x, self.y, self.z)
                     else:
                         x, y, z = self.fill_gaps(*self.decimate())
                     for n, ax in enumerate(self.axes):
                         self.zmax = self.zmax if self.zmax is not None else numpy.max(
                             self.z[n])
                         self.zmin = self.zmin if self.zmin is not None else numpy.min(
                             self.z[n])
                         data = self.data[-1]
                         if ax.firsttime:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=self.cmaps[n]
                                                    )
                             if self.showprofile:
                                 ax.plot_profile = self.pf_axes[n].plot(data['outlier_rtc'][n], self.y)[0]
                                 self.pf_axes[n].set_xlabel('')
                         else:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
-                            # ax.collections.remove(ax.collections[0]) #  error while running
+                            ax.collections.remove(ax.collections[0]) #  error while running
                             ax.plt = ax.pcolormesh(x, y, z[n].T ,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=self.cmaps[n]
                                                    )
                             if self.showprofile:
                                 ax.plot_profile.set_data(data['outlier_rtc'][n], self.y)
                                 self.pf_axes[n].set_xlabel('')
             class NIncohIntRTIPlot(Plot):
                 '''
                 Plot for data_xxxx object
                 '''
                 CODE = 'integrations_rtc' # Range Time Counts
                 colormap = 'BuGn'
                 plot_type = 'pcolorbuffer'
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     self.nrows = self.data.shape('integrations_rtc')[0]
                     self.nplots = self.nrows
                     self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
                     if not self.xlabel:
                         self.xlabel = 'Time'
                     self.ylabel = 'Height [km]'
                     if not self.titles:
                         self.titles = ['Integration Ch:{}'.format(x) for x in range(self.nrows)]
                 def update(self, dataOut):
                     data = {}
                     data['integrations_rtc'] = dataOut.nIncohInt
                     meta = {}
                     return data, meta
                 def plot(self):
                     # self.data.normalize_heights()
                     self.x = self.data.times
                     self.y = self.data.yrange
                     self.z = self.data['integrations_rtc']
                     #self.z = numpy.ma.masked_invalid(self.z)
                     if self.decimation is None:
                         x, y, z = self.fill_gaps(self.x, self.y, self.z)
                     else:
                         x, y, z = self.fill_gaps(*self.decimate())
                     for n, ax in enumerate(self.axes):
                         self.zmax = self.zmax if self.zmax is not None else numpy.max(
                             self.z[n])
                         self.zmin = self.zmin if self.zmin is not None else numpy.min(
                             self.z[n])
                         data = self.data[-1]
                         if ax.firsttime:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=self.cmaps[n]
                                                    )
                             if self.showprofile:
                                 ax.plot_profile = self.pf_axes[n].plot(data['integrations_rtc'][n], self.y)[0]
                                 self.pf_axes[n].set_xlabel('')
                         else:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
-                            # ax.collections.remove(ax.collections[0]) #  error while running
+                            ax.collections.remove(ax.collections[0]) #  error while running
                             ax.plt = ax.pcolormesh(x, y, z[n].T ,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=self.cmaps[n]
                                                    )
                             if self.showprofile:
                                 ax.plot_profile.set_data(data['integrations_rtc'][n], self.y)
                                 self.pf_axes[n].set_xlabel('')
             class RTIMapPlot(Plot):
                 '''
                 Plot for RTI data
                 Example:
                 controllerObj = Project()
                 controllerObj.setup(id = '11', name='eej_proc', description=desc)
                 ##.......................................................................................
                 ##.......................................................................................
                 readUnitConfObj = controllerObj.addReadUnit(datatype='AMISRReader', path=inPath, startDate='2023/05/24',endDate='2023/05/24',
                                 startTime='12:00:00',endTime='12:45:59',walk=1,timezone='lt',margin_days=1,code = code,nCode = nCode,
                                 nBaud = nBaud,nOsamp = nosamp,nChannels=nChannels,nFFT=NFFT,
                                 syncronization=False,shiftChannels=0)
                 volts_proc = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
                 opObj01 = volts_proc.addOperation(name='Decoder', optype='other')
                 opObj01.addParameter(name='code', value=code, format='floatlist')
                 opObj01.addParameter(name='nCode', value=1, format='int')
                 opObj01.addParameter(name='nBaud', value=nBaud, format='int')
                 opObj01.addParameter(name='osamp', value=nosamp, format='int')
                 opObj12 = volts_proc.addOperation(name='selectHeights', optype='self')
                 opObj12.addParameter(name='minHei', value='90', format='float')
                 opObj12.addParameter(name='maxHei', value='150', format='float')
                 proc_spc = controllerObj.addProcUnit(datatype='SpectraProc', inputId=volts_proc.getId())
                 proc_spc.addParameter(name='nFFTPoints', value='8', format='int')
                 opObj11 = proc_spc.addOperation(name='IncohInt', optype='other')
                 opObj11.addParameter(name='n', value='1', format='int')
                 beamMapFile = "/home/japaza/Documents/AMISR_sky_mapper/UMET_beamcodes.csv"
                 opObj12 = proc_spc.addOperation(name='RTIMapPlot', optype='external')
                 opObj12.addParameter(name='selectedHeightsList', value='95, 100, 105, 110 ', format='int')
                 opObj12.addParameter(name='bField', value='100', format='int')
                 opObj12.addParameter(name='filename', value=beamMapFile, format='str')
                 '''
                 CODE = 'rti_skymap'
                 plot_type = 'scatter'
                 titles = None
                 colormap = 'jet'
                 channelList = []
                 elevationList = []
                 azimuthList = []
                 last_noise = None
                 flag_setIndex = False
                 heights = []
                 dcosx = []
                 dcosy = []
                 fullDcosy = None
                 fullDcosy = None
                 hindex = []
                 mapFile = False
                 #####  BField    ####
                 flagBField = False
                 dcosxB = []
                 dcosyB = []
                 Bmarker = ['+','*','D','x','s','>','o','^']
                 def setup(self):
                     self.xaxis = 'Range (Km)'
                     if len(self.selectedHeightsList) > 0:
                         self.nplots = len(self.selectedHeightsList)
                     else:
                         self.nplots = 4
                     self.ncols = int(numpy.ceil(self.nplots/2))
                     self.nrows = int(numpy.ceil(self.nplots/self.ncols))
                     self.ylabel = 'dcosy'
                     self.xlabel = 'dcosx'
                     self.colorbar = True
                     self.width = 6 + 4.1*self.nrows
                     self.height = 3 + 3.5*self.ncols
                     if self.extFile!=None:
                         try:
                             pointings =  numpy.genfromtxt(self.extFile, delimiter=',')
                             full_azi = pointings[:,1]
                             full_elev = pointings[:,2]
                             self.fullDcosx = numpy.cos(numpy.radians(full_elev))*numpy.sin(numpy.radians(full_azi))
                             self.fullDcosy = numpy.cos(numpy.radians(full_elev))*numpy.cos(numpy.radians(full_azi))
                             mapFile = True
                         except Exception as e:
                             self.extFile = None
                             print(e)
                 def update_list(self,dataOut):
                     if len(self.channelList) == 0:
                         self.channelList = dataOut.channelList
                     if len(self.elevationList) == 0:
                         self.elevationList = dataOut.elevationList
                     if len(self.azimuthList) == 0:
                         self.azimuthList = dataOut.azimuthList
                     a = numpy.radians(numpy.asarray(self.azimuthList))
                     e = numpy.radians(numpy.asarray(self.elevationList))
                     self.heights = dataOut.heightList
                     self.dcosx = numpy.cos(e)*numpy.sin(a)
                     self.dcosy = numpy.cos(e)*numpy.cos(a)
                     if len(self.bFieldList)>0:
                         datetObj = datetime.datetime.fromtimestamp(dataOut.utctime)
                         doy = datetObj.timetuple().tm_yday
                         year = datetObj.year
                         # self.dcosxB, self.dcosyB
                         ObjB = BField(year=year,doy=doy,site=2,heights=self.bFieldList)
                         [dcos, alpha, nlon, nlat] = ObjB.getBField()
                         alpha_location = numpy.zeros((nlon,2,len(self.bFieldList)))
                         for ih in range(len(self.bFieldList)):
                             alpha_location[:,0,ih] = dcos[:,0,ih,0]
                             for ilon in numpy.arange(nlon):
                                 myx = (alpha[ilon,:,ih])[::-1]
                                 myy = (dcos[ilon,:,ih,0])[::-1]
                                 tck = splrep(myx,myy,s=0)
                                 mydcosx = splev(ObjB.alpha_i,tck,der=0)
                                 myx = (alpha[ilon,:,ih])[::-1]
                                 myy = (dcos[ilon,:,ih,1])[::-1]
                                 tck = splrep(myx,myy,s=0)
                                 mydcosy = splev(ObjB.alpha_i,tck,der=0)
                                 alpha_location[ilon,:,ih] = numpy.array([mydcosx, mydcosy])
                             self.dcosxB.append(alpha_location[:,0,ih])
                             self.dcosyB.append(alpha_location[:,1,ih])
                         self.flagBField = True
                     if len(self.celestialList)>0:
                         #getBField(self.bFieldList, date)
                         #pass = kwargs.get('celestial', [])
                         pass
                 def update(self, dataOut):
                     if len(self.channelList) == 0:
                         self.update_list(dataOut)
                     if not self.flag_setIndex:
                         if len(self.selectedHeightsList)>0:
                             for sel_height in self.selectedHeightsList:
                                 index_list = numpy.where(self.heights >= sel_height)
                                 index_list = index_list[0]
                                 self.hindex.append(index_list[0])
                         self.flag_setIndex = True
                     data = {}
                     meta = {}
                     data['rti_skymap'] = dataOut.getPower()
                     norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt  * dataOut.windowOfFilter
                     noise = 10*numpy.log10(dataOut.getNoise()/norm)
                     data['noise'] = noise
                     return data, meta
                 def plot(self):
                     self.x = self.dcosx
                     self.y = self.dcosy
                     self.z = self.data[-1]['rti_skymap']
                     self.z = numpy.array(self.z, dtype=float)
                     if len(self.hindex) > 0:
                         index = self.hindex
                     else:
                         index = numpy.arange(0, len(self.heights), int((len(self.heights))/4.2))
                     self.titles = ['Height {:.2f} km '.format(self.heights[i])+" " for i in index]
                     for n, ax in enumerate(self.axes):
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else numpy.nanmax(self.x)
                             self.xmin = self.xmin if self.xmin else numpy.nanmin(self.x)
                             self.ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
                             self.ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
                             self.zmax = self.zmax if self.zmax else numpy.nanmax(self.z)
                             self.zmin = self.zmin if self.zmin else numpy.nanmin(self.z)
                             if self.extFile!=None:
                                 ax.scatter(self.fullDcosx, self.fullDcosy, marker="+", s=20)
                             ax.plt = ax.scatter(self.x, self.y, c=self.z[:,index[n]], cmap = 'jet',vmin = self.zmin,
                                 s=60, marker="s", vmax = self.zmax)
                             ax.minorticks_on()
                             ax.grid(which='major', axis='both')
                             ax.grid(which='minor', axis='x')
                             if self.flagBField :
                                 for ih in range(len(self.bFieldList)):
                                     label = str(self.bFieldList[ih]) + ' km'
                                     ax.plot(self.dcosxB[ih], self.dcosyB[ih], color='k', marker=self.Bmarker[ih % 8],
                                                      label=label,   linestyle='--',  ms=4.0,lw=0.5)
                                 handles, labels = ax.get_legend_handles_labels()
                                 a = -0.05
                                 b = 1.15 - 1.19*(self.nrows)
                                 self.axes[0].legend(handles,labels, bbox_to_anchor=(a,b), prop={'size': (5.8+ 1.1*self.nplots)}, title='B Field ⊥')
                         else:
                             ax.plt = ax.scatter(self.x, self.y, c=self.z[:,index[n]], cmap = 'jet',vmin = self.zmin,
                                  s=80, marker="s", vmax = self.zmax)
                             if self.flagBField :
                                 for ih in range(len(self.bFieldList)):
                                     ax.plot (self.dcosxB[ih], self.dcosyB[ih], color='k', marker=self.Bmarker[ih % 8],
                                             linestyle='--', ms=4.0,lw=0.5)

schainpy/model/io/jroIO_param.py +3 -2

             import os
             import time
             import datetime
             import numpy
             import h5py
             import schainpy.admin
             from schainpy.model.data.jrodata import *
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.io.jroIO_base import *
             from schainpy.utils import log
             class HDFReader(Reader, ProcessingUnit):
                 """Processing unit to read HDF5 format files
                 This unit reads HDF5 files created with `HDFWriter` operation contains
                 by default two groups Data and Metadata all variables would be saved as `dataOut`
                 attributes.
                 It is possible to read any HDF5 file by given the structure in the `description`
                 parameter, also you can add extra values to metadata with the parameter `extras`.
                 Parameters:
                 -----------
                 path : str
                     Path where files are located.
                 startDate : date
                     Start date of the files
                 endDate : list
                     End date of the files
                 startTime : time
                     Start time of the files
                 endTime : time
                     End time of the files
                 description : dict, optional
                     Dictionary with the description of the HDF5 file
                 extras : dict, optional
                     Dictionary with extra metadata to be be added to `dataOut`
                 Attention:   Be carefull, add  attribute utcoffset, in the last part of reader in order to work in Local Time without time problems.
                 -----------
                 utcoffset='-18000'
                 Examples
                 --------
                 desc = {
                     'Data': {
                         'data_output': ['u', 'v', 'w'],
                         'utctime': 'timestamps',
                     }  ,
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 desc = {
                     'Data': {
                         'data_output': 'winds',
                         'utctime': 'timestamps'
                     },
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 extras = {
                     'timeZone': 300
                 }
                 reader = project.addReadUnit(
                     name='HDFReader',
                     path='/path/to/files',
                     startDate='2019/01/01',
                     endDate='2019/01/31',
                     startTime='00:00:00',
                     endTime='23:59:59',
                     utcoffset='-18000'
                     # description=json.dumps(desc),
                     # extras=json.dumps(extras),
                     )
                 """
                 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.ext = ".hdf5"
                     self.optchar = "D"
                     self.meta = {}
                     self.data = {}
                     self.open_file = h5py.File
                     self.open_mode = 'r'
                     self.description = {}
                     self.extras = {}
                     self.filefmt = "*%Y%j***"
                     self.folderfmt = "*%Y%j"
                     self.utcoffset = 0
                     self.flagUpdateDataOut = False
                     self.dataOut = Parameters()
                     self.dataOut.error=False            ## NOTE: Importante definir esto antes inicio
                     self.dataOut.flagNoData = True
                 def setup(self, **kwargs):
                     self.set_kwargs(**kwargs)
                     if not self.ext.startswith('.'):
                         self.ext = '.{}'.format(self.ext)
                     if self.online:
                         log.log("Searching files in online mode...", self.name)
                         for nTries in range(self.nTries):
                             fullpath = self.searchFilesOnLine(self.path, self.startDate,
                                 self.endDate, self.expLabel, self.ext, self.walk,
                                 self.filefmt, self.folderfmt)
                             pathname, filename = os.path.split(fullpath)
                             try:
                                 fullpath = next(fullpath)
                             except:
                                 fullpath = None
                             if fullpath:
                                 break
                             log.warning(
                                 'Waiting {} sec for a valid file in {}: try {} ...'.format(
                                     self.delay, self.path, nTries + 1),
                                 self.name)
                             time.sleep(self.delay)
                         if not(fullpath):
                             raise schainpy.admin.SchainError(
                                 'There isn\'t any valid file in {}'.format(self.path))
                         pathname, filename = os.path.split(fullpath)
                         self.year = int(filename[1:5])
                         self.doy = int(filename[5:8])
                         self.set = int(filename[8:11]) - 1
                     else:
                         log.log("Searching files in {}".format(self.path), self.name)
                         self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
                             self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
                     self.setNextFile()
                     return
                 # def readFirstHeader(self):
                 #     '''Read metadata and data'''
                 #     self.__readMetadata()
                 #     self.__readData()
                 #     self.__setBlockList()
                 #     if 'type' in self.meta:
                 #         self.dataOut = eval(self.meta['type'])()
                 #     for attr in self.meta:
                 #         setattr(self.dataOut, attr, self.meta[attr])
                 #     self.blockIndex = 0
                 #     return
                 def readFirstHeader(self):
                     '''Read metadata and data'''
                     self.__readMetadata2()
                     self.__readData()
                     self.__setBlockList()
-                    if 'type' in self.meta:
+                    # if 'type' in self.meta:
-                        self.dataOut = eval(self.meta['type'])()
+                    #     self.dataOut = eval(self.meta['type'])()
                     for attr in self.meta:
                         if "processingHeaderObj" in attr:
                             self.flagUpdateDataOut=True
                         at = attr.split('.')
                         if len(at) > 1:
                             setattr(eval("self.dataOut."+at[0]),at[1], self.meta[attr])
                         else:
                             setattr(self.dataOut, attr, self.meta[attr])
                     self.blockIndex = 0
                     if self.flagUpdateDataOut:
                         self.updateDataOut()
                     return
                 def updateDataOut(self):
                     self.dataOut.azimuthList = self.dataOut.processingHeaderObj.azimuthList
                     self.dataOut.elevationList = self.dataOut.processingHeaderObj.elevationList
                     self.dataOut.heightList = self.dataOut.processingHeaderObj.heightList
                     self.dataOut.ippSeconds = self.dataOut.processingHeaderObj.ipp
                     self.dataOut.elevationList = self.dataOut.processingHeaderObj.elevationList
                     self.dataOut.channelList = self.dataOut.processingHeaderObj.channelList
                     self.dataOut.nCohInt = self.dataOut.processingHeaderObj.nCohInt
                     self.dataOut.nFFTPoints = self.dataOut.processingHeaderObj.nFFTPoints
                     self.flagUpdateDataOut = False
                     self.dataOut.frequency = self.dataOut.radarControllerHeaderObj.frequency
                     #self.dataOut.heightList = self.dataOut.processingHeaderObj.heightList
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     startTime = self.startTime
                     endTime = self.endTime
                     thisUtcTime = self.data['utctime'] + self.utcoffset
                     # self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
                     thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
                     self.startFileDatetime = thisDatetime
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     # self.blocksPerFile = len(thisUtcTime)
                     if len(ind)==0:
                         print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.blockIndex,
                                                                                   self.blocksPerFile,
                                                                                   thisDatetime))
                         self.setNextFile()
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     '''
                     meta = {}
                     if self.description:
                         for key, value in self.description['Metadata'].items():
                             meta[key] = self.fp[value][()]
                     else:
                         grp = self.fp['Metadata']
                         for name in grp:
                             meta[name] = grp[name][()]
                     if self.extras:
                         for key, value in self.extras.items():
                             meta[key] = value
                     self.meta = meta
                     return
                 def __readMetadata2(self):
                     '''
                     Reads Metadata
                     '''
                     meta = {}
                     if self.description:
                         for key, value in self.description['Metadata'].items():
                             meta[key] = self.fp[value][()]
                     else:
                         grp = self.fp['Metadata']
                         for item in grp.values():
                             name = item.name
                             if isinstance(item, h5py.Dataset):
                                 name = name.split("/")[-1]
                                 meta[name] = item[()]
                             else:
                                 grp2 = self.fp[name]
                                 Obj = name.split("/")[-1]
                                 for item2 in grp2.values():
                                     name2 = Obj+"."+item2.name.split("/")[-1]
                                     meta[name2] = item2[()]
                     if self.extras:
                         for key, value in self.extras.items():
                             meta[key] = value
                     self.meta = meta
                     return
                 def __readData(self):
                     data = {}
                     if self.description:
                         for key, value in self.description['Data'].items():
                             if isinstance(value, str):
                                 if isinstance(self.fp[value], h5py.Dataset):
                                     data[key] = self.fp[value][()]
                                 elif isinstance(self.fp[value], h5py.Group):
                                     array = []
                                     for ch in self.fp[value]:
                                         array.append(self.fp[value][ch][()])
                                     data[key] = numpy.array(array)
                             elif isinstance(value, list):
                                 array = []
                                 for ch in value:
                                     array.append(self.fp[ch][()])
                                 data[key] = numpy.array(array)
                     else:
                         grp = self.fp['Data']
                         for name in grp:
                             if isinstance(grp[name], h5py.Dataset):
                                 array = grp[name][()]
                             elif isinstance(grp[name], h5py.Group):
                                 array = []
                                 for ch in grp[name]:
                                     array.append(grp[name][ch][()])
                                 array = numpy.array(array)
                             else:
                                 log.warning('Unknown type: {}'.format(name))
                             if name in self.description:
                                 key = self.description[name]
                             else:
                                 key = name
                             data[key] = array
                     self.data = data
                     return
                 def getData(self):
                     if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime):
                         self.dataOut.flagNoData = True
                         self.blockIndex = self.blocksPerFile
                         self.dataOut.error = True     # TERMINA EL PROGRAMA
                         return
                     for attr in self.data:
                         if self.data[attr].ndim == 1:
                             setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
                         else:
                             setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
                     self.blockIndex += 1
                     if self.blockIndex == 1:
                         log.log("Block No. {}/{} -> {}".format(
                             self.blockIndex,
                             self.blocksPerFile,
                             self.dataOut.datatime.ctime()), self.name)
                     else:
                         log.log("Block No. {}/{} ".format(
                             self.blockIndex,
                             self.blocksPerFile),self.name)
                     if self.blockIndex == self.blocksPerFile:
                         self.setNextFile()
                     self.dataOut.flagNoData = False
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     if self.blockIndex == self.blocksPerFile:
                         self.setNextFile()
                     self.getData()
                     return
             @MPDecorator
             class HDFWriter(Operation):
                 """Operation to write HDF5 files.
                 The HDF5 file contains by default two groups Data and Metadata where
                 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
                 parameters, data attributes are normaly time dependent where the metadata
                 are not.
                 It is possible to customize the structure of the HDF5 file with the
                 optional description parameter see the examples.
                 Parameters:
                 -----------
                 path : str
                     Path where files will be saved.
                 blocksPerFile : int
                     Number of blocks per file
                 metadataList : list
                     List of the dataOut attributes that will be saved as metadata
                 dataList : int
                     List of the dataOut attributes that will be saved as data
                 setType : bool
                     If True the name of the files corresponds to the timestamp of the data
                 description : dict, optional
                     Dictionary with the desired description of the HDF5 file
                 Examples
                 --------
                 desc = {
                     'data_output': {'winds': ['z', 'w', 'v']},
                     'utctime': 'timestamps',
                     'heightList': 'heights'
                 }
                 desc = {
                     'data_output': ['z', 'w', 'v'],
                     'utctime': 'timestamps',
                     'heightList': 'heights'
                 }
                 desc = {
                     'Data': {
                         'data_output': 'winds',
                         'utctime': 'timestamps'
                     },
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 writer = proc_unit.addOperation(name='HDFWriter')
                 writer.addParameter(name='path', value='/path/to/file')
                 writer.addParameter(name='blocksPerFile', value='32')
                 writer.addParameter(name='metadataList', value='heightList,timeZone')
                 writer.addParameter(name='dataList',value='data_output,utctime')
                 # writer.addParameter(name='description',value=json.dumps(desc))
                 """
                 ext = ".hdf5"
                 optchar = "D"
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 ds = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None                  #eval ??????
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 currentDay = None
                 lastTime = None
                 timeZone = "ut"
                 hourLimit = 3
                 breakDays = True
                 def __init__(self):
                     Operation.__init__(self)
                     return
                 def set_kwargs(self, **kwargs):
                     for key, value in kwargs.items():
                         setattr(self, key, value)
                 def set_kwargs_obj(self, obj, **kwargs):
                     for key, value in kwargs.items():
                         setattr(obj, key, value)
                 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None,
                             description={},timeZone = "ut",hourLimit = 3, breakDays=True, **kwargs):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = [s.strip() for s in dataList]
                     self.setType = setType
                     self.description = description
                     self.timeZone = timeZone
                     self.hourLimit = hourLimit
                     self.breakDays = breakDays
                     self.set_kwargs(**kwargs)
                     if self.metadataList is None:
                         self.metadataList = self.dataOut.metadata_list
                     self.metadataList = list(set(self.metadataList))
                     tableList = []
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         if hasattr(self.dataOut, self.dataList[i]):
                             dataAux = getattr(self.dataOut, self.dataList[i])
                             dsDict['variable'] = self.dataList[i]
                         else:
                             log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]),self.name)
                             continue
                         if dataAux is None:
                             continue
                         elif isinstance(dataAux, (int, float, numpy.integer, numpy.float_)):
                             dsDict['nDim'] = 0
                         else:
                             dsDict['nDim'] = len(dataAux.shape)
                             dsDict['shape'] = dataAux.shape
                             dsDict['dsNumber'] = dataAux.shape[0]
                             dsDict['dtype'] = dataAux.dtype
                         dsList.append(dsDict)
                     self.blockIndex = 0
                     self.dsList = dsList
                     self.currentDay = self.dataOut.datatime.date()
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     timeTuple = None
                     if self.timeZone == "lt":
                         timeTuple = time.localtime(currentTime)
                     else :
                         timeTuple = time.gmtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     if self.lastTime is None:
                         self.lastTime = currentTime
                         self.currentDay = dataDay
                         return False
                     timeDiff = currentTime - self.lastTime
                     # Si el dia es diferente o si la diferencia entre un
                     # dato y otro supera self.hourLimit
                     if (dataDay != self.currentDay) and self.breakDays:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > self.hourLimit*60*60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
                         dataList=[], setType=None, description={}, **kwargs):
                     self.dataOut = dataOut
                     self.set_kwargs_obj(self.dataOut, **kwargs)
                     if not(self.isConfig):
                         self.setup(path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList,
                                    setType=setType, description=description, **kwargs)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = None
                     if self.timeZone == "lt":
                         timeTuple = time.localtime(self.dataOut.utctime)
                     elif self.timeZone == "ut":
                         timeTuple = time.gmtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     if os.path.exists(fullpath):
                         filesList = os.listdir(fullpath)
                         filesList = [k for k in filesList if k.startswith(self.optchar)]
                         if len(filesList) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext)
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext)
                     self.filename = os.path.join(path, subfolder, file)
                 def getLabel(self, name, x=None):
                     if x is None:
                         if 'Data' in self.description:
                             data = self.description['Data']
                             if 'Metadata' in self.description:
                                 data.update(self.description['Metadata'])
                         else:
                             data = self.description
                         if name in data:
                             if isinstance(data[name], str):
                                 return data[name]
                             elif isinstance(data[name], list):
                                 return None
                             elif isinstance(data[name], dict):
                                 for key, value in data[name].items():
                                     return key
                         return name
                     else:
                         if 'Metadata' in self.description:
                             meta = self.description['Metadata']
                         else:
                             meta = self.description
                         if name in meta:
                             if isinstance(meta[name], list):
                                 return meta[name][x]
                             elif isinstance(meta[name], dict):
                                 for key, value in meta[name].items():
                                     return value[x]
                         if 'cspc' in name:
                             return 'pair{:02d}'.format(x)
                         else:
                             return 'channel{:02d}'.format(x)
                 def writeMetadata(self, fp):
                     if self.description:
                         if 'Metadata' in self.description:
                             grp = fp.create_group('Metadata')
                         else:
                             grp = fp
                     else:
                         grp = fp.create_group('Metadata')
                     for i in range(len(self.metadataList)):
                         if not hasattr(self.dataOut, self.metadataList[i]):
                             log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
                             continue
                         value = getattr(self.dataOut, self.metadataList[i])
                         if isinstance(value, bool):
                             if value is True:
                                 value = 1
                             else:
                                 value = 0
                         grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
                     return
                 def writeMetadata2(self, fp):
                     if self.description:
                         if 'Metadata' in self.description:
                             grp = fp.create_group('Metadata')
                         else:
                             grp = fp
                     else:
                         grp = fp.create_group('Metadata')
                     for i in range(len(self.metadataList)):
                         attribute = self.metadataList[i]
                         attr = attribute.split('.')
                         if len(attr) > 1:
                             if not hasattr(eval("self.dataOut."+attr[0]),attr[1]):
                                 log.warning('Metadata: {}.{} not found'.format(attr[0],attr[1]), self.name)
                                 continue
                             value = getattr(eval("self.dataOut."+attr[0]),attr[1])
                             if isinstance(value, bool):
                                 if value is True:
                                     value = 1
                                 else:
                                     value = 0
                             if isinstance(value,type(None)):
                                 log.warning("Invalid value detected, {} is None".format(attribute), self.name)
                                 value = 0
                             grp2 = None
                             if not 'Metadata/'+attr[0] in fp:
                                 grp2 = fp.create_group('Metadata/'+attr[0])
                             else:
                                 grp2 = fp['Metadata/'+attr[0]]
                             grp2.create_dataset(attr[1], data=value)
                         else:
                             if not hasattr(self.dataOut, attr[0] ):
                                 log.warning('Metadata: `{}` not found'.format(attribute), self.name)
                                 continue
                             value = getattr(self.dataOut, attr[0])
                             if isinstance(value, bool):
                                 if value is True:
                                     value = 1
                                 else:
                                     value = 0
                             if isinstance(value, type(None)):
                                 log.error("Value {} is None".format(attribute),self.name)
                             grp.create_dataset(self.getLabel(attribute), data=value)
                     return
                 def writeData(self, fp):
                     if self.description:
                         if 'Data' in self.description:
                             grp = fp.create_group('Data')
                         else:
                             grp = fp
                     else:
                         grp = fp.create_group('Data')
                     dtsets = []
                     data = []
                     for dsInfo in self.dsList:
                         if dsInfo['nDim'] == 0:
                             ds = grp.create_dataset(
                                 self.getLabel(dsInfo['variable']),
                                 (self.blocksPerFile,),
                                 chunks=True,
                                 dtype=numpy.float64)
                             dtsets.append(ds)
                             data.append((dsInfo['variable'], -1))
                         else:
                             label = self.getLabel(dsInfo['variable'])
                             if label is not None:
                                 sgrp = grp.create_group(label)
                             else:
                                 sgrp = grp
                             for i in range(dsInfo['dsNumber']):
                                 ds = sgrp.create_dataset(
                                     self.getLabel(dsInfo['variable'], i),
                                     (self.blocksPerFile,) + dsInfo['shape'][1:],
                                     chunks=True,
                                     dtype=dsInfo['dtype'])
                                 dtsets.append(ds)
                                 data.append((dsInfo['variable'], i))
                     fp.flush()
                     log.log('Creating file: {}'.format(fp.filename), self.name)
                     self.ds = dtsets
                     self.data = data
                     self.firsttime = True
                     return
                 def putData(self):
                     if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
                         self.closeFile()
                         self.setNextFile()
                         self.dataOut.flagNoData = False
                         self.blockIndex = 0
                     if self.blockIndex == 0:
                         #Setting HDF5 File
                         self.fp = h5py.File(self.filename, 'w')
                         #write metadata
                         self.writeMetadata2(self.fp)
                         #Write data
                         self.writeData(self.fp)
                         log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
                     elif (self.blockIndex % 10 ==0):
                         log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
                     else:
                         log.log('Block No. {}/{}'.format(self.blockIndex+1, self.blocksPerFile), self.name)
                     for i, ds in enumerate(self.ds):
                         attr, ch = self.data[i]
                         if ch == -1:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)
                         else:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
                     self.blockIndex += 1
                     self.fp.flush()
                     self.dataOut.flagNoData = True
                 def closeFile(self):
                     if self.blockIndex != self.blocksPerFile:
                         for ds in self.ds:
                             ds.resize(self.blockIndex, axis=0)
                     if self.fp:
                         self.fp.flush()
                         self.fp.close()
                 def close(self):
                     self.closeFile()

schainpy/model/io/utilsIO.py +24 -13

             """
             Utilities for IO modules
             @modified: Joab Apaza
             @email: roj-op01@igp.gob.pe, joab.apaza32@gmail.com
             """
             ################################################################################
             ################################################################################
             import os
             from datetime import datetime
             import numpy
             from schainpy.model.data.jrodata import Parameters
             import itertools
             import numpy
             import h5py
             import re
             import time
             from schainpy.utils import log
             ################################################################################
             ################################################################################
             ################################################################################
             def folder_in_range(folder, start_date, end_date, pattern):
                 """
                 Check whether folder is bettwen start_date and end_date
                 Args:
                     folder (str): Folder to check
                     start_date (date): Initial date
                     end_date (date): Final date
                     pattern (str): Datetime format of the folder
                 Returns:
                     bool: True for success, False otherwise
                 """
                 try:
                     dt = datetime.strptime(folder, pattern)
                 except:
                     raise ValueError('Folder {} does not match {} format'.format(folder, pattern))
                 return start_date <= dt.date() <= end_date
             ################################################################################
             ################################################################################
             ################################################################################
             def getHei_index( minHei, maxHei, heightList):
                 try:
                     if (minHei < heightList[0]):
                         minHei = heightList[0]
                     if (maxHei > heightList[-1]):
                         maxHei = heightList[-1]
                     minIndex = 0
                     maxIndex = 0
                     heights = numpy.asarray(heightList)
                     inda = numpy.where(heights >= minHei)
                     indb = numpy.where(heights <= maxHei)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(heightList)
                     return minIndex,maxIndex
                 except Exception as e:
                     log.error("In getHei_index: ", __name__)
                     log.error(e , __name__)
             ################################################################################
             ################################################################################
             ################################################################################
             class MergeH5(object):
                 """Processing unit to read HDF5 format files
                 This unit reads HDF5 files created with `HDFWriter` operation when channels area
                 processed by separated. Then merge all channels in a single files.
                 "example"
                 nChannels = 4
                 pathOut = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/merged"
                 p0 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch0"
                 p1 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch1"
                 p2 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch2"
                 p3 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch3"
                 list = ['data_spc','data_cspc','nIncohInt','utctime']
                 merger = MergeH5(nChannels,pathOut,list, p0, p1,p2,p3)
                 merger.run()
                 The file example_FULLmultiprocessing_merge.txt show an application for AMISR data
                 """
                 # #__attrs__ = ['paths', 'nChannels']
                 isConfig = False
                 inPaths = None
                 nChannels = None
                 ch_dataIn = []
                 channelList = []
                 def __init__(self,nChannels, pOut, dataList, *args):
                     self.inPaths = [p for p in args]
                     #print(self.inPaths)
                     if len(self.inPaths) != nChannels:
                         print("ERROR, number of channels different from iput paths {} != {}".format(nChannels, len(args)))
                         return
                     self.pathOut = pOut
                     self.dataList = dataList
                     self.nChannels = len(self.inPaths)
                     self.ch_dataIn = [Parameters() for p in args]
                     self.dataOut = Parameters()
                     self.channelList = [n for n in range(nChannels)]
                     self.blocksPerFile = None
                     self.date = None
                     self.ext = ".hdf5$"
                     self.dataList = dataList
                     self.optchar = "D"
                     self.meta = {}
                     self.data = {}
                     self.open_file = h5py.File
                     self.open_mode = 'r'
                     self.description = {}
                     self.extras = {}
                     self.filefmt = "*%Y%j***"
                     self.folderfmt = "*%Y%j"
                     self.flag_spc = False
                     self.flag_pow = False
                     self.flag_snr = False
                     self.flag_nIcoh = False
                     self.flagProcessingHeader = False
                     self.flagControllerHeader = False
                 def setup(self):
                     # if not self.ext.startswith('.'):
                     #     self.ext = '.{}'.format(self.ext)
                     self.filenameList = self.searchFiles(self.inPaths, None)
                     self.nfiles = len(self.filenameList[0])
                 def searchFiles(self, paths, date, walk=True):
                     # self.paths = path
                     #self.date = startDate
                     #self.walk = walk
                     filenameList = [[] for n in range(self.nChannels)]
                     ch = 0
                     for path in paths:
                         if os.path.exists(path):
                             print("Searching files in {}".format(path))
                             filenameList[ch] = self.getH5files(path, walk)
                             print("Found: ")
                             for f in filenameList[ch]:
                                 print(f)
                         else:
                             self.status = 0
                             print('Path:%s does not exists'%path)
                             return 0
                         ch+=1
                     return filenameList
                 def getH5files(self, path, walk):
                     dirnameList = []
                     pat = '(\d)+.'+self.ext
                     if walk:
                         for root, dirs, files in os.walk(path):
                             for dir in dirs:
                                 #print(os.path.join(root,dir))
                                 files = [re.search(pat,x) for x in os.listdir(os.path.join(root,dir))]
                                 #print(files)
                                 files = [x for x in files if x!=None]
                                 files = [x.string for x in files]
                                 files = [os.path.join(root,dir,x) for x in files if x!=None]
                                 files.sort()
                                 dirnameList += files
                         return dirnameList
                     else:
                         dirnameList = [re.search(pat,x) for x in os.listdir(path)]
                         dirnameList = [x for x in dirnameList if x!=None]
                         dirnameList = [x.string for x in dirnameList]
                         dirnameList = [x for x in dirnameList if x!=None]
                         dirnameList.sort()
                     return dirnameList
                 def readFile(self,fp,ch):
                     '''Read metadata and data'''
                     self.readMetadata(fp,ch)
-                    #print(self.metadataList)
+                    # print(self.metadataList)
                     data = self.readData(fp)
                     for attr in self.meta:
                         if "processingHeaderObj" in attr:
                             self.flagProcessingHeader=True
                         if "radarControllerHeaderObj" in attr:
                             self.flagControllerHeader=True
                         at = attr.split('.')
-                        #print("AT ", at)
+                        # print("AT ", at)
                         if len(at) > 1:
                             setattr(eval("self.ch_dataIn[ch]."+at[0]),at[1], self.meta[attr])
                         else:
                             setattr(self.ch_dataIn[ch], attr, self.meta[attr])
                     self.fill_dataIn(data, self.ch_dataIn[ch])
                     return
                 def readMetadata(self, fp, ch):
                     '''
                     Reads Metadata
                     '''
                     meta = {}
                     self.metadataList = []
                     grp = fp['Metadata']
                     for item in grp.values():
                         name = item.name
                         if isinstance(item, h5py.Dataset):
                             name = name.split("/")[-1]
                             if 'List' in name:
                                 meta[name] = item[()].tolist()
                             else:
                                 meta[name] = item[()]
                             self.metadataList.append(name)
                         else:
                             grp2 = fp[name]
                             Obj = name.split("/")[-1]
                             #print(Obj)
                             for item2 in grp2.values():
                                 name2 = Obj+"."+item2.name.split("/")[-1]
                                 if 'List' in name2:
                                     meta[name2] = item2[()].tolist()
                                 else:
                                     meta[name2] = item2[()]
                                 self.metadataList.append(name2)
                     if not self.meta:
                         self.meta = meta.copy()
                         for key in list(self.meta.keys()):
                             if "channelList" in key:
                                 self.meta["channelList"] =[n for n in range(self.nChannels)]
                             if "processingHeaderObj" in key:
                                 self.meta["processingHeaderObj.channelList"] =[n for n in range(self.nChannels)]
                             if "radarControllerHeaderObj" in key:
                                 self.meta["radarControllerHeaderObj.channelList"] =[n for n in range(self.nChannels)]
                         return 1
                     else:
                         for k in list(self.meta.keys()):
                             if 'List' in k and 'channel' not in k and "height" not in k and "radarControllerHeaderObj" not in k:
                                 self.meta[k] += meta[k]
                     #print("Metadata: ",self.meta)
                     return 1
                 def fill_dataIn(self,data, dataIn):
                     for attr in data:
                         if data[attr].ndim == 1:
                             setattr(dataIn, attr, data[attr][:])
                         else:
                             setattr(dataIn, attr, numpy.squeeze(data[attr][:,:]))
-                    #print("shape in", dataIn.data_spc.shape, len(dataIn.data_spc))
+                    # print("shape in", dataIn.data_spc.shape, len(dataIn.data_spc))
                     if self.flag_spc:
                         if dataIn.data_spc.ndim > 3:
                             dataIn.data_spc = dataIn.data_spc[0]
                             #print("shape in", dataIn.data_spc.shape)
                 def getBlocksPerFile(self):
                     b = numpy.zeros(self.nChannels)
                     for i in range(self.nChannels):
                         if self.flag_spc:
                             b[i] = self.ch_dataIn[i].data_spc.shape[0] #number of blocks
                         elif self.flag_pow:
                             b[i] = self.ch_dataIn[i].data_pow.shape[0] #number of blocks
                         elif self.flag_snr:
                             b[i] = self.ch_dataIn[i].data_snr.shape[0] #number of blocks
                     self.blocksPerFile = int(b.min())
                     iresh_ch = numpy.where(b > self.blocksPerFile)[0]
                     if len(iresh_ch) > 0:
                         for ich in iresh_ch:
                             for i in range(len(self.dataList)):
                                 if hasattr(self.ch_dataIn[ich], self.dataList[i]):
                                     # print("reshaping ", self.dataList[i])
                                     # print(getattr(self.ch_dataIn[ich], self.dataList[i]).shape)
                                     dataAux = getattr(self.ch_dataIn[ich], self.dataList[i])
                                     setattr(self.ch_dataIn[ich], self.dataList[i], None)
                                     setattr(self.ch_dataIn[ich], self.dataList[i], dataAux[0:self.blocksPerFile])
                                     # print(getattr(self.ch_dataIn[ich], self.dataList[i]).shape)
                     else:
+                        # log.error("Channels number error,iresh_ch=", iresh_ch)
                         return
                 def getLabel(self, name, x=None):
                     if x is None:
                         if 'Data' in self.description:
                             data = self.description['Data']
                             if 'Metadata' in self.description:
                                 data.update(self.description['Metadata'])
                         else:
                             data = self.description
                         if name in data:
                             if isinstance(data[name], str):
                                 return data[name]
                             elif isinstance(data[name], list):
                                 return None
                             elif isinstance(data[name], dict):
                                 for key, value in data[name].items():
                                     return key
                         return name
                     else:
                         if 'Metadata' in self.description:
                             meta = self.description['Metadata']
                         else:
                             meta = self.description
                         if name in meta:
                             if isinstance(meta[name], list):
                                 return meta[name][x]
                             elif isinstance(meta[name], dict):
                                 for key, value in meta[name].items():
                                     return value[x]
                         if 'cspc' in name:
                             return 'pair{:02d}'.format(x)
                         else:
                             return 'channel{:02d}'.format(x)
                 def readData(self, fp):
-                    #print("read fp: ", fp)
+                    # print("read fp: ", fp)
                     data = {}
                     grp = fp['Data']
                     for name in grp:
                         if "spc" in name:
                             self.flag_spc = True
                         if "pow" in name:
                             self.flag_pow = True
                         if "snr" in name:
                             self.flag_snr = True
                         if "nIncohInt" in name:
                             self.flag_nIcoh = True
+                        # print("spc:",self.flag_spc," pow:",self.flag_pow," snr:", self.flag_snr)
                         if isinstance(grp[name], h5py.Dataset):
                             array = grp[name][()]
                         elif isinstance(grp[name], h5py.Group):
                             array = []
                             for ch in grp[name]:
                                 array.append(grp[name][ch][()])
                             array = numpy.array(array)
                         else:
                             print('Unknown type: {}'.format(name))
                         data[name] = array
                     return data
                 def getDataOut(self):
+                    # print("Getting DataOut")
                     self.dataOut = self.ch_dataIn[0].copy()  #dataIn    #blocks, fft, hei for metadata
                     if self.flagProcessingHeader:
                         self.dataOut.processingHeaderObj = self.ch_dataIn[0].processingHeaderObj.copy()
                         self.dataOut.heightList = self.dataOut.processingHeaderObj.heightList
                         self.dataOut.ippSeconds = self.dataOut.processingHeaderObj.ipp
                         self.dataOut.channelList = self.dataOut.processingHeaderObj.channelList
                         self.dataOut.nCohInt = self.dataOut.processingHeaderObj.nCohInt
                         self.dataOut.nFFTPoints = self.dataOut.processingHeaderObj.nFFTPoints
                     if self.flagControllerHeader:
                         self.dataOut.radarControllerHeaderObj = self.ch_dataIn[0].radarControllerHeaderObj.copy()
                         self.dataOut.frequency = self.dataOut.radarControllerHeaderObj.frequency
                     #--------------------------------------------------------------------
                     #--------------------------------------------------------------------
                     if self.flag_spc:
                         if self.dataOut.data_spc.ndim < 3:
                             print("shape spc in: ",self.dataOut.data_spc.shape )
                             return 0
                     if self.flag_pow:
                         if self.dataOut.data_pow.ndim < 2:
                             print("shape pow in: ",self.dataOut.data_pow.shape )
                             return 0
                     if self.flag_snr:
                         if self.dataOut.data_snr.ndim < 2:
                             print("shape snr in: ",self.dataOut.data_snr.shape )
                             return 0
                     self.dataOut.data_spc = None
                     self.dataOut.data_cspc = None
                     self.dataOut.data_pow = None
                     self.dataOut.data_snr = None
                     self.dataOut.utctime = None
                     self.dataOut.nIncohInt = None
                     #--------------------------------------------------------------------
                     if self.flag_spc:
                         spc = [data.data_spc for data in self.ch_dataIn]
                         self.dataOut.data_spc = numpy.stack(spc, axis=1)  #blocks, ch, fft, hei
                     #--------------------------------------------------------------------
                     if self.flag_pow:
                         pow = [data.data_pow for data in self.ch_dataIn]
                         self.dataOut.data_pow = numpy.stack(pow, axis=1)  #blocks, ch, fft, hei
                     #--------------------------------------------------------------------
                     if self.flag_snr:
                         snr = [data.data_snr for data in self.ch_dataIn]
                         self.dataOut.data_snr = numpy.stack(snr, axis=1)  #blocks, ch, fft, hei
                     #--------------------------------------------------------------------
                     time = [data.utctime for data in self.ch_dataIn]
                     time = numpy.asarray(time).mean(axis=0)
                     time = numpy.squeeze(time)
                     self.dataOut.utctime = time
                     #--------------------------------------------------------------------
                     if self.flag_nIcoh:
                         ints = [data.nIncohInt for data in self.ch_dataIn]
                         self.dataOut.nIncohInt = numpy.stack(ints, axis=1)
                         if self.dataOut.nIncohInt.ndim > 3:
                             aux = self.dataOut.nIncohInt
                             self.dataOut.nIncohInt = None
                             self.dataOut.nIncohInt = aux[0]
                         if self.dataOut.nIncohInt.ndim < 3:
                             nIncohInt = numpy.repeat(self.dataOut.nIncohInt, self.dataOut.nHeights).reshape(self.blocksPerFile,self.nChannels, self.dataOut.nHeights)
                             #nIncohInt = numpy.reshape(nIncohInt, (self.blocksPerFile,self.nChannels, self.dataOut.nHeights))
                             self.dataOut.nIncohInt = None
                             self.dataOut.nIncohInt = nIncohInt
                         if (self.dataOut.nIncohInt.shape)[0]==self.nChannels:   ## ch,blocks, hei
                             self.dataOut.nIncohInt =  numpy.swapaxes(self.dataOut.nIncohInt, 0, 1) ## blocks,ch, hei
                     else:
                          self.dataOut.nIncohInt = self.ch_dataIn[0].nIncohInt
                     #--------------------------------------------------------------------
-                    #print("utcTime: ", time.shape)
+                    # print("utcTime: ", time.shape)
-                    #print("data_spc ",self.dataOut.data_spc.shape)
+                    # print("data_spc ",self.dataOut.data_spc.shape)
                     if "data_cspc" in self.dataList:
                         pairsList = [pair for pair in itertools.combinations(self.channelList, 2)]
                         #print("PairsList: ", pairsList)
                         self.dataOut.pairsList = pairsList
                         cspc = []
                         for i, j in pairsList:
                             cspc.append(self.ch_dataIn[i].data_spc*numpy.conjugate(self.ch_dataIn[j].data_spc)) #blocks, fft, hei
                         cspc = numpy.asarray(cspc) # #  pairs, blocks, fft, hei
                         #print("cspc: ",cspc.shape)
                         self.dataOut.data_cspc = numpy.swapaxes(cspc, 0, 1) ## blocks, pairs, fft, hei
                         #print("dataOut.data_cspc: ",self.dataOut.data_cspc.shape)
                     #if "data_pow" in self.dataList:
                     return 1
                 # def writeMetadata(self, fp):
                 #
                 #
                 #     grp = fp.create_group('Metadata')
                 #
                 #     for i in range(len(self.metadataList)):
                 #         if not hasattr(self.dataOut, self.metadataList[i]):
                 #             print('Metadata: `{}` not found'.format(self.metadataList[i]))
                 #             continue
                 #         value = getattr(self.dataOut, self.metadataList[i])
                 #         if isinstance(value, bool):
                 #             if value is True:
                 #                 value = 1
                 #             else:
                 #                 value = 0
                 #         grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
                 #     return
                 def writeMetadata(self, fp):
                     grp = fp.create_group('Metadata')
                     for i in range(len(self.metadataList)):
                         attribute = self.metadataList[i]
                         attr = attribute.split('.')
                         if '' in attr:
                             attr.remove('')
                         #print(attr)
                         if len(attr) > 1:
                             if not hasattr(eval("self.dataOut."+attr[0]),attr[1]):
                                 print('Metadata: {}.{} not found'.format(attr[0],attr[1]))
                                 continue
                             value = getattr(eval("self.dataOut."+attr[0]),attr[1])
                             if isinstance(value, bool):
                                 if value is True:
                                     value = 1
                                 else:
                                     value = 0
                             grp2 = None
                             if not 'Metadata/'+attr[0] in fp:
                                 grp2 = fp.create_group('Metadata/'+attr[0])
                             else:
                                 grp2 = fp['Metadata/'+attr[0]]
                             grp2.create_dataset(attr[1], data=value)
                         else:
                             if not hasattr(self.dataOut, attr[0] ):
                                 print('Metadata: `{}` not found'.format(attribute))
                                 continue
                             value = getattr(self.dataOut, attr[0])
                             if isinstance(value, bool):
                                 if value is True:
                                     value = 1
                                 else:
                                     value = 0
                             if isinstance(value, type(None)):
                                 print("------ERROR, value {} is None".format(attribute))
                             grp.create_dataset(self.getLabel(attribute), data=value)
                     return
                 def getDsList(self):
+                    # print("Getting DS List", self.dataList)
                     dsList =[]
+                    dataAux = None
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         if hasattr(self.dataOut, self.dataList[i]):
                             dataAux = getattr(self.dataOut, self.dataList[i])
                             dsDict['variable'] = self.dataList[i]
                         else:
                             print('Attribute {} not found in dataOut'.format(self.dataList[i]))
                             continue
                         if dataAux is None:
                             continue
-                        elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
+                        elif isinstance(dataAux, (int, float, numpy.int_, numpy.float_)):
                             dsDict['nDim'] = 0
                         else:
                             dsDict['nDim'] = len(dataAux.shape) -1
                             dsDict['shape'] = dataAux.shape
                             if len(dsDict['shape'])>=2:
                                 dsDict['dsNumber'] = dataAux.shape[1]
                             else:
                                 dsDict['dsNumber'] = 1
                             dsDict['dtype'] = dataAux.dtype
                             # if len(dataAux.shape) == 4:
                             #     dsDict['nDim'] = len(dataAux.shape) -1
                             #     dsDict['shape'] = dataAux.shape
                             #     dsDict['dsNumber'] = dataAux.shape[1]
                             #     dsDict['dtype'] = dataAux.dtype
                             # else:
                             #     dsDict['nDim'] = len(dataAux.shape)
                             #     dsDict['shape'] = dataAux.shape
                             #     dsDict['dsNumber'] = dataAux.shape[0]
                             #     dsDict['dtype'] = dataAux.dtype
                         dsList.append(dsDict)
-                    #print(dsList)
+                    # print("dsList: ", dsList)
                     self.dsList = dsList
                 def clean_dataIn(self):
                     for ch in range(self.nChannels):
                         self.ch_dataIn[ch].data_spc = None
                         self.ch_dataIn[ch].utctime = None
                         self.ch_dataIn[ch].nIncohInt = None
                     self.meta ={}
                     self.blocksPerFile = None
                 def writeData(self, outFilename):
                     self.getDsList()
                     fp = h5py.File(outFilename, 'w')
+                    # print("--> Merged file: ",fp)
                     self.writeMetadata(fp)
                     grp = fp.create_group('Data')
                     dtsets = []
                     data = []
                     for dsInfo in self.dsList:
                         if dsInfo['nDim'] == 0:
                             ds = grp.create_dataset(
                                 self.getLabel(dsInfo['variable']),(self.blocksPerFile, ),chunks=True,dtype=numpy.float64)
                             dtsets.append(ds)
                             data.append((dsInfo['variable'], -1))
                         else:
                             label = self.getLabel(dsInfo['variable'])
                             if label is not None:
                                 sgrp = grp.create_group(label)
                             else:
                                 sgrp = grp
                             k = -1*(dsInfo['nDim'] - 1)
-                            #print(k, dsInfo['shape'], dsInfo['shape'][k:])
+                            # print(k, dsInfo['shape'], dsInfo['shape'][k:])
                             for i in range(dsInfo['dsNumber']):
                                 ds = sgrp.create_dataset(
                                     self.getLabel(dsInfo['variable'], i),(self.blocksPerFile, ) + dsInfo['shape'][k:],
                                     chunks=True,
                                     dtype=dsInfo['dtype'])
                                 dtsets.append(ds)
                                 data.append((dsInfo['variable'], i))
                     #print("\n",dtsets)
                     print('Creating merged file: {}'.format(fp.filename))
                     for i, ds in enumerate(dtsets):
                         attr, ch = data[i]
                         if ch == -1:
                             ds[:] = getattr(self.dataOut, attr)
                         else:
                             #print(ds, getattr(self.dataOut, attr)[ch].shape)
                             aux = getattr(self.dataOut, attr)# block, ch, ...
                             aux = numpy.swapaxes(aux,0,1) # ch, blocks, ...
                             #print(ds.shape, aux.shape)
                             #ds[:] = getattr(self.dataOut, attr)[ch]
                             ds[:] = aux[ch]
                     fp.flush()
                     fp.close()
                     self.clean_dataIn()
                     return
                 def run(self):
                     if not(self.isConfig):
                         self.setup()
                         self.isConfig = True
                     for nf in range(self.nfiles):
                         name = None
                         for ch in range(self.nChannels):
                             name = self.filenameList[ch][nf]
                             filename = os.path.join(self.inPaths[ch], name)
                             fp = h5py.File(filename, 'r')
-                            #print("Opening file: ",filename)
+                            print("Opening file: ",filename)
                             self.readFile(fp,ch)
                             fp.close()
                         if self.blocksPerFile == None:
                             self.getBlocksPerFile()
                             print("blocks per file: ", self.blocksPerFile)
                         if not self.getDataOut():
                             print("Error getting DataOut invalid number of blocks")
                             return
                         name = name[-16:]
-                        #print("Final name out: ", name)
+                        # print("Final name out: ", name)
                         outFile = os.path.join(self.pathOut, name)
-                        #print("Outfile: ", outFile)
+                        # print("Outfile: ", outFile)
                         self.writeData(outFile)

schainpy/model/proc/jroproc_spectra.py +4 -3

             # Copyright (c) 2012-2020 Jicamarca Radio Observatory
             # All rights reserved.
             #
             # Distributed under the terms of the BSD 3-clause license.
             """Spectra processing Unit and operations
             Here you will find the processing unit `SpectraProc` and several operations
             to work with Spectra data type
             """
             import time
             import itertools
             import numpy
             from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
             from schainpy.model.data.jrodata import Spectra
             from schainpy.model.data.jrodata import hildebrand_sekhon
             from schainpy.model.data import _noise
             from schainpy.utils import log
             import matplotlib.pyplot as plt
             from schainpy.model.io.utilsIO import getHei_index
             import datetime
             class SpectraProc(ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.buffer = None
                     self.firstdatatime = None
                     self.profIndex = 0
                     self.dataOut = Spectra()
                     self.dataOut.error=False
                     self.id_min = None
                     self.id_max = None
                     self.setupReq = False #Agregar a todas las unidades de proc
                     self.nsamplesFFT = 0
                 def __updateSpecFromVoltage(self):
                     self.dataOut.timeZone = self.dataIn.timeZone
                     self.dataOut.dstFlag = self.dataIn.dstFlag
                     self.dataOut.errorCount = self.dataIn.errorCount
                     self.dataOut.useLocalTime = self.dataIn.useLocalTime
                     try:
                         self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
                     except:
                         pass
                     self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
                     self.dataOut.ippSeconds = self.dataIn.ippSeconds
                     self.dataOut.ipp = self.dataIn.ipp
                     self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
                     self.dataOut.channelList = self.dataIn.channelList
                     self.dataOut.heightList = self.dataIn.heightList
                     self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                     self.dataOut.nProfiles = self.dataOut.nFFTPoints
                     self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
                     self.dataOut.utctime = self.firstdatatime
                     self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
                     self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.nCohInt = self.dataIn.nCohInt
                     self.dataOut.nIncohInt = 1
                     self.dataOut.deltaHeight = self.dataIn.deltaHeight
                     self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
                     self.dataOut.frequency = self.dataIn.frequency
                     self.dataOut.realtime = self.dataIn.realtime
                     self.dataOut.azimuth = self.dataIn.azimuth
                     self.dataOut.zenith = self.dataIn.zenith
                     self.dataOut.codeList = self.dataIn.codeList
                     self.dataOut.azimuthList = self.dataIn.azimuthList
                     self.dataOut.elevationList = self.dataIn.elevationList
                     self.dataOut.code = self.dataIn.code
                     self.dataOut.nCode = self.dataIn.nCode
                     self.dataOut.flagProfilesByRange = self.dataIn.flagProfilesByRange
                     self.dataOut.nProfilesByRange = self.dataIn.nProfilesByRange
                     self.dataOut.runNextUnit = self.dataIn.runNextUnit
                     try:
                         self.dataOut.step = self.dataIn.step
                     except:
                         pass
                 def __getFft(self):
                     """
                     Convierte valores de Voltaje a Spectra
                     Affected:
                         self.dataOut.data_spc
                         self.dataOut.data_cspc
                         self.dataOut.data_dc
                         self.dataOut.heightList
                         self.profIndex
                         self.buffer
                         self.dataOut.flagNoData
                     """
                     fft_volt = numpy.fft.fft(
                         self.buffer, n=self.dataOut.nFFTPoints, axis=1)
                     fft_volt = fft_volt.astype(numpy.dtype('complex'))
                     dc = fft_volt[:, 0, :]
                     # calculo de self-spectra
                     fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
                     spc = fft_volt * numpy.conjugate(fft_volt)
                     spc = spc.real
                     blocksize = 0
                     blocksize += dc.size
                     blocksize += spc.size
                     cspc = None
                     pairIndex = 0
                     if self.dataOut.pairsList != None:
                         # calculo de cross-spectra
                         cspc = numpy.zeros(
                             (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
                         for pair in self.dataOut.pairsList:
                             if pair[0] not in self.dataOut.channelList:
                                 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
                                     str(pair), str(self.dataOut.channelList)))
                             if pair[1] not in self.dataOut.channelList:
                                 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
                                     str(pair), str(self.dataOut.channelList)))
                             cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
                                 numpy.conjugate(fft_volt[pair[1], :, :])
                             pairIndex += 1
                         blocksize += cspc.size
                     self.dataOut.data_spc = spc
                     self.dataOut.data_cspc = cspc
                     self.dataOut.data_dc = dc
                     self.dataOut.blockSize = blocksize
                     self.dataOut.flagShiftFFT = False
                 def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False,
                         zeroPad=False, zeroPoints=0, runNextUnit=0):
                     self.dataIn.runNextUnit = runNextUnit
                     try:
-                        type = self.dataIn.type.decode("utf-8")
+                        _type = self.dataIn.type.decode("utf-8")
-                        self.dataIn.type = type
+                        self.dataIn.type = _type
                     except Exception as e:
-                        # print("spc -> ",e)
+                        #print("spc -> ",self.dataIn.type, e)
                         pass
                     if self.dataIn.type == "Spectra":
                         #print("AQUI")
                         try:
                             self.dataOut.copy(self.dataIn)
                             self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
                             self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
                             self.dataOut.nProfiles = self.dataOut.nFFTPoints
                             #self.dataOut.nHeights = len(self.dataOut.heightList)
                         except Exception as e:
                             print("Error dataIn ",e)
                         if shift_fft:
                             #desplaza a la derecha en el eje 2 determinadas posiciones
                             shift = int(self.dataOut.nFFTPoints/2)
                             self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
                             if self.dataOut.data_cspc is not None:
                                 #desplaza a la derecha en el eje 2 determinadas posiciones
                                 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
                         if pairsList:
                             self.__selectPairs(pairsList)
                     elif self.dataIn.type == "Voltage":
                         self.dataOut.flagNoData = True
                         self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
                         self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
                         if nFFTPoints == None:
                             raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
                         if nProfiles == None:
                             nProfiles = nFFTPoints
                         if ippFactor == None:
                             self.dataOut.ippFactor = self.dataIn.ippFactor
                         else:
                             self.dataOut.ippFactor = ippFactor
                         if self.buffer is None:
                             if not zeroPad:
                                 self.buffer = numpy.zeros((self.dataIn.nChannels,
                                                        nProfiles,
                                                        self.dataIn.nHeights),
                                                       dtype='complex')
                                 zeroPoints = 0
                             else:
                                 self.buffer = numpy.zeros((self.dataIn.nChannels,
                                                        nFFTPoints+int(zeroPoints),
                                                        self.dataIn.nHeights),
                                                       dtype='complex')
                         self.dataOut.nFFTPoints = nFFTPoints
                         if self.buffer is None:
                             self.buffer = numpy.zeros((self.dataIn.nChannels,
                                                        nProfiles,
                                                        self.dataIn.nHeights),
                                                       dtype='complex')
                         if self.dataIn.flagDataAsBlock:
                             nVoltProfiles = self.dataIn.data.shape[1]
                             zeroPoints = 0
                             if nVoltProfiles == nProfiles or zeroPad:
                                 self.buffer = self.dataIn.data.copy()
                                 self.profIndex = nVoltProfiles
                             elif nVoltProfiles < nProfiles:
                                 if self.profIndex == 0:
                                     self.id_min = 0
                                     self.id_max = nVoltProfiles
                                 self.buffer[:, self.id_min:self.id_max,
                                             :] = self.dataIn.data
                                 self.profIndex += nVoltProfiles
                                 self.id_min += nVoltProfiles
                                 self.id_max += nVoltProfiles
                             elif nVoltProfiles > nProfiles:
                                 self.reader.bypass = True
                                 if self.profIndex == 0:
                                     self.id_min = 0
                                     self.id_max = nProfiles
                                 self.buffer = self.dataIn.data[:, self.id_min:self.id_max,:]
                                 self.profIndex += nProfiles
                                 self.id_min += nProfiles
                                 self.id_max += nProfiles
                                 if self.id_max == nVoltProfiles:
                                     self.reader.bypass = False
                             else:
                                 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
                                     self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
                                 self.dataOut.flagNoData = True
                         else:
                             self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
                             self.profIndex += 1
                         if self.firstdatatime == None:
                             self.firstdatatime = self.dataIn.utctime
                         if self.profIndex == nProfiles or (zeroPad and zeroPoints==0):
                             self.__updateSpecFromVoltage()
                             if pairsList == None:
                                 self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)]
                             else:
                                 self.dataOut.pairsList = pairsList
                             self.__getFft()
                             self.dataOut.flagNoData = False
                             self.firstdatatime = None
                             self.nsamplesFFT = self.profIndex
                             #if not self.reader.bypass:
                             self.profIndex = 0
                         #update Processing Header:
                         self.dataOut.processingHeaderObj.dtype = "Spectra"
                         self.dataOut.processingHeaderObj.nFFTPoints = self.dataOut.nFFTPoints
                         self.dataOut.processingHeaderObj.nSamplesFFT = self.nsamplesFFT
                         self.dataOut.processingHeaderObj.nIncohInt = 1
                     elif self.dataIn.type == "Parameters":  #when get data from h5 spc file
                         self.dataOut.data_spc = self.dataIn.data_spc
                         self.dataOut.data_cspc = self.dataIn.data_cspc
                         self.dataOut.data_outlier = self.dataIn.data_outlier
                         self.dataOut.nProfiles = self.dataIn.nProfiles
                         self.dataOut.nIncohInt = self.dataIn.nIncohInt
                         self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
                         self.dataOut.ippFactor = self.dataIn.ippFactor
                         self.dataOut.max_nIncohInt = self.dataIn.max_nIncohInt
                         self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
                         self.dataOut.ProcessingHeader = self.dataIn.ProcessingHeader.copy()
                         self.dataOut.ippSeconds = self.dataIn.ippSeconds
                         self.dataOut.ipp = self.dataIn.ipp
                         #self.dataOut.abscissaList = self.dataIn.getVelRange(1)
                         #self.dataOut.spc_noise = self.dataIn.getNoise()
                         #self.dataOut.spc_range = (self.dataIn.getFreqRange(1) , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1))
                         # self.dataOut.normFactor = self.dataIn.normFactor
                         if hasattr(self.dataIn, 'channelList'):
                             self.dataOut.channelList = self.dataIn.channelList
                         if hasattr(self.dataIn, 'pairsList'):
                             self.dataOut.pairsList = self.dataIn.pairsList
                             self.dataOut.groupList = self.dataIn.pairsList
                         self.dataOut.flagNoData = False
                         if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels
                             self.dataOut.ChanDist = self.dataIn.ChanDist
                         else: self.dataOut.ChanDist = None
                         #if hasattr(self.dataIn, 'VelRange'): #Velocities range
                         #    self.dataOut.VelRange = self.dataIn.VelRange
                         #else: self.dataOut.VelRange = None
                     else:
                         raise ValueError("The type of input object '%s' is not valid".format(
                             self.dataIn.type))
                     # print("SPC done")
                 def __selectPairs(self, pairsList):
                     if not pairsList:
                         return
                     pairs = []
                     pairsIndex = []
                     for pair in pairsList:
                         if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
                             continue
                         pairs.append(pair)
                         pairsIndex.append(pairs.index(pair))
                     self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
                     self.dataOut.pairsList = pairs
                     return
                 def selectFFTs(self, minFFT, maxFFT ):
                     """
                     Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
                     minFFT<= FFT <= maxFFT
                     """
                     if (minFFT > maxFFT):
                         raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
                     if (minFFT < self.dataOut.getFreqRange()[0]):
                         minFFT = self.dataOut.getFreqRange()[0]
                     if (maxFFT > self.dataOut.getFreqRange()[-1]):
                         maxFFT = self.dataOut.getFreqRange()[-1]
                     minIndex = 0
                     maxIndex = 0
                     FFTs = self.dataOut.getFreqRange()
                     inda = numpy.where(FFTs >= minFFT)
                     indb = numpy.where(FFTs <= maxFFT)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(FFTs)
                     self.selectFFTsByIndex(minIndex, maxIndex)
                     return 1
                 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
                     newheis = numpy.where(
                         self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
                     if hei_ref != None:
                         newheis = numpy.where(self.dataOut.heightList > hei_ref)
                     minIndex = min(newheis[0])
                     maxIndex = max(newheis[0])
                     data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
                     heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
                     # determina indices
                     nheis = int(self.dataOut.radarControllerHeaderObj.txB /
                                 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
                     avg_dB = 10 * \
                         numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
                     beacon_dB = numpy.sort(avg_dB)[-nheis:]
                     beacon_heiIndexList = []
                     for val in avg_dB.tolist():
                         if val >= beacon_dB[0]:
                             beacon_heiIndexList.append(avg_dB.tolist().index(val))
                     data_cspc = None
                     if self.dataOut.data_cspc is not None:
                         data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
                     data_dc = None
                     if self.dataOut.data_dc is not None:
                         data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_cspc = data_cspc
                     self.dataOut.data_dc = data_dc
                     self.dataOut.heightList = heightList
                     self.dataOut.beacon_heiIndexList = beacon_heiIndexList
                     return 1
                 def selectFFTsByIndex(self, minIndex, maxIndex):
                     """
                     """
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
                     if (maxIndex >= self.dataOut.nProfiles):
                         maxIndex = self.dataOut.nProfiles-1
                     #Spectra
                     data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
                     data_cspc = None
                     if self.dataOut.data_cspc is not None:
                         data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
                     data_dc = None
                     if self.dataOut.data_dc is not None:
                         data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_cspc = data_cspc
                     self.dataOut.data_dc = data_dc
                     self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
                     self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
                     self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
                     return 1
                 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
                     # validacion de rango
                     if minHei == None:
                         minHei = self.dataOut.heightList[0]
                     if maxHei == None:
                         maxHei = self.dataOut.heightList[-1]
                     if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
                         print('minHei: %.2f is out of the heights range' % (minHei))
                         print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
                         minHei = self.dataOut.heightList[0]
                     if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
                         print('maxHei: %.2f is out of the heights range' % (maxHei))
                         print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
                         maxHei = self.dataOut.heightList[-1]
                     # validacion de velocidades
                     velrange = self.dataOut.getVelRange(1)
                     if minVel == None:
                         minVel = velrange[0]
                     if maxVel == None:
                         maxVel = velrange[-1]
                     if (minVel < velrange[0]) or (minVel > maxVel):
                         print('minVel: %.2f is out of the velocity range' % (minVel))
                         print('minVel is setting to %.2f' % (velrange[0]))
                         minVel = velrange[0]
                     if (maxVel > velrange[-1]) or (maxVel < minVel):
                         print('maxVel: %.2f is out of the velocity range' % (maxVel))
                         print('maxVel is setting to %.2f' % (velrange[-1]))
                         maxVel = velrange[-1]
                     # seleccion de indices para rango
                     minIndex = 0
                     maxIndex = 0
                     heights = self.dataOut.heightList
                     inda = numpy.where(heights >= minHei)
                     indb = numpy.where(heights <= maxHei)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(heights)
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError("some value in (%d,%d) is not valid" % (
                             minIndex, maxIndex))
                     if (maxIndex >= self.dataOut.nHeights):
                         maxIndex = self.dataOut.nHeights - 1
                     # seleccion de indices para velocidades
                     indminvel = numpy.where(velrange >= minVel)
                     indmaxvel = numpy.where(velrange <= maxVel)
                     try:
                         minIndexVel = indminvel[0][0]
                     except:
                         minIndexVel = 0
                     try:
                         maxIndexVel = indmaxvel[0][-1]
                     except:
                         maxIndexVel = len(velrange)
                     # seleccion del espectro
                     data_spc = self.dataOut.data_spc[:,
                                                      minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
                     # estimacion de ruido
                     noise = numpy.zeros(self.dataOut.nChannels)
                     for channel in range(self.dataOut.nChannels):
                         daux = data_spc[channel, :, :]
                         sortdata = numpy.sort(daux, axis=None)
                         noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt)
                     self.dataOut.noise_estimation = noise.copy()
                     return 1
             class GetSNR(Operation):
                 '''
                 Written by R. Flores
                 '''
                 """Operation to get SNR.
                 Parameters:
                 -----------
                 Example
                 --------
                 op = proc_unit.addOperation(name='GetSNR', optype='other')
                 """
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                 def run(self,dataOut):
                     noise = dataOut.getNoise(ymin_index=-10) #Región superior donde solo debería de haber ruido
                     dataOut.data_snr = (dataOut.data_spc.sum(axis=1)-noise[:,None]*dataOut.nFFTPoints)/(noise[:,None]*dataOut.nFFTPoints) #It works apparently
                     dataOut.snl = numpy.log10(dataOut.data_snr)
                     dataOut.snl = numpy.where(dataOut.data_snr<.01, numpy.nan, dataOut.snl)
                     return dataOut
             class removeDC(Operation):
                 def run(self, dataOut, mode=2):
                     self.dataOut = dataOut
                     jspectra = self.dataOut.data_spc
                     jcspectra = self.dataOut.data_cspc
                     num_chan = jspectra.shape[0]
                     num_hei = jspectra.shape[2]
                     if jcspectra is not None:
                         jcspectraExist = True
                         num_pairs = jcspectra.shape[0]
                     else:
                         jcspectraExist = False
                     freq_dc = int(jspectra.shape[1] / 2)
                     ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
                     ind_vel = ind_vel.astype(int)
                     if ind_vel[0] < 0:
                         ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
                     if mode == 1:
                         jspectra[:, freq_dc, :] = (
                             jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2  # CORRECCION
                         if jcspectraExist:
                             jcspectra[:, freq_dc, :] = (
                                 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
                     if mode == 2:
                         vel = numpy.array([-2, -1, 1, 2])
                         xx = numpy.zeros([4, 4])
                         for fil in range(4):
                             xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
                         xx_inv = numpy.linalg.inv(xx)
                         xx_aux = xx_inv[0, :]
                         for ich in range(num_chan):
                             yy = jspectra[ich, ind_vel, :]
                             jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
                             junkid = jspectra[ich, freq_dc, :] <= 0
                             cjunkid = sum(junkid)
                             if cjunkid.any():
                                 jspectra[ich, freq_dc, junkid.nonzero()] = (
                                     jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
                         if jcspectraExist:
                             for ip in range(num_pairs):
                                 yy = jcspectra[ip, ind_vel, :]
                                 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
                     self.dataOut.data_spc = jspectra
                     self.dataOut.data_cspc = jcspectra
                     return self.dataOut
             class getNoiseB(Operation):
                 """
                 Get noise from custom heights and frequency ranges,
                 offset for additional manual correction
                 J. Apaza -> developed to amisr isr spectra
                 """
                 __slots__ =('offset','warnings', 'isConfig', 'minIndex','maxIndex','minIndexFFT','maxIndexFFT')
                 def __init__(self):
                     Operation.__init__(self)
                     self.isConfig = False
                 def setup(self, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
                     self.warnings = warnings
                     if minHei == None:
                         minHei = self.dataOut.heightList[0]
                     if maxHei == None:
                         maxHei = self.dataOut.heightList[-1]
                     if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
                         if self.warnings:
                             print('minHei: %.2f is out of the heights range' % (minHei))
                             print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
                         minHei = self.dataOut.heightList[0]
                     if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
                         if self.warnings:
                             print('maxHei: %.2f is out of the heights range' % (maxHei))
                             print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
                         maxHei = self.dataOut.heightList[-1]
                     #indices relativos a los puntos de fft, puede ser de acuerdo a velocidad o frecuencia
                     minIndexFFT = 0
                     maxIndexFFT = 0
                     # validacion de velocidades
                     indminPoint = None
                     indmaxPoint = None
                     if self.dataOut.type == 'Spectra':
                         if minVel == None and maxVel == None :
                             freqrange = self.dataOut.getFreqRange(1)
                             if minFreq == None:
                                 minFreq = freqrange[0]
                             if maxFreq == None:
                                 maxFreq = freqrange[-1]
                             if (minFreq < freqrange[0]) or (minFreq > maxFreq):
                                 if self.warnings:
                                     print('minFreq: %.2f is out of the frequency range' % (minFreq))
                                     print('minFreq is setting to %.2f' % (freqrange[0]))
                                 minFreq = freqrange[0]
                             if (maxFreq > freqrange[-1]) or (maxFreq < minFreq):
                                 if self.warnings:
                                     print('maxFreq: %.2f is out of the frequency range' % (maxFreq))
                                     print('maxFreq is setting to %.2f' % (freqrange[-1]))
                                 maxFreq = freqrange[-1]
                             indminPoint = numpy.where(freqrange >= minFreq)
                             indmaxPoint = numpy.where(freqrange <= maxFreq)
                         else:
                             velrange = self.dataOut.getVelRange(1)
                             if minVel == None:
                                 minVel = velrange[0]
                             if maxVel == None:
                                 maxVel = velrange[-1]
                             if (minVel < velrange[0]) or (minVel > maxVel):
                                 if self.warnings:
                                     print('minVel: %.2f is out of the velocity range' % (minVel))
                                     print('minVel is setting to %.2f' % (velrange[0]))
                                 minVel = velrange[0]
                             if (maxVel > velrange[-1]) or (maxVel < minVel):
                                 if self.warnings:
                                     print('maxVel: %.2f is out of the velocity range' % (maxVel))
                                     print('maxVel is setting to %.2f' % (velrange[-1]))
                                 maxVel = velrange[-1]
                             indminPoint = numpy.where(velrange >= minVel)
                             indmaxPoint = numpy.where(velrange <= maxVel)
                     # seleccion de indices para rango  REEMPLAZAR FOR FUNCION EXTERNA LUEGO
                     # minIndex = 0
                     # maxIndex = 0
                     # heights = self.dataOut.heightList
                     # inda = numpy.where(heights >= minHei)
                     # indb = numpy.where(heights <= maxHei)
                     # try:
                     #     minIndex = inda[0][0]
                     # except:
                     #     minIndex = 0
                     # try:
                     #     maxIndex = indb[0][-1]
                     # except:
                     #     maxIndex = len(heights)
                     # if (minIndex < 0) or (minIndex > maxIndex):
                     #     raise ValueError("some value in (%d,%d) is not valid" % (
                     #         minIndex, maxIndex))
                     # if (maxIndex >= self.dataOut.nHeights):
                     #     maxIndex = self.dataOut.nHeights - 1
                     minIndex, maxIndex = getHei_index(minHei,maxHei,self.dataOut.heightList)
                     #############################################################3
                     # seleccion de indices para velocidades
                     if self.dataOut.type == 'Spectra':
                         try:
                             minIndexFFT = indminPoint[0][0]
                         except:
                             minIndexFFT = 0
                         try:
                             maxIndexFFT = indmaxPoint[0][-1]
                         except:
                             maxIndexFFT = len( self.dataOut.getFreqRange(1))
                     self.minIndex, self.maxIndex, self.minIndexFFT, self.maxIndexFFT = minIndex, maxIndex, minIndexFFT, maxIndexFFT
                     self.isConfig = True
                     self.offset = 1
                     if offset!=None:
                         self.offset = 10**(offset/10)
                 def run(self, dataOut, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
                     self.dataOut = dataOut
                     if not self.isConfig:
                         self.setup(offset, minHei, maxHei,minVel, maxVel, minFreq, maxFreq, warnings)
                     self.dataOut.noise_estimation = None
                     noise = None
                     if self.dataOut.type == 'Voltage':
                         noise = self.dataOut.getNoise(ymin_index=self.minIndex, ymax_index=self.maxIndex)
                     elif self.dataOut.type == 'Spectra':
                         noise = numpy.zeros( self.dataOut.nChannels)
                         norm = 1
                         for channel in range( self.dataOut.nChannels):
                             if not hasattr(self.dataOut.nIncohInt,'__len__'):
                                 norm = 1
                             else:
                                 norm = self.dataOut.max_nIncohInt[channel]/self.dataOut.nIncohInt[channel, self.minIndex:self.maxIndex]
                             daux =  self.dataOut.data_spc[channel,self.minIndexFFT:self.maxIndexFFT, self.minIndex:self.maxIndex]
                             daux = numpy.multiply(daux, norm)
                             sortdata = numpy.sort(daux, axis=None)
                             noise[channel] = _noise.hildebrand_sekhon(sortdata, self.dataOut.max_nIncohInt[channel])/self.offset
                     else:
                         noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex)
                     self.dataOut.noise_estimation = noise.copy() # dataOut.noise
                     return self.dataOut
                 def getNoiseByMean(self,data):
                     #data debe estar ordenado
                     data = numpy.mean(data,axis=1)
                     sortdata = numpy.sort(data, axis=None)
                     pnoise = None
                     j = 0
                     mean = numpy.mean(sortdata)
                     min = numpy.min(sortdata)
                     delta = mean - min
                     indexes = numpy.where(sortdata > (mean+delta))[0] #only array of indexes
                     #print(len(indexes))
                     if len(indexes)==0:
                         pnoise = numpy.mean(sortdata)
                     else:
                         j = indexes[0]
                         pnoise = numpy.mean(sortdata[0:j])
                     return pnoise
                 def getNoiseByHS(self,data, navg):
                     #data debe estar ordenado
                     #data = numpy.mean(data,axis=1)
                     sortdata = numpy.sort(data, axis=None)
                     lenOfData = len(sortdata)
                     nums_min = lenOfData*0.2
                     if nums_min <= 5:
                         nums_min = 5
                     sump = 0.
                     sumq = 0.
                     j = 0
                     cont = 1
                     while((cont == 1)and(j < lenOfData)):
                         sump += sortdata[j]
                         sumq += sortdata[j]**2
                         #sumq -= sump**2
                         if j > nums_min:
                             rtest = float(j)/(j-1) + 1.0/navg
                             #if ((sumq*j) > (sump**2)):
                             if ((sumq*j) > (rtest*sump**2)):
                                 j = j - 1
                                 sump = sump - sortdata[j]
                                 sumq = sumq - sortdata[j]**2
                                 cont = 0
                         j += 1
                     lnoise = sump / j
                     return lnoise
             class removeInterference(Operation):
                 def removeInterference2(self):
                     cspc = self.dataOut.data_cspc
                     spc = self.dataOut.data_spc
                     Heights = numpy.arange(cspc.shape[2])
                     realCspc = numpy.abs(cspc)
                     for i in range(cspc.shape[0]):
                         LinePower= numpy.sum(realCspc[i], axis=0)
                         Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
                         SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
                         InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
                         InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
                         InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
                         InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
                         #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
                         if len(InterferenceRange)<int(cspc.shape[1]*0.3):
                             cspc[i,InterferenceRange,:] = numpy.NaN
                     self.dataOut.data_cspc = cspc
                 def removeInterference(self, interf=2, hei_interf=None, nhei_interf=None, offhei_interf=None):
                     jspectra = self.dataOut.data_spc
                     jcspectra = self.dataOut.data_cspc
                     jnoise = self.dataOut.getNoise()
                     num_incoh = self.dataOut.nIncohInt
                     num_channel = jspectra.shape[0]
                     num_prof = jspectra.shape[1]
                     num_hei = jspectra.shape[2]
                     # hei_interf
                     if hei_interf is None:
                         count_hei = int(num_hei / 2)
                         hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
                         hei_interf = numpy.asarray(hei_interf)[0]
                     # nhei_interf
                     if (nhei_interf == None):
                         nhei_interf = 5
                     if (nhei_interf < 1):
                         nhei_interf = 1
                     if (nhei_interf > count_hei):
                         nhei_interf = count_hei
                     if (offhei_interf == None):
                         offhei_interf = 0
                     ind_hei = list(range(num_hei))
             #         mask_prof = numpy.asarray(range(num_prof - 2)) + 1
             #         mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
                     mask_prof = numpy.asarray(list(range(num_prof)))
                     num_mask_prof = mask_prof.size
                     comp_mask_prof = [0, num_prof / 2]
                     # noise_exist:    Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
                     if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
                         jnoise = numpy.nan
                     noise_exist = jnoise[0] < numpy.Inf
                     # Subrutina de Remocion de la Interferencia
                     for ich in range(num_channel):
                         # Se ordena los espectros segun su potencia (menor a mayor)
                         power = jspectra[ich, mask_prof, :]
                         power = power[:, hei_interf]
                         power = power.sum(axis=0)
                         psort = power.ravel().argsort()
                         # Se estima la interferencia promedio en los Espectros de Potencia empleando
                         junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
                             offhei_interf, nhei_interf + offhei_interf))]]]
                         if noise_exist:
                             #    tmp_noise = jnoise[ich] / num_prof
                             tmp_noise = jnoise[ich]
                         junkspc_interf = junkspc_interf - tmp_noise
                         #junkspc_interf[:,comp_mask_prof] = 0
                         jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
                         jspc_interf = jspc_interf.transpose()
                         # Calculando el espectro de interferencia promedio
                         noiseid = numpy.where(
                             jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
                         noiseid = noiseid[0]
                         cnoiseid = noiseid.size
                         interfid = numpy.where(
                             jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
                         interfid = interfid[0]
                         cinterfid = interfid.size
                         if (cnoiseid > 0):
                             jspc_interf[noiseid] = 0
                         # Expandiendo los perfiles a limpiar
                         if (cinterfid > 0):
                             new_interfid = (
                                 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
                             new_interfid = numpy.asarray(new_interfid)
                             new_interfid = {x for x in new_interfid}
                             new_interfid = numpy.array(list(new_interfid))
                             new_cinterfid = new_interfid.size
                         else:
                             new_cinterfid = 0
                         for ip in range(new_cinterfid):
                             ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
                             jspc_interf[new_interfid[ip]
                                         ] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
                         jspectra[ich, :, ind_hei] = jspectra[ich, :,
                                                              ind_hei] - jspc_interf  # Corregir indices
                         # Removiendo la interferencia del punto de mayor interferencia
                         ListAux = jspc_interf[mask_prof].tolist()
                         maxid = ListAux.index(max(ListAux))
                         if cinterfid > 0:
                             for ip in range(cinterfid * (interf == 2) - 1):
                                 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
                                        (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
                                 cind = len(ind)
                                 if (cind > 0):
                                     jspectra[ich, interfid[ip], ind] = tmp_noise * \
                                         (1 + (numpy.random.uniform(cind) - 0.5) /
                                          numpy.sqrt(num_incoh))
                             ind = numpy.array([-2, -1, 1, 2])
                             xx = numpy.zeros([4, 4])
                             for id1 in range(4):
                                 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
                             xx_inv = numpy.linalg.inv(xx)
                             xx = xx_inv[:, 0]
                             ind = (ind + maxid + num_mask_prof) % num_mask_prof
                             yy = jspectra[ich, mask_prof[ind], :]
                             jspectra[ich, mask_prof[maxid], :] = numpy.dot(
                                 yy.transpose(), xx)
                         indAux = (jspectra[ich, :, :] < tmp_noise *
                                   (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
                         jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
                             (1 - 1 / numpy.sqrt(num_incoh))
                     # Remocion de Interferencia en el Cross Spectra
                     if jcspectra is None:
                         return jspectra, jcspectra
                     num_pairs = int(jcspectra.size / (num_prof * num_hei))
                     jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
                     for ip in range(num_pairs):
                         #-------------------------------------------
                         cspower = numpy.abs(jcspectra[ip, mask_prof, :])
                         cspower = cspower[:, hei_interf]
                         cspower = cspower.sum(axis=0)
                         cspsort = cspower.ravel().argsort()
                         junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
                             offhei_interf, nhei_interf + offhei_interf))]]]
                         junkcspc_interf = junkcspc_interf.transpose()
                         jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
                         ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
                         median_real = int(numpy.median(numpy.real(
                             junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
                         median_imag = int(numpy.median(numpy.imag(
                             junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
                         comp_mask_prof = [int(e) for e in comp_mask_prof]
                         junkcspc_interf[comp_mask_prof, :] = numpy.complex_(
                             median_real, median_imag)
                         for iprof in range(num_prof):
                             ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
                             jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
                         # Removiendo la Interferencia
                         jcspectra[ip, :, ind_hei] = jcspectra[ip,
                                                               :, ind_hei] - jcspc_interf
                         ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
                         maxid = ListAux.index(max(ListAux))
                         ind = numpy.array([-2, -1, 1, 2])
                         xx = numpy.zeros([4, 4])
                         for id1 in range(4):
                             xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
                         xx_inv = numpy.linalg.inv(xx)
                         xx = xx_inv[:, 0]
                         ind = (ind + maxid + num_mask_prof) % num_mask_prof
                         yy = jcspectra[ip, mask_prof[ind], :]
                         jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
                     # Guardar Resultados
                     self.dataOut.data_spc = jspectra
                     self.dataOut.data_cspc = jcspectra
                     return 1
                 def run(self, dataOut, interf=2,hei_interf=None, nhei_interf=None, offhei_interf=None, mode=1):
                     self.dataOut = dataOut
                     if mode == 1:
                         self.removeInterference(interf=2,hei_interf=None, nhei_interf=None, offhei_interf=None)
                     elif mode == 2:
                         self.removeInterference2()
                     return self.dataOut
             class deflip(Operation):
                 def run(self, dataOut):
                     # arreglo 1: (num_chan, num_profiles, num_heights)
                     self.dataOut = dataOut
                     # JULIA-oblicua, indice 2
                     # arreglo 2: (num_profiles, num_heights)
                     jspectra = self.dataOut.data_spc[2]
                     jspectra_tmp=numpy.zeros(jspectra.shape)
                     num_profiles=jspectra.shape[0]
                     freq_dc = int(num_profiles / 2)
                     # Flip con for
                     for j in range(num_profiles):
                      jspectra_tmp[num_profiles-j-1]= jspectra[j]
                     # Intercambio perfil de DC con perfil inmediato anterior
                     jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1]
                     jspectra_tmp[freq_dc]= jspectra[freq_dc]
                     # canal modificado es re-escrito en el arreglo de canales
                     self.dataOut.data_spc[2] = jspectra_tmp
                     return self.dataOut
             class IncohInt(Operation):
                 __profIndex = 0
                 __withOverapping = False
                 __byTime = False
                 __initime = None
                 __lastdatatime = None
                 __integrationtime = None
                 __buffer_spc = None
                 __buffer_cspc = None
                 __buffer_dc = None
                 __dataReady = False
                 __timeInterval = None
                 incohInt = 0
                 nOutliers = 0
                 n = None
                 _flagProfilesByRange = False
                 _nProfilesByRange = 0
                 def __init__(self):
                     Operation.__init__(self)
                 def setup(self, n=None, timeInterval=None, overlapping=False):
                     """
                     Set the parameters of the integration class.
                     Inputs:
                         n        :    Number of coherent integrations
                         timeInterval   :    Time of integration. If the parameter "n" is selected this one does not work
                         overlapping    :
                     """
                     self.__initime = None
                     self.__lastdatatime = 0
                     self.__buffer_spc = 0
                     self.__buffer_cspc = 0
                     self.__buffer_dc = 0
                     self.__profIndex = 0
                     self.__dataReady = False
                     self.__byTime = False
                     self.incohInt = 0
                     self.nOutliers = 0
                     if n is None and timeInterval is None:
                         raise ValueError("n or timeInterval should be specified ...")
                     if n is not None:
                         self.n = int(n)
                     else:
                         self.__integrationtime = int(timeInterval)
                         self.n = None
                         self.__byTime = True
                 def putData(self, data_spc, data_cspc, data_dc):
                     """
                     Add a profile to the __buffer_spc and increase in one the __profileIndex
                     """
                     if data_spc.all() == numpy.nan :
                         print("nan ")
                         return
                     self.__buffer_spc += data_spc
                     if data_cspc is None:
                         self.__buffer_cspc = None
                     else:
                         self.__buffer_cspc += data_cspc
                     if data_dc is None:
                         self.__buffer_dc = None
                     else:
                         self.__buffer_dc += data_dc
                     self.__profIndex += 1
                     return
                 def pushData(self):
                     """
                     Return the sum of the last profiles and the profiles used in the sum.
                     Affected:
                     self.__profileIndex
                     """
                     data_spc = self.__buffer_spc
                     data_cspc = self.__buffer_cspc
                     data_dc = self.__buffer_dc
                     n = self.__profIndex
                     self.__buffer_spc = 0
                     self.__buffer_cspc = 0
                     self.__buffer_dc = 0
                     return data_spc, data_cspc, data_dc, n
                 def byProfiles(self, *args):
                     self.__dataReady = False
                     avgdata_spc = None
                     avgdata_cspc = None
                     avgdata_dc = None
                     self.putData(*args)
                     if self.__profIndex == self.n:
                         avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata_spc, avgdata_cspc, avgdata_dc
                 def byTime(self, datatime, *args):
                     self.__dataReady = False
                     avgdata_spc = None
                     avgdata_cspc = None
                     avgdata_dc = None
                     self.putData(*args)
                     if (datatime - self.__initime) >= self.__integrationtime:
                         avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata_spc, avgdata_cspc, avgdata_dc
                 def integrate(self, datatime, *args):
                     if self.__profIndex == 0:
                         self.__initime = datatime
                     if self.__byTime:
                         avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
                             datatime, *args)
                     else:
                         avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
                     if not self.__dataReady:
                         return None, None, None, None
                     return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
                 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
                     if n == 1:
                         return dataOut
                     if dataOut.flagNoData == True:
                         return dataOut
                     if dataOut.flagProfilesByRange == True:
                         self._flagProfilesByRange = True
                     dataOut.flagNoData = True
                     dataOut.processingHeaderObj.timeIncohInt = timeInterval
                     if not self.isConfig:
                         self._nProfilesByRange = numpy.zeros((1,len(dataOut.heightList)))
                         self.setup(n, timeInterval, overlapping)
                         self.isConfig = True
                     avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
                                                                                         dataOut.data_spc,
                                                                                         dataOut.data_cspc,
                                                                                         dataOut.data_dc)
                     self.incohInt += dataOut.nIncohInt
                     if isinstance(dataOut.data_outlier,numpy.ndarray) or isinstance(dataOut.data_outlier,int) or isinstance(dataOut.data_outlier, float):
                         self.nOutliers += dataOut.data_outlier
                     if self._flagProfilesByRange:
                         dataOut.flagProfilesByRange = True
                         self._nProfilesByRange += dataOut.nProfilesByRange
                     if self.__dataReady:
                         #print("prof: ",dataOut.max_nIncohInt,self.__profIndex)
                         dataOut.data_spc = avgdata_spc
                         dataOut.data_cspc = avgdata_cspc
                         dataOut.data_dc = avgdata_dc
                         dataOut.nIncohInt = self.incohInt
                         dataOut.data_outlier = self.nOutliers
                         dataOut.utctime = avgdatatime
                         dataOut.flagNoData = False
                         self.incohInt = 0
                         self.nOutliers = 0
                         self.__profIndex = 0
                         dataOut.nProfilesByRange = self._nProfilesByRange
                         self._nProfilesByRange = numpy.zeros((1,len(dataOut.heightList)))
                         self._flagProfilesByRange = False
                         # print("IncohInt Done")
                     return dataOut
             class IntegrationFaradaySpectra(Operation):
                 __profIndex = 0
                 __withOverapping = False
                 __byTime = False
                 __initime = None
                 __lastdatatime = None
                 __integrationtime = None
                 __buffer_spc = None
                 __buffer_cspc = None
                 __buffer_dc = None
                 __dataReady = False
                 __timeInterval = None
                 n_ints = None #matriz de numero de integracions (CH,HEI)
                 n = None
                 minHei_ind = None
                 maxHei_ind = None
                 navg = 1.0
                 factor = 0.0
                 dataoutliers = None # (CHANNELS, HEIGHTS)
                 _flagProfilesByRange = False
                 _nProfilesByRange = 0
                 def __init__(self):
                     Operation.__init__(self)
                 def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None, minHei=None, maxHei=None, avg=1,factor=0.75):
                     """
                     Set the parameters of the integration class.
                     Inputs:
                         n        :    Number of coherent integrations
                         timeInterval   :    Time of integration. If the parameter "n" is selected this one does not work
                         overlapping    :
                     """
                     self.__initime = None
                     self.__lastdatatime = 0
                     self.__buffer_spc = []
                     self.__buffer_cspc = []
                     self.__buffer_dc = 0
                     self.__profIndex = 0
                     self.__dataReady = False
                     self.__byTime = False
                     self.factor = factor
                     self.navg = avg
                     #self.ByLags = dataOut.ByLags   ###REDEFINIR
                     self.ByLags = False
                     self.maxProfilesInt = 0
                     self.__nChannels = dataOut.nChannels
                     if DPL != None:
                         self.DPL=DPL
                     else:
                         #self.DPL=dataOut.DPL    ###REDEFINIR
                         self.DPL=0
                     if n is None and timeInterval is None:
                         raise ValueError("n or timeInterval should be specified ...")
                     if n is not None:
                         self.n = int(n)
                     else:
                         self.__integrationtime = int(timeInterval)
                         self.n = None
                         self.__byTime = True
                     if minHei == None:
                         minHei = self.dataOut.heightList[0]
                     if maxHei == None:
                         maxHei = self.dataOut.heightList[-1]
                     if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
                         print('minHei: %.2f is out of the heights range' % (minHei))
                         print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
                         minHei = self.dataOut.heightList[0]
                     if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
                         print('maxHei: %.2f is out of the heights range' % (maxHei))
                         print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
                         maxHei = self.dataOut.heightList[-1]
                     ind_list1 = numpy.where(self.dataOut.heightList >= minHei)
                     ind_list2 = numpy.where(self.dataOut.heightList <= maxHei)
                     self.minHei_ind = ind_list1[0][0]
                     self.maxHei_ind = ind_list2[0][-1]
                 def putData(self, data_spc, data_cspc, data_dc):
                     """
                     Add a profile to the __buffer_spc and increase in one the __profileIndex
                     """
                     self.__buffer_spc.append(data_spc)
                     if self.__nChannels < 2:
                         self.__buffer_cspc = None
                     else:
                         self.__buffer_cspc.append(data_cspc)
                     if data_dc is None:
                         self.__buffer_dc = None
                     else:
                         self.__buffer_dc += data_dc
                     self.__profIndex += 1
                     return
                 def hildebrand_sekhon_Integration(self,sortdata,navg, factor):
                     #data debe estar ordenado
                     #sortdata = numpy.sort(data, axis=None)
                     #sortID=data.argsort()
                     lenOfData = len(sortdata)
                     nums_min = lenOfData*factor
                     if nums_min <= 5:
                         nums_min = 5
                     sump = 0.
                     sumq = 0.
                     j = 0
                     cont = 1
                     while((cont == 1)and(j < lenOfData)):
                         sump += sortdata[j]
                         sumq += sortdata[j]**2
                         if j > nums_min:
                             rtest = float(j)/(j-1) + 1.0/navg
                             if ((sumq*j) > (rtest*sump**2)):
                                 j = j - 1
                                 sump = sump - sortdata[j]
                                 sumq = sumq - sortdata[j]**2
                                 cont = 0
                         j += 1
                     #lnoise = sump / j
                     #print("H S done")
                     #return j,sortID
                     return j
                 def pushData(self):
                     """
                     Return the sum of the last profiles and the profiles used in the sum.
                     Affected:
                     self.__profileIndex
                     """
                     bufferH=None
                     buffer=None
                     buffer1=None
                     buffer_cspc=None
                     #print("aes: ", self.__buffer_cspc)
                     self.__buffer_spc=numpy.array(self.__buffer_spc)
                     if self.__nChannels > 1 :
                         self.__buffer_cspc=numpy.array(self.__buffer_cspc)
                     #print("FREQ_DC",self.__buffer_spc.shape,self.__buffer_cspc.shape)
                     freq_dc = int(self.__buffer_spc.shape[2] / 2)
                     #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights)
                     self.dataOutliers = numpy.zeros((self.nChannels,self.nHeights)) # --> almacen de outliers
                     for k in range(self.minHei_ind,self.maxHei_ind):
                         if self.__nChannels > 1:
                             buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k])
                         outliers_IDs_cspc=[]
                         cspc_outliers_exist=False
                         for i in range(self.nChannels):#dataOut.nChannels):
                             buffer1=numpy.copy(self.__buffer_spc[:,i,:,k])
                             indexes=[]
                             #sortIDs=[]
                             outliers_IDs=[]
                             for j in range(self.nProfiles): #frecuencias en el tiempo
                                 # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0
                                 #     continue
                                 # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1
                                 #     continue
                                 buffer=buffer1[:,j]
                                 sortdata = numpy.sort(buffer, axis=None)
                                 sortID=buffer.argsort()
                                 index = _noise.hildebrand_sekhon2(sortdata,self.navg)
                                 #index,sortID=self.hildebrand_sekhon_Integration(buffer,1,self.factor)
                                 # fig,ax = plt.subplots()
                                 # ax.set_title(str(k)+" "+str(j))
                                 # x=range(len(sortdata))
                                 # ax.scatter(x,sortdata)
                                 # ax.axvline(index)
                                 # plt.show()
                                 indexes.append(index)
                                 #sortIDs.append(sortID)
                                 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
                             #print("Outliers: ",outliers_IDs)
                             outliers_IDs=numpy.array(outliers_IDs)
                             outliers_IDs=outliers_IDs.ravel()
                             outliers_IDs=numpy.unique(outliers_IDs)
                             outliers_IDs=outliers_IDs.astype(numpy.dtype('int64'))
                             indexes=numpy.array(indexes)
                             indexmin=numpy.min(indexes)
                             #print(indexmin,buffer1.shape[0], k)
                             # fig,ax = plt.subplots()
                             # ax.plot(sortdata)
                             # ax2 = ax.twinx()
                             # x=range(len(indexes))
                             # #plt.scatter(x,indexes)
                             # ax2.scatter(x,indexes)
                             # plt.show()
                             if indexmin != buffer1.shape[0]:
                                 if self.__nChannels > 1:
                                     cspc_outliers_exist= True
                                 lt=outliers_IDs
                                 #avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0)
                                 for p in list(outliers_IDs):
                                     #buffer1[p,:]=avg
                                     buffer1[p,:] = numpy.NaN
                                 self.dataOutliers[i,k] = len(outliers_IDs)
                             self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1)
                             if self.__nChannels > 1:
                                 outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs)
                         if self.__nChannels > 1:
                             outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64'))
                         if cspc_outliers_exist:
                             lt=outliers_IDs_cspc
                             #avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0)
                             for p in list(outliers_IDs_cspc):
                                 #buffer_cspc[p,:]=avg
                                 buffer_cspc[p,:] = numpy.NaN
                         if self.__nChannels > 1:
                             self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc)
                     nOutliers = len(outliers_IDs)
                     #print("Outliers  n: ",self.dataOutliers,nOutliers)
                     buffer=None
                     bufferH=None
                     buffer1=None
                     buffer_cspc=None
                     buffer=None
                     #data_spc = numpy.sum(self.__buffer_spc,axis=0)
                     data_spc = numpy.nansum(self.__buffer_spc,axis=0)
                     if self.__nChannels > 1:
                         #data_cspc = numpy.sum(self.__buffer_cspc,axis=0)
                         data_cspc = numpy.nansum(self.__buffer_cspc,axis=0)
                     else:
                         data_cspc = None
                     data_dc = self.__buffer_dc
                     #(CH, HEIGH)
                     self.maxProfilesInt = self.__profIndex - 1
                     n = self.__profIndex - self.dataOutliers # n becomes a matrix
                     self.__buffer_spc = []
                     self.__buffer_cspc = []
                     self.__buffer_dc = 0
                     self.__profIndex = 0
                     #print("cleaned ",data_cspc)
                     return data_spc, data_cspc, data_dc, n
                 def byProfiles(self, *args):
                     self.__dataReady = False
                     avgdata_spc = None
                     avgdata_cspc = None
                     avgdata_dc = None
                     self.putData(*args)
                     if self.__profIndex >= self.n:
                         avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
                         self.n_ints = n
                         self.__dataReady = True
                     return avgdata_spc, avgdata_cspc, avgdata_dc
                 def byTime(self, datatime, *args):
                     self.__dataReady = False
                     avgdata_spc = None
                     avgdata_cspc = None
                     avgdata_dc = None
                     self.putData(*args)
                     if (datatime - self.__initime) >= self.__integrationtime:
                         avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
                         self.n_ints = n
                         self.__dataReady = True
                     return avgdata_spc, avgdata_cspc, avgdata_dc
                 def integrate(self, datatime, *args):
                     if self.__profIndex == 0:
                         self.__initime = datatime
                     if self.__byTime:
                         avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
                             datatime, *args)
                     else:
                         avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
                     if not self.__dataReady:
                         return None, None, None, None
                     #print("integrate", avgdata_cspc)
                     return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
                 def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False, minHei=None, maxHei=None, avg=1, factor=0.75):
                     self.dataOut = dataOut
                     if n == 1:
                         return self.dataOut
                     self.dataOut.processingHeaderObj.timeIncohInt = timeInterval
                     if dataOut.flagProfilesByRange:
                         self._flagProfilesByRange = True
                     if self.dataOut.nChannels == 1:
                         self.dataOut.data_cspc = None #si es un solo canal no vale la pena acumular DATOS
                     #print("IN spc:", self.dataOut.data_spc.shape, self.dataOut.data_cspc)
                     if not self.isConfig:
                         self.setup(self.dataOut, n, timeInterval, overlapping,DPL ,minHei, maxHei, avg, factor)
                         self.isConfig = True
                     if not self.ByLags:
                         self.nProfiles=self.dataOut.nProfiles
                         self.nChannels=self.dataOut.nChannels
                         self.nHeights=self.dataOut.nHeights
                         avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
                                                                                             self.dataOut.data_spc,
                                                                                             self.dataOut.data_cspc,
                                                                                             self.dataOut.data_dc)
                     else:
                         self.nProfiles=self.dataOut.nProfiles
                         self.nChannels=self.dataOut.nChannels
                         self.nHeights=self.dataOut.nHeights
                         avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
                                                                                             self.dataOut.dataLag_spc,
                                                                                             self.dataOut.dataLag_cspc,
                                                                                             self.dataOut.dataLag_dc)
                     self.dataOut.flagNoData = True
                     if self._flagProfilesByRange:
                         dataOut.flagProfilesByRange = True
                         self._nProfilesByRange += dataOut.nProfilesByRange
                     if self.__dataReady:
                         if not self.ByLags:
                             if self.nChannels == 1:
                                 #print("f int", avgdata_spc.shape)
                                 self.dataOut.data_spc = avgdata_spc
                                 self.dataOut.data_cspc = None
                             else:
                                 self.dataOut.data_spc = numpy.squeeze(avgdata_spc)
                                 self.dataOut.data_cspc = numpy.squeeze(avgdata_cspc)
                             self.dataOut.data_dc = avgdata_dc
                             self.dataOut.data_outlier = self.dataOutliers
                         else:
                             self.dataOut.dataLag_spc = avgdata_spc
                             self.dataOut.dataLag_cspc = avgdata_cspc
                             self.dataOut.dataLag_dc = avgdata_dc
                             self.dataOut.data_spc=self.dataOut.dataLag_spc[:,:,:,self.dataOut.LagPlot]
                             self.dataOut.data_cspc=self.dataOut.dataLag_cspc[:,:,:,self.dataOut.LagPlot]
                             self.dataOut.data_dc=self.dataOut.dataLag_dc[:,:,self.dataOut.LagPlot]
                         self.dataOut.nIncohInt *= self.n_ints
                         self.dataOut.utctime = avgdatatime
                         self.dataOut.flagNoData = False
                         dataOut.nProfilesByRange = self._nProfilesByRange
                         self._nProfilesByRange = numpy.zeros((1,len(dataOut.heightList)))
                         self._flagProfilesByRange = False
                     return self.dataOut
             class dopplerFlip(Operation):
                 def run(self, dataOut, chann = None):
                     # arreglo 1: (num_chan, num_profiles, num_heights)
                     self.dataOut = dataOut
                     # JULIA-oblicua, indice 2
                     # arreglo 2: (num_profiles, num_heights)
                     jspectra = self.dataOut.data_spc[chann]
                     jspectra_tmp = numpy.zeros(jspectra.shape)
                     num_profiles = jspectra.shape[0]
                     freq_dc = int(num_profiles / 2)
                     # Flip con for
                     for j in range(num_profiles):
                         jspectra_tmp[num_profiles-j-1]= jspectra[j]
                     # Intercambio perfil de DC con perfil inmediato anterior
                     jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1]
                     jspectra_tmp[freq_dc]= jspectra[freq_dc]
                     # canal modificado es re-escrito en el arreglo de canales
                     self.dataOut.data_spc[chann] = jspectra_tmp
                     return self.dataOut

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages