schain Commit - r1326:6584204c3602 · Jicamarca Repository

Rewrite Param reader/writer as HDF reader/writer

Juan C. Espinoza -

r1326:6584204c3602

parent child

Context file:

r1326:6584204c3602

Collapse all files

schainpy/model/data/jrodata.py +5 -6

             '''
             $Author: murco $
             $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
             '''
             import copy
             import numpy
             import datetime
             import json
             import schainpy.admin
             from schainpy.utils import log
             from .jroheaderIO import SystemHeader, RadarControllerHeader
             from schainpy.model.data import _noise
             def getNumpyDtype(dataTypeCode):
                 if dataTypeCode == 0:
                     numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
                 elif dataTypeCode == 1:
                     numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
                 elif dataTypeCode == 2:
                     numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
                 elif dataTypeCode == 3:
                     numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
                 elif dataTypeCode == 4:
                     numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                 elif dataTypeCode == 5:
                     numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
                 else:
                     raise ValueError('dataTypeCode was not defined')
                 return numpyDtype
             def getDataTypeCode(numpyDtype):
                 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
                     datatype = 0
                 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
                     datatype = 1
                 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
                     datatype = 2
                 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
                     datatype = 3
                 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
                     datatype = 4
                 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
                     datatype = 5
                 else:
                     datatype = None
                 return datatype
             def hildebrand_sekhon(data, navg):
                 """
                 This method is for the objective determination of the noise level in Doppler spectra. This
                 implementation technique is based on the fact that the standard deviation of the spectral
                 densities is equal to the mean spectral density for white Gaussian noise
                 Inputs:
                     Data    :    heights
                     navg    :    numbers of averages
                 Return:
                     mean    :    noise's level
                 """
                 sortdata = numpy.sort(data, axis=None)
                 '''
                 lenOfData = len(sortdata)
                 nums_min = lenOfData*0.2
                 if nums_min <= 5:
                     nums_min = 5
                 sump = 0.
                 sumq = 0.
                 j = 0
                 cont = 1
                 while((cont == 1)and(j < lenOfData)):
                     sump += sortdata[j]
                     sumq += sortdata[j]**2
                     if j > nums_min:
                         rtest = float(j)/(j-1) + 1.0/navg
                         if ((sumq*j) > (rtest*sump**2)):
                             j = j - 1
                             sump = sump - sortdata[j]
                             sumq = sumq - sortdata[j]**2
                             cont = 0
                     j += 1
                 lnoise = sump / j
                 '''
                 return _noise.hildebrand_sekhon(sortdata, navg)
             class Beam:
                 def __init__(self):
                     self.codeList = []
                     self.azimuthList = []
                     self.zenithList = []
             class GenericData(object):
                 flagNoData = True
                 def copy(self, inputObj=None):
                     if inputObj == None:
                         return copy.deepcopy(self)
                     for key in list(inputObj.__dict__.keys()):
                         attribute = inputObj.__dict__[key]
                         # If this attribute is a tuple or list
                         if type(inputObj.__dict__[key]) in (tuple, list):
                             self.__dict__[key] = attribute[:]
                             continue
                         # If this attribute is another object or instance
                         if hasattr(attribute, '__dict__'):
                             self.__dict__[key] = attribute.copy()
                             continue
                         self.__dict__[key] = inputObj.__dict__[key]
                 def deepcopy(self):
                     return copy.deepcopy(self)
                 def isEmpty(self):
                     return self.flagNoData
                 def isReady(self):
                     return not self.flagNoData
             class JROData(GenericData):
                 #    m_BasicHeader = BasicHeader()
                 #    m_ProcessingHeader = ProcessingHeader()
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
             #    data = None
                 type = None
                 datatype = None  # dtype but in string
             #     dtype = None
             #    nChannels = None
             #    nHeights = None
                 nProfiles = None
                 heightList = None
                 channelList = None
                 flagDiscontinuousBlock = False
                 useLocalTime = False
                 utctime = None
                 timeZone = None
                 dstFlag = None
                 errorCount = None
                 blocksize = None
             #     nCode = None
             #     nBaud = None
             #     code = None
                 flagDecodeData = False  # asumo q la data no esta decodificada
                 flagDeflipData = False  # asumo q la data no esta sin flip
                 flagShiftFFT = False
             #     ippSeconds = None
             #     timeInterval = None
                 nCohInt = None
             #     noise = None
                 windowOfFilter = 1
                 # Speed of ligth
                 C = 3e8
                 frequency = 49.92e6
                 realtime = False
                 beacon_heiIndexList = None
                 last_block = None
                 blocknow = None
                 azimuth = None
                 zenith = None
                 beam = Beam()
                 profileIndex = None
                 error = None
                 data = None
                 nmodes = None
                 def __str__(self):
                     return '{} - {}'.format(self.type, self.getDatatime())
                 def getNoise(self):
                     raise NotImplementedError
                 def getNChannels(self):
                     return len(self.channelList)
                 def getChannelIndexList(self):
                     return list(range(self.nChannels))
                 def getNHeights(self):
                     return len(self.heightList)
                 def getHeiRange(self, extrapoints=0):
                     heis = self.heightList
             #        deltah = self.heightList[1] - self.heightList[0]
             #
             #        heis.append(self.heightList[-1])
                     return heis
                 def getDeltaH(self):
                     delta = self.heightList[1] - self.heightList[0]
                     return delta
                 def getltctime(self):
                     if self.useLocalTime:
                         return self.utctime - self.timeZone * 60
                     return self.utctime
                 def getDatatime(self):
                     datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
                     return datatimeValue
                 def getTimeRange(self):
                     datatime = []
                     datatime.append(self.ltctime)
                     datatime.append(self.ltctime + self.timeInterval + 1)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getFmaxTimeResponse(self):
                     period = (10**-6) * self.getDeltaH() / (0.15)
                     PRF = 1. / (period * self.nCohInt)
                     fmax = PRF
                     return fmax
                 def getFmax(self):
                     PRF = 1. / (self.ippSeconds * self.nCohInt)
                     fmax = PRF
                     return fmax
                 def getVmax(self):
                     _lambda = self.C / self.frequency
                     vmax = self.getFmax() * _lambda / 2
                     return vmax
                 def get_ippSeconds(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.ippSeconds
                 def set_ippSeconds(self, ippSeconds):
                     '''
                     '''
                     self.radarControllerHeaderObj.ippSeconds = ippSeconds
                     return
                 def get_dtype(self):
                     '''
                     '''
                     return getNumpyDtype(self.datatype)
                 def set_dtype(self, numpyDtype):
                     '''
                     '''
                     self.datatype = getDataTypeCode(numpyDtype)
                 def get_code(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.code
                 def set_code(self, code):
                     '''
                     '''
                     self.radarControllerHeaderObj.code = code
                     return
                 def get_ncode(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.nCode
                 def set_ncode(self, nCode):
                     '''
                     '''
                     self.radarControllerHeaderObj.nCode = nCode
                     return
                 def get_nbaud(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.nBaud
                 def set_nbaud(self, nBaud):
                     '''
                     '''
                     self.radarControllerHeaderObj.nBaud = nBaud
                     return
                 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
                 channelIndexList = property(
                     getChannelIndexList, "I'm the 'channelIndexList' property.")
                 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
                 #noise = property(getNoise, "I'm the 'nHeights' property.")
                 datatime = property(getDatatime, "I'm the 'datatime' property")
                 ltctime = property(getltctime, "I'm the 'ltctime' property")
                 ippSeconds = property(get_ippSeconds, set_ippSeconds)
                 dtype = property(get_dtype, set_dtype)
             #     timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
                 code = property(get_code, set_code)
                 nCode = property(get_ncode, set_ncode)
                 nBaud = property(get_nbaud, set_nbaud)
             class Voltage(JROData):
                 # data es un numpy array de 2 dmensiones (canales, alturas)
                 data = None
                 dataPP_POW   = None
                 dataPP_DOP   = None
                 dataPP_WIDTH = None
                 dataPP_SNR   = None
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.useLocalTime = True
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Voltage"
                     self.data = None
             #         self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
-                    self.timeZone = None
+                    self.timeZone = 0
                     self.dstFlag = None
                     self.errorCount = None
                     self.nCohInt = None
                     self.blocksize = None
                     self.flagCohInt = False
                     self.flagDecodeData = False  # asumo q la data no esta decodificada
                     self.flagDeflipData = False  # asumo q la data no esta sin flip
                     self.flagShiftFFT = False
                     self.flagDataAsBlock = False  # Asumo que la data es leida perfil a perfil
                     self.profileIndex = 0
                 def getNoisebyHildebrand(self, channel=None):
                     """
                     Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                     Return:
                         noiselevel
                     """
                     if channel != None:
                         data = self.data[channel]
                         nChannels = 1
                     else:
                         data = self.data
                         nChannels = self.nChannels
                     noise = numpy.zeros(nChannels)
                     power = data * numpy.conjugate(data)
                     for thisChannel in range(nChannels):
                         if nChannels == 1:
                             daux = power[:].real
                         else:
                             daux = power[thisChannel, :].real
                         noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
                     return noise
                 def getNoise(self, type=1, channel=None):
                     if type == 1:
                         noise = self.getNoisebyHildebrand(channel)
                     return noise
                 def getPower(self, channel=None):
                     if channel != None:
                         data = self.data[channel]
                     else:
                         data = self.data
                     power = data * numpy.conjugate(data)
                     powerdB = 10 * numpy.log10(power.real)
                     powerdB = numpy.squeeze(powerdB)
                     return powerdB
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt
                     return timeInterval
                 noise = property(getNoise, "I'm the 'nHeights' property.")
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
             class Spectra(JROData):
                 # data spc es un numpy array de 2 dmensiones (canales, perfiles, alturas)
                 data_spc = None
                 # data cspc es un numpy array de 2 dmensiones (canales, pares, alturas)
                 data_cspc = None
                 # data dc es un numpy array de 2 dmensiones (canales, alturas)
                 data_dc = None
                 # data power
                 data_pwr = None
                 nFFTPoints = None
             #     nPairs = None
                 pairsList = None
                 nIncohInt = None
                 wavelength = None  # Necesario para cacular el rango de velocidad desde la frecuencia
                 nCohInt = None  # se requiere para determinar el valor de timeInterval
                 ippFactor = None
                 profileIndex = 0
                 plotting = "spectra"
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.useLocalTime = True
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Spectra"
+                    self.timeZone = 0
             #        self.data = None
             #         self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.pairsList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.nCohInt = None
                     self.nIncohInt = None
                     self.blocksize = None
                     self.nFFTPoints = None
                     self.wavelength = None
                     self.flagDecodeData = False  # asumo q la data no esta decodificada
                     self.flagDeflipData = False  # asumo q la data no esta sin flip
                     self.flagShiftFFT = False
                     self.ippFactor = 1
                     #self.noise = None
                     self.beacon_heiIndexList = []
                     self.noise_estimation = None
                 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                     """
                     Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                     Return:
                         noiselevel
                     """
                     noise = numpy.zeros(self.nChannels)
                     for channel in range(self.nChannels):
                         daux = self.data_spc[channel,
                                              xmin_index:xmax_index, ymin_index:ymax_index]
                         noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
                     return noise
                 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                     if self.noise_estimation is not None:
                         # this was estimated by getNoise Operation defined in jroproc_spectra.py
                         return self.noise_estimation
                     else:
                         noise = self.getNoisebyHildebrand(
                             xmin_index, xmax_index, ymin_index, ymax_index)
                         return noise
                 def getFreqRangeTimeResponse(self, extrapoints=0):
                     deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
                     freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
                     return freqrange
                 def getAcfRange(self, extrapoints=0):
                     deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
                     freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
                     return freqrange
                 def getFreqRange(self, extrapoints=0):
                     deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
                     freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
                     return freqrange
                 def getVelRange(self, extrapoints=0):
                     deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
                     velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
                     if self.nmodes:
                         return velrange/self.nmodes
                     else:
                         return velrange
                 def getNPairs(self):
                     return len(self.pairsList)
                 def getPairsIndexList(self):
                     return list(range(self.nPairs))
                 def getNormFactor(self):
                     pwcode = 1
                     if self.flagDecodeData:
                         pwcode = numpy.sum(self.code[0]**2)
                     #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
                     normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
                     return normFactor
                 def getFlagCspc(self):
                     if self.data_cspc is None:
                         return True
                     return False
                 def getFlagDc(self):
                     if self.data_dc is None:
                         return True
                     return False
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
                     if self.nmodes:
                         return self.nmodes*timeInterval
                     else:
                         return timeInterval
                 def getPower(self):
                     factor = self.normFactor
                     z = self.data_spc / factor
                     z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                     avg = numpy.average(z, axis=1)
                     return 10 * numpy.log10(avg)
                 def getCoherence(self, pairsList=None, phase=False):
                     z = []
                     if pairsList is None:
                         pairsIndexList = self.pairsIndexList
                     else:
                         pairsIndexList = []
                         for pair in pairsList:
                             if pair not in self.pairsList:
                                 raise ValueError("Pair %s is not in dataOut.pairsList" % (
                                     pair))
                             pairsIndexList.append(self.pairsList.index(pair))
                     for i in range(len(pairsIndexList)):
                         pair = self.pairsList[pairsIndexList[i]]
                         ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
                         powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
                         powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
                         avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
                         if phase:
                             data = numpy.arctan2(avgcoherenceComplex.imag,
                                                  avgcoherenceComplex.real) * 180 / numpy.pi
                         else:
                             data = numpy.abs(avgcoherenceComplex)
                         z.append(data)
                     return numpy.array(z)
                 def setValue(self, value):
                     print("This property should not be initialized")
                     return
                 nPairs = property(getNPairs, setValue, "I'm the 'nPairs' property.")
                 pairsIndexList = property(
                     getPairsIndexList, setValue, "I'm the 'pairsIndexList' property.")
                 normFactor = property(getNormFactor, setValue,
                                       "I'm the 'getNormFactor' property.")
                 flag_cspc = property(getFlagCspc, setValue)
                 flag_dc = property(getFlagDc, setValue)
                 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
                 timeInterval = property(getTimeInterval, setValue,
                                         "I'm the 'timeInterval' property")
             class SpectraHeis(Spectra):
                 data_spc = None
                 data_cspc = None
                 data_dc = None
                 nFFTPoints = None
             #     nPairs = None
                 pairsList = None
                 nCohInt = None
                 nIncohInt = None
                 def __init__(self):
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "SpectraHeis"
             #         self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
             #         self.nPairs = 0
                     self.utctime = None
                     self.blocksize = None
                     self.profileIndex = 0
                     self.nCohInt = 1
                     self.nIncohInt = 1
                 def getNormFactor(self):
                     pwcode = 1
                     if self.flagDecodeData:
                         pwcode = numpy.sum(self.code[0]**2)
                     normFactor = self.nIncohInt * self.nCohInt * pwcode
                     return normFactor
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     return timeInterval
                 normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
             class Fits(JROData):
                 heightList = None
                 channelList = None
                 flagNoData = True
                 flagDiscontinuousBlock = False
                 useLocalTime = False
                 utctime = None
-                timeZone = None
             #     ippSeconds = None
             #     timeInterval = None
                 nCohInt = None
                 nIncohInt = None
                 noise = None
                 windowOfFilter = 1
                 # Speed of ligth
                 C = 3e8
                 frequency = 49.92e6
                 realtime = False
                 def __init__(self):
                     self.type = "Fits"
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #         self.channelIndexList = None
                     self.flagNoData = True
                     self.utctime = None
                     self.nCohInt = 1
                     self.nIncohInt = 1
                     self.useLocalTime = True
                     self.profileIndex = 0
             #         self.utctime = None
-            #         self.timeZone = None
+                    self.timeZone = 0
             #         self.ltctime = None
             #         self.timeInterval = None
             #         self.header = None
             #         self.data_header = None
             #         self.data = None
             #         self.datatime = None
             #         self.flagNoData = False
             #         self.expName = ''
             #         self.nChannels = None
             #         self.nSamples = None
             #         self.dataBlocksPerFile = None
             #         self.comments = ''
             #
                 def getltctime(self):
                     if self.useLocalTime:
                         return self.utctime - self.timeZone * 60
                     return self.utctime
                 def getDatatime(self):
                     datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
                     return datatime
                 def getTimeRange(self):
                     datatime = []
                     datatime.append(self.ltctime)
                     datatime.append(self.ltctime + self.timeInterval)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getHeiRange(self):
                     heis = self.heightList
                     return heis
                 def getNHeights(self):
                     return len(self.heightList)
                 def getNChannels(self):
                     return len(self.channelList)
                 def getChannelIndexList(self):
                     return list(range(self.nChannels))
                 def getNoise(self, type=1):
                     #noise = numpy.zeros(self.nChannels)
                     if type == 1:
                         noise = self.getNoisebyHildebrand()
                     if type == 2:
                         noise = self.getNoisebySort()
                     if type == 3:
                         noise = self.getNoisebyWindow()
                     return noise
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     return timeInterval
                 def get_ippSeconds(self):
                     '''
                     '''
                     return self.ipp_sec
                 datatime = property(getDatatime, "I'm the 'datatime' property")
                 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
                 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
                 channelIndexList = property(
                     getChannelIndexList, "I'm the 'channelIndexList' property.")
                 noise = property(getNoise, "I'm the 'nHeights' property.")
                 ltctime = property(getltctime, "I'm the 'ltctime' property")
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
                 ippSeconds = property(get_ippSeconds, '')
             class Correlation(JROData):
                 noise = None
                 SNR = None
                 #--------------------------------------------------
                 mode = None
                 split = False
                 data_cf = None
                 lags = None
                 lagRange = None
                 pairsList = None
                 normFactor = None
                 #--------------------------------------------------
             #     calculateVelocity = None
                 nLags = None
                 nPairs = None
                 nAvg = None
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Correlation"
                     self.data = None
                     self.dtype = None
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
-                    self.timeZone = None
+                    self.timeZone = 0
                     self.dstFlag = None
                     self.errorCount = None
                     self.blocksize = None
                     self.flagDecodeData = False  # asumo q la data no esta decodificada
                     self.flagDeflipData = False  # asumo q la data no esta sin flip
                     self.pairsList = None
                     self.nPoints = None
                 def getPairsList(self):
                     return self.pairsList
                 def getNoise(self, mode=2):
                     indR = numpy.where(self.lagR == 0)[0][0]
                     indT = numpy.where(self.lagT == 0)[0][0]
                     jspectra0 = self.data_corr[:, :, indR, :]
                     jspectra = copy.copy(jspectra0)
                     num_chan = jspectra.shape[0]
                     num_hei = jspectra.shape[2]
                     freq_dc = jspectra.shape[1] / 2
                     ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
                     if ind_vel[0] < 0:
                         ind_vel[list(range(0, 1))] = ind_vel[list(
                             range(0, 1))] + self.num_prof
                     if mode == 1:
                         jspectra[:, freq_dc, :] = (
                             jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2  # CORRECCION
                     if mode == 2:
                         vel = numpy.array([-2, -1, 1, 2])
                         xx = numpy.zeros([4, 4])
                         for fil in range(4):
                             xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
                         xx_inv = numpy.linalg.inv(xx)
                         xx_aux = xx_inv[0, :]
                         for ich in range(num_chan):
                             yy = jspectra[ich, ind_vel, :]
                             jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
                             junkid = jspectra[ich, freq_dc, :] <= 0
                             cjunkid = sum(junkid)
                             if cjunkid.any():
                                 jspectra[ich, freq_dc, junkid.nonzero()] = (
                                     jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
                     noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
                     return noise
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nProfiles
                     return timeInterval
                 def splitFunctions(self):
                     pairsList = self.pairsList
                     ccf_pairs = []
                     acf_pairs = []
                     ccf_ind = []
                     acf_ind = []
                     for l in range(len(pairsList)):
                         chan0 = pairsList[l][0]
                         chan1 = pairsList[l][1]
                         # Obteniendo pares de Autocorrelacion
                         if chan0 == chan1:
                             acf_pairs.append(chan0)
                             acf_ind.append(l)
                         else:
                             ccf_pairs.append(pairsList[l])
                             ccf_ind.append(l)
                     data_acf = self.data_cf[acf_ind]
                     data_ccf = self.data_cf[ccf_ind]
                     return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
                 def getNormFactor(self):
                     acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
                     acf_pairs = numpy.array(acf_pairs)
                     normFactor = numpy.zeros((self.nPairs, self.nHeights))
                     for p in range(self.nPairs):
                         pair = self.pairsList[p]
                         ch0 = pair[0]
                         ch1 = pair[1]
                         ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
                         ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
                         normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
                     return normFactor
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
                 normFactor = property(getNormFactor, "I'm the 'normFactor property'")
             class Parameters(Spectra):
                 experimentInfo = None  # Information about the experiment
                 # Information from previous data
                 inputUnit = None  # Type of data to be processed
                 operation = None  # Type of operation to parametrize
                 # normFactor = None       #Normalization Factor
                 groupList = None  # List of Pairs, Groups, etc
                 # Parameters
                 data_param = None  # Parameters obtained
                 data_pre = None  # Data Pre Parametrization
                 data_SNR = None  # Signal to Noise Ratio
             #     heightRange = None      #Heights
                 abscissaList = None  # Abscissa, can be velocities, lags or time
             #    noise = None            #Noise Potency
                 utctimeInit = None  # Initial UTC time
                 paramInterval = None  # Time interval to calculate Parameters in seconds
                 useLocalTime = True
                 # Fitting
                 data_error = None  # Error of the estimation
                 constants = None
                 library = None
                 # Output signal
                 outputInterval = None  # Time interval to calculate output signal in seconds
                 data_output = None  # Out signal
                 nAvg = None
                 noise_estimation = None
                 GauSPC = None  # Fit gaussian SPC
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Parameters"
+                    self.timeZone = 0
                 def getTimeRange1(self, interval):
                     datatime = []
                     if self.useLocalTime:
                         time1 = self.utctimeInit - self.timeZone * 60
                     else:
                         time1 = self.utctimeInit
                     datatime.append(time1)
                     datatime.append(time1 + interval)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getTimeInterval(self):
                     if hasattr(self, 'timeInterval1'):
                         return self.timeInterval1
                     else:
                         return self.paramInterval
                 def setValue(self, value):
                     print("This property should not be initialized")
                     return
                 def getNoise(self):
                     return self.spc_noise
                 timeInterval = property(getTimeInterval)
                 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
             class PlotterData(object):
                 '''
                 Object to hold data to be plotted
                 '''
                 MAXNUMX = 200
                 MAXNUMY = 200
                 def __init__(self, code, throttle_value, exp_code, buffering=True, snr=False):
                     self.key = code
                     self.throttle = throttle_value
                     self.exp_code = exp_code
                     self.buffering = buffering
                     self.ready = False
                     self.flagNoData = False
                     self.localtime = False
                     self.data = {}
                     self.meta = {}
                     self.__heights = []
                     if 'snr' in code:
                         self.plottypes = ['snr']
                     elif code == 'spc':
                         self.plottypes = ['spc', 'noise', 'rti']
                     elif code == 'cspc':
                         self.plottypes = ['cspc', 'spc', 'noise', 'rti']
                     elif code == 'rti':
                         self.plottypes = ['noise', 'rti']
                     else:
                         self.plottypes = [code]
                     if 'snr' not in self.plottypes and snr:
                         self.plottypes.append('snr')
                     for plot in self.plottypes:
                         self.data[plot] = {}
                 def __str__(self):
                     dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
                     return 'Data[{}][{}]'.format(';'.join(dum), len(self.times))
                 def __len__(self):
                     return len(self.data[self.key])
                 def __getitem__(self, key):
                     if key not in self.data:
                         raise KeyError(log.error('Missing key: {}'.format(key)))
                     if 'spc' in key or not self.buffering:
                         ret = self.data[key][self.tm]
                     elif 'scope' in key:
                         ret = numpy.array(self.data[key][float(self.tm)])
                     else:
                         ret = numpy.array([self.data[key][x] for x in self.times])
                         if ret.ndim > 1:
                             ret = numpy.swapaxes(ret, 0, 1)
                     return ret
                 def __contains__(self, key):
                     return key in self.data
                 def setup(self):
                     '''
                     Configure object
                     '''
                     self.type = ''
                     self.ready = False
                     del self.data
                     self.data = {}
                     self.__heights = []
                     self.__all_heights = set()
                     for plot in self.plottypes:
                         if 'snr' in plot:
                             plot = 'snr'
                         elif 'spc_moments' == plot:
                             plot = 'moments'
                         self.data[plot] = {}
                     if 'spc' in self.data or 'rti' in self.data or 'cspc' in self.data or 'moments' in self.data:
                         self.data['noise'] = {}
                         self.data['rti'] = {}
                         if 'noise' not in self.plottypes:
                             self.plottypes.append('noise')
                         if 'rti' not in self.plottypes:
                             self.plottypes.append('rti')
                 def shape(self, key):
                     '''
                     Get the shape of the one-element data for the given key
                     '''
                     if len(self.data[key]):
                         if 'spc' in key or not self.buffering:
                             return self.data[key].shape
                         return self.data[key][self.times[0]].shape
                     return (0,)
                 def update(self, dataOut, tm):
                     '''
                     Update data object with new dataOut
                     '''
                     self.profileIndex = dataOut.profileIndex
                     self.tm = tm
                     self.type = dataOut.type
                     self.parameters = getattr(dataOut, 'parameters', [])
                     if hasattr(dataOut, 'meta'):
                         self.meta.update(dataOut.meta)
                     if hasattr(dataOut, 'pairsList'):
                         self.pairs = dataOut.pairsList
                     self.interval = dataOut.getTimeInterval()
                     self.localtime = dataOut.useLocalTime
                     if True in ['spc' in ptype for ptype in self.plottypes]:
                         self.xrange = (dataOut.getFreqRange(1)/1000.,
                                        dataOut.getAcfRange(1), dataOut.getVelRange(1))
                     self.__heights.append(dataOut.heightList)
                     self.__all_heights.update(dataOut.heightList)
                     for plot in self.plottypes:
                         if plot in ('spc', 'spc_moments', 'spc_cut'):
                             z = dataOut.data_spc/dataOut.normFactor
                             buffer = 10*numpy.log10(z)
                         if plot == 'cspc':
                             buffer = (dataOut.data_spc, dataOut.data_cspc)
                         if plot == 'noise':
                             buffer = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
                         if plot in ('rti', 'spcprofile'):
                             buffer = dataOut.getPower()
                         if plot == 'snr_db':
                             buffer = dataOut.data_SNR
                         if plot == 'snr':
                             buffer = 10*numpy.log10(dataOut.data_SNR)
                         if plot == 'dop':
                             buffer = dataOut.data_DOP
                         if plot == 'pow':
                             buffer = 10*numpy.log10(dataOut.data_POW)
                         if plot == 'width':
                             buffer = dataOut.data_WIDTH
                         if plot == 'coh':
                             buffer = dataOut.getCoherence()
                         if plot == 'phase':
                             buffer = dataOut.getCoherence(phase=True)
                         if plot == 'output':
                             buffer = dataOut.data_output
                         if plot == 'param':
                             buffer = dataOut.data_param
                         if plot == 'scope':
                             buffer = dataOut.data
                             self.flagDataAsBlock = dataOut.flagDataAsBlock
                             self.nProfiles = dataOut.nProfiles
                         if plot == 'pp_power':
                             buffer = dataOut.dataPP_POWER
                             self.flagDataAsBlock = dataOut.flagDataAsBlock
                             self.nProfiles = dataOut.nProfiles
                         if plot == 'pp_signal':
                             buffer = dataOut.dataPP_POW
                             self.flagDataAsBlock = dataOut.flagDataAsBlock
                             self.nProfiles = dataOut.nProfiles
                         if plot == 'pp_velocity':
                             buffer = dataOut.dataPP_DOP
                             self.flagDataAsBlock = dataOut.flagDataAsBlock
                             self.nProfiles = dataOut.nProfiles
                         if plot == 'pp_specwidth':
                             buffer = dataOut.dataPP_WIDTH
                             self.flagDataAsBlock = dataOut.flagDataAsBlock
                             self.nProfiles = dataOut.nProfiles
                         if plot == 'spc':
                             self.data['spc'][tm] = buffer
                         elif plot == 'cspc':
                             self.data['cspc'][tm] = buffer
                         elif plot == 'spc_moments':
                             self.data['spc'][tm] = buffer
                             self.data['moments'][tm] = dataOut.moments
                         else:
                             if self.buffering:
                                 self.data[plot][tm] = buffer
                             else:
                                 self.data[plot][tm] = buffer
                     if dataOut.channelList is None:
                         self.channels = range(buffer.shape[0])
                     else:
                         self.channels = dataOut.channelList
                     if buffer is None:
                         self.flagNoData = True
                         raise schainpy.admin.SchainWarning('Attribute data_{} is empty'.format(self.key))
                 def normalize_heights(self):
                     '''
                     Ensure same-dimension of the data for different heighList
                     '''
                     H = numpy.array(list(self.__all_heights))
                     H.sort()
                     for key in self.data:
                         shape = self.shape(key)[:-1] + H.shape
                         for tm, obj in list(self.data[key].items()):
                             h = self.__heights[self.times.tolist().index(tm)]
                             if H.size == h.size:
                                 continue
                             index = numpy.where(numpy.in1d(H, h))[0]
                             dummy = numpy.zeros(shape) + numpy.nan
                             if len(shape) == 2:
                                 dummy[:, index] = obj
                             else:
                                 dummy[index] = obj
                             self.data[key][tm] = dummy
                     self.__heights = [H for tm in self.times]
                 def jsonify(self, tm, plot_name, plot_type, decimate=False):
                     '''
                     Convert data to json
                     '''
                     dy = int(self.heights.size/self.MAXNUMY) + 1
                     if self.key in ('spc', 'cspc'):
                         dx = int(self.data[self.key][tm].shape[1]/self.MAXNUMX) + 1
                         data = self.roundFloats(
                             self.data[self.key][tm][::, ::dx, ::dy].tolist())
                     else:
                         if self.key is 'noise':
                             data = [[x] for x in self.roundFloats(self.data[self.key][tm].tolist())]
                         else:
                             data = self.roundFloats(self.data[self.key][tm][::, ::dy].tolist())
                     meta = {}
                     ret = {
                         'plot': plot_name,
                         'code': self.exp_code,
                         'time': float(tm),
                         'data': data,
                         }
                     meta['type'] = plot_type
                     meta['interval'] = float(self.interval)
                     meta['localtime'] = self.localtime
                     meta['yrange'] = self.roundFloats(self.heights[::dy].tolist())
                     if 'spc' in self.data or 'cspc' in self.data:
                         meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
                     else:
                         meta['xrange'] = []
                     meta.update(self.meta)
                     ret['metadata'] = meta
                     return json.dumps(ret)
                 @property
                 def times(self):
                     '''
                     Return the list of times of the current data
                     '''
                     ret = numpy.array([*self.data[self.key]])
                     if self:
                         ret.sort()
                     return ret
                 @property
                 def min_time(self):
                     '''
                     Return the minimun time value
                     '''
                     return self.times[0]
                 @property
                 def max_time(self):
                     '''
                     Return the maximun time value
                     '''
                     return self.times[-1]
                 @property
                 def heights(self):
                     '''
                     Return the list of heights of the current data
                     '''
                     return numpy.array(self.__heights[-1])
                 @staticmethod
                 def roundFloats(obj):
                     if isinstance(obj, list):
                         return list(map(PlotterData.roundFloats, obj))
                     elif isinstance(obj, float):
                         return round(obj, 2)

schainpy/model/io/jroIO_param.py +261 -1076

This diff has been collapsed as it changes many lines, (1337 lines changed) Show them Hide them
	@@ -1,1435 +1,620
	1	import numpy
	2	import time
	3	import os	1	import os
	4	import ~~h5py~~	2	import time
	5	import re
	6	import datetime	3	import datetime
	7		4
			5	import numpy
			6	import h5py
			7
	8	import schainpy.admin	8	import schainpy.admin
	9	from schainpy.model.data.jrodata import *	9	from schainpy.model.data.jrodata import *
	10	from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator	10	from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
	11	from schainpy.model.io.jroIO_base import *	11	from schainpy.model.io.jroIO_base import *
	12	from schainpy.utils import log	12	from schainpy.utils import log
	13		13
	14		14
	15	class ~~Param~~Reader(~~JROData~~Reader,ProcessingUnit):	15	class HDFReader(Reader, ProcessingUnit):
	16	'''	16	"""Processing unit to read HDF5 format files
	17	Reads HDF5 format files	17
	18	path	18	This unit reads HDF5 files created with `HDFWriter` operation contains
	19	startDate	19	by default two groups Data and Metadata all variables would be saved as `dataOut`
	20	endDate	20	attributes.
	21	startTime	21	It is possible to read any HDF5 file by given the structure in the `description`
	22	endTime	22	parameter, also you can add extra values to metadata with the parameter `extras`.
	23	'''	23
	24		24	Parameters:
	25	ext = ".hdf5"	25	-----------
	26	optchar = "D"	26	path : str
	27	timezone = None	27	Path where files are located.
	28	start~~Time~~ = ~~Non~~e	28	startDate : date
	29	endTime = None	29	Start date of the files
	30	fileIndex = None	30	endDate : list
	31	utcList = None #To select data in the utctime list	31	End date of the files
	32	blockList = None #List to blocks to be read from the file	32	startTime : time
	33	blocksPerFile = None #Number of blocks to be read	33	Start time of the files
	34	blockIndex = None	34	endTime : time
	35	path = None	35	End time of the files
	36	#List of Files	36	description : dict, optional
	37	filenameList = None	37	Dictionary with the description of the HDF5 file
	38	datetimeList = None	38	extras : dict, optional
	39	#Hdf5 File	39	Dictionary with extra metadata to be be added to `dataOut`
	40	listMetaname = None	40
	41	listMeta = None	41	Examples
	42	listDataname = None	42	--------
	43	listData = None	43
	44	listShapes = None	44	desc = {
	45	fp = None	45	'Data': {
	46	#dataOut reconstruction	46	'data_output': ['u', 'v', 'w'],
	47	dataOut = None	47	'utctime': 'timestamps',
	48		48	} ,
	49	def __init__(self):#, **kwargs):	49	'Metadata': {
	50	ProcessingUnit.__init__(self) #, **kwargs)	50	'heightList': 'heights'
	51	self.dataOut = Parameters()	51	}
	52	return	52	}
	53		53
	54	def setup(self, **kwargs):	54	desc = {
	55		55	'Data': {
	56	path = kwargs['path']	56	'data_output': 'winds',
	57	startDate = kwargs['startDate']	57	'utctime': 'timestamps'
	58	endDate = kwargs['endDate']	58	},
	59	startTime = kwargs['startTime']	59	'Metadata': {
	60	endTime = kwargs['endTime']	60	'heightList': 'heights'
	61	walk = kwargs['walk']	61	}
	62	if 'ext' in kwargs:	62	}
	63	ext = kwargs['ext']	63
	64	else:	64	extras = {
	65	ext = '.hdf5'	65	'timeZone': 300
	66	if 'timezone' in kwargs:	66	}
	67	self.timezone = kwargs['timezone']	67
	68	else:	68	reader = project.addReadUnit(
	69	self.timezone = 'lt'	69	name='HDFReader',
	70		70	path='/path/to/files',
	71	print("[Reading] Searching files in offline mode ...")	71	startDate='2019/01/01',
	72	pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,	72	endDate='2019/01/31',
	73	startTime=startTime, endTime=endTime,	73	startTime='00:00:00',
	74	ext=ext, walk=walk)	74	endTime='23:59:59',
	75		75	# description=json.dumps(desc),
	76	if not(filenameList):	76	# extras=json.dumps(extras),
	77	print("There is no files into the folder: %s"%(path))	77	)
	78	sys.exit(-1)
	79
	80	self.fileIndex = -1
	81	self.startTime = startTime
	82	self.endTime = endTime
	83
	84	self.__readMetadata()
	85
	86	self.__setNextFileOffline()
	87
	88	return
	89
	90	def searchFilesOffLine(self,
	91	path,
	92	startDate=None,
	93	endDate=None,
	94	startTime=datetime.time(0,0,0),
	95	endTime=datetime.time(23,59,59),
	96	ext='.hdf5',
	97	walk=True):
	98
	99	expLabel = ''
	100	self.filenameList = []
	101	self.datetimeList = []
	102
	103	pathList = []
	104
	105	JRODataObj = JRODataReader()
	106	dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
	107
	108	if dateList == []:
	109	print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
	110	datetime.datetime.combine(startDate,startTime).ctime(),
	111	datetime.datetime.combine(endDate,endTime).ctime()))
	112
	113	return None, None
	114
	115	if len(dateList) > 1:
	116	print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
	117	else:
	118	print("[Reading] data was found for the date %s" %(dateList[0]))
	119
	120	filenameList = []
	121	datetimeList = []
	122
	123	#----------------------------------------------------------------------------------
	124
	125	for thisPath in pathList:
	126
	127	fileList = glob.glob1(thisPath, "*%s" %ext)
	128	fileList.sort()
	129
	130	for file in fileList:
	131
	132	filename = os.path.join(thisPath,file)
	133
	134	if not isFileInDateRange(filename, startDate, endDate):
	135	continue
	136
	137	thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
	138
	139	if not(thisDatetime):
	140	continue
	141
	142	filenameList.append(filename)
	143	datetimeList.append(thisDatetime)
	144
	145	if not(filenameList):
	146	print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
	147	return None, None
	148
	149	print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
	150	print()
	151
	152	self.filenameList = filenameList
	153	self.datetimeList = datetimeList
	154
	155	return pathList, filenameList
	156
	157	def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
	158
	159	"""
	160	Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
	161
	162	Inputs:
	163	filename : nombre completo del archivo de datos en formato Jicamarca (.r)
	164	startDate : fecha inicial del rango seleccionado en formato datetime.date
	165	endDate : fecha final del rango seleccionado en formato datetime.date
	166	startTime : tiempo inicial del rango seleccionado en formato datetime.time
	167	endTime : tiempo final del rango seleccionado en formato datetime.time
	168
	169	Return:
	170	Boolean : Retorna True si el archivo de datos contiene datos en el rango de
	171	fecha especificado, de lo contrario retorna False.
	172
	173	Excepciones:
	174	Si el archivo no existe o no puede ser abierto
	175	Si la cabecera no puede ser leida.
	176
	177	"""
	178
	179	try:
	180	fp = h5py.File(filename,'r')
	181	grp1 = fp['Data']
	182
	183	except IOError:
	184	traceback.print_exc()
	185	raise IOError("The file %s can't be opened" %(filename))
	186
	187	#In case has utctime attribute
	188	grp2 = grp1['utctime']
	189	# thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
	190	thisUtcTime = grp2.value[0]
	191
	192	fp.close()
	193
	194	if self.timezone == 'lt':
	195	thisUtcTime -= 5*3600
	196
	197	thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
	198	thisDate = thisDatetime.date()
	199	thisTime = thisDatetime.time()
	200
	201	startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
	202	endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
	203
	204	#General case
	205	# o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
	206	#-----------o----------------------------o-----------
	207	# startTime endTime
	208
	209	if endTime >= startTime:
	210	thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
	211	if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
	212	return thisDatetime
	213	return None
	214
	215	#If endTime < startTime then endTime belongs to the next day
	216	#<<<<<<<<<<<o o>>>>>>>>>>>
	217	#-----------o----------------------------o-----------
	218	# endTime startTime
	219
	220	if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
	221	return None
	222
	223	if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
	224	return None
	225
	226	if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
	227	return None
	228
	229	return thisDatetime
	230
	231	def __setNextFileOffline(self):
	232
	233	self.fileIndex += 1
	234	idFile = self.fileIndex
	235
	236	if not(idFile < len(self.filenameList)):
	237	raise schainpy.admin.SchainError("No more Files")
	238	return 0
	239
	240	filename = self.filenameList[idFile]
	241	filePointer = h5py.File(filename,'r')
	242	self.filename = filename
	243	self.fp = filePointer
	244
	245	print("Setting the file: %s"%self.filename)
	246
	247	self.__setBlockList()
	248	self.__readData()
	249	self.blockIndex = 0
	250	return 1
	251
	252	def __setBlockList(self):
	253	'''
	254	Selects the data within the times defined
	255
	256	self.fp
	257	self.startTime
	258	self.endTime
	259
	260	self.blockList
	261	self.blocksPerFile
	262
	263	'''
	264	fp = self.fp
	265	startTime = self.startTime
	266	endTime = self.endTime
	267
	268	grp = fp['Data']
	269	thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
	270
	271	#ERROOOOR
	272	if self.timezone == 'lt':
	273	thisUtcTime -= 5*3600
	274
	275	thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
	276
	277	thisDate = thisDatetime.date()
	278	thisTime = thisDatetime.time()
	279
	280	startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
	281	endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
	282
	283	ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
	284
	285	self.blockList = ind
	286	self.blocksPerFile = len(ind)
	287
	288	return
	289
	290	def __readMetadata(self):
	291	'''
	292	Reads Metadata
	293
	294	self.pathMeta
	295	self.listShapes
	296	self.listMetaname
	297	self.listMeta
	298
	299	'''
	300
	301	filename = self.filenameList[0]
	302	fp = h5py.File(filename,'r')
	303	gp = fp['Metadata']
	304
	305	listMetaname = []
	306	listMetadata = []
	307	for item in list(gp.items()):
	308	name = item[0]
	309
	310	if name=='array dimensions':
	311	table = gp[name][:]
	312	listShapes = {}
	313	for shapes in table:
	314	listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
	315	else:
	316	data = gp[name].value
	317	listMetaname.append(name)
	318	listMetadata.append(data)
	319
	320	self.listShapes = listShapes
	321	self.listMetaname = listMetaname
	322	self.listMeta = listMetadata
	323
	324	fp.close()
	325	return
	326
	327	def __readData(self):
	328	grp = self.fp['Data']
	329	listdataname = []
	330	listdata = []
	331
	332	for item in list(grp.items()):
	333	name = item[0]
	334	listdataname.append(name)
	335
	336	array = self.__setDataArray(grp[name],self.listShapes[name])
	337	listdata.append(array)
	338
	339	self.listDataname = listdataname
	340	self.listData = listdata
	341	return
	342
	343	def __setDataArray(self, dataset, shapes):
	344
	345	nDims = shapes[0]
	346	nDim2 = shapes[1] #Dimension 0
	347	nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
	348	nDim0 = shapes[3] #Dimension 2, number of samples or ranges
	349	mode = shapes[4] #Mode of storing
	350	blockList = self.blockList
	351	blocksPerFile = self.blocksPerFile
	352
	353	#Depending on what mode the data was stored
	354	if mode == 0: #Divided in channels
	355	arrayData = dataset.value.astype(numpy.float)[0][blockList]
	356	if mode == 1: #Divided in parameter
	357	strds = 'table'
	358	nDatas = nDim1
	359	newShapes = (blocksPerFile,nDim2,nDim0)
	360	elif mode==2: #Concatenated in a table
	361	strds = 'table0'
	362	arrayData = dataset[strds].value
	363	#Selecting part of the dataset
	364	utctime = arrayData[:,0]
	365	u, indices = numpy.unique(utctime, return_index=True)
	366
	367	if blockList.size != indices.size:
	368	indMin = indices[blockList[0]]
	369	if blockList[1] + 1 >= indices.size:
	370	arrayData = arrayData[indMin:,:]
	371	else:
	372	indMax = indices[blockList[1] + 1]
	373	arrayData = arrayData[indMin:indMax,:]
	374	return arrayData
	375
	376	# One dimension
	377	if nDims == 0:
	378	arrayData = dataset.value.astype(numpy.float)[0][blockList]
	379
	380	# Two dimensions
	381	elif nDims == 2:
	382	arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
	383	newShapes = (blocksPerFile,nDim0)
	384	nDatas = nDim1
	385
	386	for i in range(nDatas):
	387	data = dataset[strds + str(i)].value
	388	arrayData[:,i,:] = data[blockList,:]
	389
	390	# Three dimensions
	391	else:
	392	arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
	393	for i in range(nDatas):
	394
	395	data = dataset[strds + str(i)].value
	396
	397	for b in range(blockList.size):
	398	arrayData[b,:,i,:] = data[:,:,blockList[b]]
	399
	400	return arrayData
	401
	402	def __setDataOut(self):
	403	listMeta = self.listMeta
	404	listMetaname = self.listMetaname
	405	listDataname = self.listDataname
	406	listData = self.listData
	407	listShapes = self.listShapes
	408
	409	blockIndex = self.blockIndex
	410	# blockList = self.blockList
	411
	412	for i in range(len(listMeta)):
	413	setattr(self.dataOut,listMetaname[i],listMeta[i])
	414
	415	for j in range(len(listData)):
	416	nShapes = listShapes[listDataname[j]][0]
	417	mode = listShapes[listDataname[j]][4]
	418	if nShapes == 1:
	419	setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
	420	elif nShapes > 1:
	421	setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
	422	elif mode==0:
	423	setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
	424	#Mode Meteors
	425	elif mode ==2:
	426	selectedData = self.__selectDataMode2(listData[j], blockIndex)
	427	setattr(self.dataOut, listDataname[j], selectedData)
	428	return
	429
	430	def __selectDataMode2(self, data, blockIndex):
	431	utctime = data[:,0]
	432	aux, indices = numpy.unique(utctime, return_inverse=True)
	433	selInd = numpy.where(indices == blockIndex)[0]
	434	selData = data[selInd,:]
	435
	436	return selData
	437
	438	def getData(self):
	439
	440	if self.blockIndex==self.blocksPerFile:
	441	if not( self.__setNextFileOffline() ):
	442	self.dataOut.flagNoData = True
	443	return 0
	444
	445	self.__setDataOut()
	446	self.dataOut.flagNoData = False
	447
	448	self.blockIndex += 1
	449
	450	return
	451
	452	def run(self, **kwargs):
	453
	454	if not(self.isConfig):
	455	self.setup(**kwargs)
	456	self.isConfig = True
	457
	458	self.getData()
	459
	460	return
	461
	462	@MPDecorator
	463	class ParamWriter(Operation):
	464	'''
	465	HDF5 Writer, stores parameters data in HDF5 format files
	466
	467	path: path where the files will be stored
	468	blocksPerFile: number of blocks that will be saved in per HDF5 format file
	469	mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
	470	metadataList: list of attributes that will be stored as metadata
	471	dataList: list of attributes that will be stores as data
	472	'''
	473
	474	ext = ".hdf5"
	475	optchar = "D"
	476	metaoptchar = "M"
	477	metaFile = None
	478	filename = None
	479	path = None
	480	setFile = None
	481	fp = None
	482	grp = None
	483	ds = None
	484	firsttime = True
	485	#Configurations
	486	blocksPerFile = None
	487	blockIndex = None
	488	dataOut = None
	489	#Data Arrays
	490	dataList = None
	491	metadataList = None
	492	dsList = None #List of dictionaries with dataset properties
	493	tableDim = None
	494	dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
	495	currentDay = None
	496	lastTime = None
	497	setType = None
	498
	499	def __init__(self):
	500
	501	Operation.__init__(self)
	502	return
	503
	504	def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
	505	self.path = path
	506	self.blocksPerFile = blocksPerFile
	507	self.metadataList = metadataList
	508	self.dataList = dataList
	509	self.dataOut = dataOut
	510	self.mode = mode
	511	if self.mode is not None:
	512	self.mode = numpy.zeros(len(self.dataList)) + mode
	513	else:
	514	self.mode = numpy.ones(len(self.dataList))
	515
	516	self.setType = setType
	517
	518	arrayDim = numpy.zeros((len(self.dataList),5))
	519
	520	#Table dimensions
	521	dtype0 = self.dtype
	522	tableList = []
	523
	524	#Dictionary and list of tables
	525	dsList = []
	526
	527	for i in range(len(self.dataList)):
	528	dsDict = {}
	529	dataAux = getattr(self.dataOut, self.dataList[i])
	530	dsDict['variable'] = self.dataList[i]
	531	#--------------------- Conditionals ------------------------
	532	#There is no data
	533
	534	if dataAux is None:
	535
	536	return 0
	537
	538	if isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
	539	dsDict['mode'] = 0
	540	dsDict['nDim'] = 0
	541	arrayDim[i,0] = 0
	542	dsList.append(dsDict)
	543
	544	#Mode 2: meteors
	545	elif self.mode[i] == 2:
	546	dsDict['dsName'] = 'table0'
	547	dsDict['mode'] = 2 # Mode meteors
	548	dsDict['shape'] = dataAux.shape[-1]
	549	dsDict['nDim'] = 0
	550	dsDict['dsNumber'] = 1
	551	arrayDim[i,3] = dataAux.shape[-1]
	552	arrayDim[i,4] = self.mode[i] #Mode the data was stored
	553	dsList.append(dsDict)
	554
	555	#Mode 1
	556	else:
	557	arrayDim0 = dataAux.shape #Data dimensions
	558	arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
	559	arrayDim[i,4] = self.mode[i] #Mode the data was stored
	560	strtable = 'table'
	561	dsDict['mode'] = 1 # Mode parameters
	562
	563	# Three-dimension arrays
	564	if len(arrayDim0) == 3:
	565	arrayDim[i,1:-1] = numpy.array(arrayDim0)
	566	nTables = int(arrayDim[i,2])
	567	dsDict['dsNumber'] = nTables
	568	dsDict['shape'] = arrayDim[i,2:4]
	569	dsDict['nDim'] = 3
	570
	571	for j in range(nTables):
	572	dsDict = dsDict.copy()
	573	dsDict['dsName'] = strtable + str(j)
	574	dsList.append(dsDict)
	575
	576	# Two-dimension arrays
	577	elif len(arrayDim0) == 2:
	578	arrayDim[i,2:-1] = numpy.array(arrayDim0)
	579	nTables = int(arrayDim[i,2])
	580	dsDict['dsNumber'] = nTables
	581	dsDict['shape'] = arrayDim[i,3]
	582	dsDict['nDim'] = 2
	583
	584	for j in range(nTables):
	585	dsDict = dsDict.copy()
	586	dsDict['dsName'] = strtable + str(j)
	587	dsList.append(dsDict)
	588
	589	# One-dimension arrays
	590	elif len(arrayDim0) == 1:
	591	arrayDim[i,3] = arrayDim0[0]
	592	dsDict['shape'] = arrayDim0[0]
	593	dsDict['dsNumber'] = 1
	594	dsDict['dsName'] = strtable + str(0)
	595	dsDict['nDim'] = 1
	596	dsList.append(dsDict)
	597
	598	table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
	599	tableList.append(table)
	600
	601	self.dsList = dsList
	602	self.tableDim = numpy.array(tableList, dtype = dtype0)
	603	self.blockIndex = 0
	604	timeTuple = time.localtime(dataOut.utctime)
	605	self.currentDay = timeTuple.tm_yday
	606
	607	def putMetadata(self):
	608
	609	fp = self.createMetadataFile()
	610	self.writeMetadata(fp)
	611	fp.close()
	612	return
	613
	614	def createMetadataFile(self):
	615	ext = self.ext
	616	path = self.path
	617	setFile = self.setFile
	618
	619	timeTuple = time.localtime(self.dataOut.utctime)
	620
	621	subfolder = ''
	622	fullpath = os.path.join( path, subfolder )
	623
	624	if not( os.path.exists(fullpath) ):
	625	os.mkdir(fullpath)
	626	setFile = -1 #inicializo mi contador de seteo
	627
	628	subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
	629	fullpath = os.path.join( path, subfolder )
	630
	631	if not( os.path.exists(fullpath) ):
	632	os.mkdir(fullpath)
	633	setFile = -1 #inicializo mi contador de seteo
	634
	635	else:
	636	filesList = os.listdir( fullpath )
	637	filesList = sorted( filesList, key=str.lower )
	638	if len( filesList ) > 0:
	639	filesList = [k for k in filesList if k.startswith(self.metaoptchar)]
	640	filen = filesList[-1]
	641	# el filename debera tener el siguiente formato
	642	# 0 1234 567 89A BCDE (hex)
	643	# x YYYY DDD SSS .ext
	644	if isNumber( filen[8:11] ):
	645	setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
	646	else:
	647	setFile = -1
	648	else:
	649	setFile = -1 #inicializo mi contador de seteo
	650
	651	if self.setType is None:
	652	setFile += 1
	653	file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
	654	timeTuple.tm_year,
	655	timeTuple.tm_yday,
	656	setFile,
	657	ext )
	658	else:
	659	setFile = timeTuple.tm_hour*60+timeTuple.tm_min
	660	file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
	661	timeTuple.tm_year,
	662	timeTuple.tm_yday,
	663	setFile,
	664	ext )
	665
	666	filename = os.path.join( path, subfolder, file )
	667	self.metaFile = file
	668	#Setting HDF5 File
	669	fp = h5py.File(filename,'w')
	670
	671	return fp
	672
	673	def writeMetadata(self, fp):
	674
	675	grp = fp.create_group("Metadata")
	676	grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
	677
	678	for i in range(len(self.metadataList)):
	679	grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
	680	return
	681
	682	def timeFlag(self):
	683	currentTime = self.dataOut.utctime
	684
	685	if self.lastTime is None:
	686	self.lastTime = currentTime
	687
	688	#Day
	689	timeTuple = time.localtime(currentTime)
	690	dataDay = timeTuple.tm_yday
	691
	692	#Time
	693	timeDiff = currentTime - self.lastTime
	694
	695	#Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
	696	if dataDay != self.currentDay:
	697	self.currentDay = dataDay
	698	return True
	699	elif timeDiff > 36060:
	700	self.lastTime = currentTime
	701	return True
	702	else:
	703	self.lastTime = currentTime
	704	return False
	705
	706	def setNextFile(self):
	707
	708	ext = self.ext
	709	path = self.path
	710	setFile = self.setFile
	711	mode = self.mode
	712
	713	timeTuple = time.localtime(self.dataOut.utctime)
	714	subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
	715
	716	fullpath = os.path.join( path, subfolder )
	717
	718	if os.path.exists(fullpath):
	719	filesList = os.listdir( fullpath )
	720	filesList = [k for k in filesList if 'M' in k]
	721	if len( filesList ) > 0:
	722	filesList = sorted( filesList, key=str.lower )
	723	filen = filesList[-1]
	724	# el filename debera tener el siguiente formato
	725	# 0 1234 567 89A BCDE (hex)
	726	# x YYYY DDD SSS .ext
	727	if isNumber( filen[8:11] ):
	728	setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
	729	else:
	730	setFile = -1
	731	else:
	732	setFile = -1 #inicializo mi contador de seteo
	733	else:
	734	os.makedirs(fullpath)
	735	setFile = -1 #inicializo mi contador de seteo
	736
	737	if self.setType is None:
	738	setFile += 1
	739	file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
	740	timeTuple.tm_year,
	741	timeTuple.tm_yday,
	742	setFile,
	743	ext )
	744	else:
	745	setFile = timeTuple.tm_hour*60+timeTuple.tm_min
	746	file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
	747	timeTuple.tm_year,
	748	timeTuple.tm_yday,
	749	setFile,
	750	ext )
	751
	752	filename = os.path.join( path, subfolder, file )
	753
	754	#Setting HDF5 File
	755	fp = h5py.File(filename,'w')
	756	#write metadata
	757	self.writeMetadata(fp)
	758	#Write data
	759	grp = fp.create_group("Data")
	760	ds = []
	761	data = []
	762	dsList = self.dsList
	763	i = 0
	764	while i < len(dsList):
	765	dsInfo = dsList[i]
	766	#One-dimension data
	767	if dsInfo['mode'] == 0:
	768	ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
	769	ds.append(ds0)
	770	data.append([])
	771	i += 1
	772	continue
	773
	774	elif dsInfo['mode'] == 2:
	775	grp0 = grp.create_group(dsInfo['variable'])
	776	ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
	777	ds.append(ds0)
	778	data.append([])
	779	i += 1
	780	continue
	781
	782	elif dsInfo['mode'] == 1:
	783	grp0 = grp.create_group(dsInfo['variable'])
	784
	785	for j in range(dsInfo['dsNumber']):
	786	dsInfo = dsList[i]
	787	tableName = dsInfo['dsName']
	788
	789
	790	if dsInfo['nDim'] == 3:
	791	shape = dsInfo['shape'].astype(int)
	792	ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
	793	else:
	794	shape = int(dsInfo['shape'])
	795	ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
	796
	797	ds.append(ds0)
	798	data.append([])
	799	i += 1
	800
	801	fp.flush()
	802	fp.close()
	803
	804	log.log('creating file: {}'.format(filename), 'Writing')
	805	self.filename = filename
	806	self.ds = ds
	807	self.data = data
	808	self.firsttime = True
	809	self.blockIndex = 0
	810	return
	811
	812	def putData(self):
	813
	814	if self.blockIndex == self.blocksPerFile or self.timeFlag():
	815	self.setNextFile()
	816
	817	self.readBlock()
	818	self.setBlock() #Prepare data to be written
	819	self.writeBlock() #Write data
	820
	821	return
	822
	823	def readBlock(self):
	824
	825	'''
	826	data Array configured
	827
	828
	829	self.data
	830	'''
	831	dsList = self.dsList
	832	ds = self.ds
	833	#Setting HDF5 File
	834	fp = h5py.File(self.filename,'r+')
	835	grp = fp["Data"]
	836	ind = 0
	837
	838	while ind < len(dsList):
	839	dsInfo = dsList[ind]
	840
	841	if dsInfo['mode'] == 0:
	842	ds0 = grp[dsInfo['variable']]
	843	ds[ind] = ds0
	844	ind += 1
	845	else:
	846
	847	grp0 = grp[dsInfo['variable']]
	848
	849	for j in range(dsInfo['dsNumber']):
	850	dsInfo = dsList[ind]
	851	ds0 = grp0[dsInfo['dsName']]
	852	ds[ind] = ds0
	853	ind += 1
	854
	855	self.fp = fp
	856	self.grp = grp
	857	self.ds = ds
	858
	859	return
	860
	861	def setBlock(self):
	862	'''
	863	data Array configured
	864
	865
	866	self.data
	867	'''
	868	#Creating Arrays
	869	dsList = self.dsList
	870	data = self.data
	871	ind = 0
	872
	873	while ind < len(dsList):
	874	dsInfo = dsList[ind]
	875	dataAux = getattr(self.dataOut, dsInfo['variable'])
	876
	877	mode = dsInfo['mode']
	878	nDim = dsInfo['nDim']
	879
	880	if mode == 0 or mode == 2 or nDim == 1:
	881	data[ind] = dataAux
	882	ind += 1
	883	# elif nDim == 1:
	884	# data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
	885	# ind += 1
	886	elif nDim == 2:
	887	for j in range(dsInfo['dsNumber']):
	888	data[ind] = dataAux[j,:]
	889	ind += 1
	890	elif nDim == 3:
	891	for j in range(dsInfo['dsNumber']):
	892	data[ind] = dataAux[:,j,:]
	893	ind += 1
	894
	895	self.data = data
	896	return
	897
	898	def writeBlock(self):
	899	'''
	900	Saves the block in the HDF5 file
	901	'''
	902	dsList = self.dsList
	903
	904	for i in range(len(self.ds)):
	905	dsInfo = dsList[i]
	906	nDim = dsInfo['nDim']
	907	mode = dsInfo['mode']
	908
	909	# First time
	910	if self.firsttime:
	911	if type(self.data[i]) == numpy.ndarray:
	912
	913	if nDim == 3:
	914	self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
	915	self.ds[i].resize(self.data[i].shape)
	916	if mode == 2:
	917	self.ds[i].resize(self.data[i].shape)
	918	self.ds[i][:] = self.data[i]
	919	else:
	920
	921	# From second time
	922	# Meteors!
	923	if mode == 2:
	924	dataShape = self.data[i].shape
	925	dsShape = self.ds[i].shape
	926	self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
	927	self.ds[i][dsShape[0]:,:] = self.data[i]
	928	# No dimension
	929	elif mode == 0:
	930	self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
	931	self.ds[i][0,-1] = self.data[i]
	932	# One dimension
	933	elif nDim == 1:
	934	self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
	935	self.ds[i][-1,:] = self.data[i]
	936	# Two dimension
	937	elif nDim == 2:
	938	self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
	939	self.ds[i][self.blockIndex,:] = self.data[i]
	940	# Three dimensions
	941	elif nDim == 3:
	942	self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
	943	self.ds[i][:,:,-1] = self.data[i]
	944
	945	self.firsttime = False
	946	self.blockIndex += 1
	947
	948	#Close to save changes
	949	self.fp.flush()
	950	self.fp.close()
	951	return
	952
	953	def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
	954
	955	self.dataOut = dataOut
	956	if not(self.isConfig):
	957	self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
	958	metadataList=metadataList, dataList=dataList, mode=mode,
	959	setType=setType)
	960
	961	self.isConfig = True
	962	self.setNextFile()
	963
	964	self.putData()
	965	return
	966
	967		78
			79	"""
	968		80
	969	class ParameterReader(Reader, ProcessingUnit):	81	__attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
	970	'''
	971	Reads HDF5 format files
	972	'''
	973		82
	974	def __init__(self):	83	def __init__(self):
	975	ProcessingUnit.__init__(self)	84	ProcessingUnit.__init__(self)
	976	self.dataOut = Parameters()	85	self.dataOut = Parameters()
	977	self.ext = ".hdf5"	86	self.ext = ".hdf5"
	978	self.optchar = "D"	87	self.optchar = "D"
	979	self.~~timezone~~ = ~~"lt"~~	88	self.meta = {}
	980	self.~~listMetaname~~ = []	89	self.data = {}
	981	self.listMeta = []
	982	self.listDataname = []
	983	self.listData = []
	984	self.listShapes = []
	985	self.open_file = h5py.File	90	self.open_file = h5py.File
	986	self.open_mode = 'r'	91	self.open_mode = 'r'
	987	self.~~metadata~~ = ~~False~~	92	self.description = {}
			93	self.extras = {}
	988	self.filefmt = "%Y%j**"	94	self.filefmt = "%Y%j**"
	989	self.folderfmt = "*%Y%j"	95	self.folderfmt = "*%Y%j"
	990		96
	991	def setup(self, **kwargs):	97	def setup(self, **kwargs):
	992		98
	993	self.set_kwargs(**kwargs)	99	self.set_kwargs(**kwargs)
	994	if not self.ext.startswith('.'):	100	if not self.ext.startswith('.'):
	995	self.ext = '.{}'.format(self.ext)	101	self.ext = '.{}'.format(self.ext)
	996		102
	997	if self.online:	103	if self.online:
	998	log.log("Searching files in online mode...", self.name)	104	log.log("Searching files in online mode...", self.name)
	999		105
	1000	for nTries in range(self.nTries):	106	for nTries in range(self.nTries):
	1001	fullpath = self.searchFilesOnLine(self.path, self.startDate,	107	fullpath = self.searchFilesOnLine(self.path, self.startDate,
	1002	self.endDate, self.expLabel, self.ext, self.walk,	108	self.endDate, self.expLabel, self.ext, self.walk,
	1003	self.filefmt, self.folderfmt)	109	self.filefmt, self.folderfmt)
	1004
	1005	try:	110	try:
	1006	fullpath = next(fullpath)	111	fullpath = next(fullpath)
	1007	except:	112	except:
	1008	fullpath = None	113	fullpath = None
	1009		114
	1010	if fullpath:	115	if fullpath:
	1011	break	116	break
	1012		117
	1013	log.warning(	118	log.warning(
	1014	'Waiting {} sec for a valid file in {}: try {} ...'.format(	119	'Waiting {} sec for a valid file in {}: try {} ...'.format(
	1015	self.delay, self.path, nTries + 1),	120	self.delay, self.path, nTries + 1),
	1016	self.name)	121	self.name)
	1017	time.sleep(self.delay)	122	time.sleep(self.delay)
	1018		123
	1019	if not(fullpath):	124	if not(fullpath):
	1020	raise schainpy.admin.SchainError(	125	raise schainpy.admin.SchainError(
	1021	'There isn\'t any valid file in {}'.format(self.path))	126	'There isn\'t any valid file in {}'.format(self.path))
	1022		127
	1023	pathname, filename = os.path.split(fullpath)	128	pathname, filename = os.path.split(fullpath)
	1024	self.year = int(filename[1:5])	129	self.year = int(filename[1:5])
	1025	self.doy = int(filename[5:8])	130	self.doy = int(filename[5:8])
	1026	self.set = int(filename[8:11]) - 1	131	self.set = int(filename[8:11]) - 1
	1027	else:	132	else:
	1028	log.log("Searching files in {}".format(self.path), self.name)	133	log.log("Searching files in {}".format(self.path), self.name)
	1029	self.filenameList = self.searchFilesOffLine(self.path, self.startDate,	134	self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
	1030	self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)	135	self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
	1031		136
	1032	self.setNextFile()	137	self.setNextFile()
	1033		138
	1034	return	139	return
	1035		140
	1036	def readFirstHeader(self):	141	def readFirstHeader(self):
	1037	'''Read metadata and data'''	142	'''Read metadata and data'''
	1038		143
	1039	self.__readMetadata()	144	self.__readMetadata()
	1040	self.__readData()	145	self.__readData()
	1041	self.__setBlockList()	146	self.__setBlockList()
			147
			148	if 'type' in self.meta:
			149	self.dataOut = eval(self.meta['type'])()
			150
			151	for attr in self.meta:
			152	setattr(self.dataOut, attr, self.meta[attr])
			153
	1042	self.blockIndex = 0	154	self.blockIndex = 0
	1043		155
	1044	return	156	return
	1045		157
	1046	def __setBlockList(self):	158	def __setBlockList(self):
	1047	'''	159	'''
	1048	Selects the data within the times defined	160	Selects the data within the times defined
	1049		161
	1050	self.fp	162	self.fp
	1051	self.startTime	163	self.startTime
	1052	self.endTime	164	self.endTime
	1053	self.blockList	165	self.blockList
	1054	self.blocksPerFile	166	self.blocksPerFile
	1055		167
	1056	'''	168	'''
	1057		169
	1058	startTime = self.startTime	170	startTime = self.startTime
	1059	endTime = self.endTime	171	endTime = self.endTime
	1060		172
	1061	~~index~~ = self.~~listDataname~~.~~index~~('utctime')	173	thisUtcTime = self.data['utctime']
	1062	thisUtcTime = self.listData[index]
	1063	self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])	174	self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
	1064		175
	1065	if self.timezone == 'lt':	176	thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
	1066	thisUtcTime -= 5*3600
	1067
	1068	thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
	1069		177
	1070	thisDate = thisDatetime.date()	178	thisDate = thisDatetime.date()
	1071	thisTime = thisDatetime.time()	179	thisTime = thisDatetime.time()
	1072		180
	1073	startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()	181	startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
	1074	endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()	182	endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
	1075		183
	1076	ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]	184	ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
	1077		185
	1078	self.blockList = ind	186	self.blockList = ind
	1079	self.blocksPerFile = len(ind)	187	self.blocksPerFile = len(ind)
	1080	return	188	return
	1081		189
	1082	def __readMetadata(self):	190	def __readMetadata(self):
	1083	'''	191	'''
	1084	Reads Metadata	192	Reads Metadata
	1085	'''	193	'''
	1086		194
	1087	~~listMetaname~~ = []	195	meta = {}
	1088	listMetadata = []	196
	1089	if 'Metadata' in self.fp:	197	if self.description:
	1090	gp = self.fp['Metadata']	198	for key, value in self.description['Metadata'].items():
	1091	for item in list(gp.items()):	199	meta[key] = self.fp[value].value
	1092	name = item[0]
	1093
	1094	if name=='variables':
	1095	table = gp[name][:]
	1096	listShapes = {}
	1097	for shapes in table:
	1098	listShapes[shapes[0].decode()] = numpy.array([shapes[1]])
	1099	else:
	1100	data = gp[name].value
	1101	listMetaname.append(name)
	1102	listMetadata.append(data)
	1103	elif self.metadata:
	1104	metadata = json.loads(self.metadata)
	1105	listShapes = {}
	1106	for tup in metadata:
	1107	name, values, dim = tup
	1108	if dim == -1:
	1109	listMetaname.append(name)
	1110	listMetadata.append(self.fp[values].value)
	1111	else:
	1112	listShapes[name] = numpy.array([dim])
	1113	else:	200	else:
	1114	raise IOError('Missing Metadata group in file or metadata info')	201	grp = self.fp['Metadata']
			202	for name in grp:
			203	meta[name] = grp[name].value
	1115		204
	1116	self.listShapes = listShapes	205	if self.extras:
	1117	self.listMetaname = listMetaname	206	for key, value in self.extras.items():
	1118	self.listMeta = listMetadata	207	meta[key] = value
			208	self.meta = meta
	1119		209
	1120	return	210	return
	1121		211
	1122	def __readData(self):	212	def __readData(self):
	1123		213
	1124	~~list~~data~~name~~ = []	214	data = {}
	1125	listdata = []
	1126		215
	1127	if ~~'Data'~~ in self.fp:	216	if self.description:
			217	for key, value in self.description['Data'].items():
			218	if isinstance(value, str):
			219	if isinstance(self.fp[value], h5py.Dataset):
			220	data[key] = self.fp[value].value
			221	elif isinstance(self.fp[value], h5py.Group):
			222	array = []
			223	for ch in self.fp[value]:
			224	array.append(self.fp[value][ch].value)
			225	data[key] = numpy.array(array)
			226	elif isinstance(value, list):
			227	array = []
			228	for ch in value:
			229	array.append(self.fp[ch].value)
			230	data[key] = numpy.array(array)
			231	else:
	1128	grp = self.fp['Data']	232	grp = self.fp['Data']
	1129	for ~~item~~ in ~~list~~(grp.~~items())~~:	233	for name in grp:
	1130	name = item[0]	234	if isinstance(grp[name], h5py.Dataset):
	1131	listdataname.append(name)
	1132	dim = self.listShapes[name][0]
	1133	if dim == 0:
	1134	array = grp[name].value	235	array = grp[name].value
	1135	else:	236	elif isinstance(grp[name], h5py.Group):
	1136	array = []	237	array = []
	1137	for i in ~~range~~(~~dim~~):	238	for ch in grp[name]:
	1138	array.append(grp[name][~~'table{:02d}'~~.~~format~~(i)].value)	239	array.append(grp[name][ch].value)
	1139	array = numpy.array(array)	240	array = numpy.array(array)
	1140
	1141	listdata.append(array)
	1142	elif self.metadata:
	1143	metadata = json.loads(self.metadata)
	1144	for tup in metadata:
	1145	name, values, dim = tup
	1146	listdataname.append(name)
	1147	if dim == -1:
	1148	continue
	1149	elif dim == 0:
	1150	array = self.fp[values].value
	1151	else:	241	else:
	1152	array = []	242	log.warning('Unknown type: {}'.format(name))
	1153	for var in values:	243
	1154	array.append(self.fp[var].value)	244	if name in self.description:
	1155	array = numpy.array(array)	245	key = self.description[name]
	1156	~~listdata~~.~~append~~(~~array~~)	246	else:
	1157	else:	247	key = name
	1158	raise IOError('Missing Data group in file or metadata info')	248	data[key] = array
	1159		249
	1160	self.listDataname = listdataname	250	self.data = data
	1161	self.listData = listdata
	1162	return	251	return
	1163		252
	1164	def getData(self):	253	def getData(self):
	1165		254
	1166	for i in ~~range~~(~~len~~(self.~~listMeta~~)):	255	for attr in self.data:
	1167	setattr(self.dataOut, self.listMetaname[i], self.listMeta[i])	256	if self.data[attr].ndim == 1:
	1168		257	setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
	1169	for j in range(len(self.listData)):
	1170	dim = self.listShapes[self.listDataname[j]][0]
	1171	if dim == 0:
	1172	setattr(self.dataOut, self.listDataname[j], self.listData[j][self.blockIndex])
	1173	else:	258	else:
	1174	setattr(self.dataOut, ~~self~~.~~listDataname~~[j], self.~~listD~~ata[j][:,self.blockIndex])	259	setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
	1175		260
	1176	self.dataOut.paramInterval = self.interval
	1177	self.dataOut.flagNoData = False	261	self.dataOut.flagNoData = False
	1178	self.blockIndex += 1	262	self.blockIndex += 1
	1179		263
			264	log.log("Block No. {}/{} -> {}".format(
			265	self.blockIndex,
			266	self.blocksPerFile,
			267	self.dataOut.datatime.ctime()), self.name)
			268
	1180	return	269	return
	1181		270
	1182	def run(self, **kwargs):	271	def run(self, **kwargs):
	1183		272
	1184	if not(self.isConfig):	273	if not(self.isConfig):
	1185	self.setup(**kwargs)	274	self.setup(**kwargs)
	1186	self.isConfig = True	275	self.isConfig = True
	1187		276
	1188	if self.blockIndex == self.blocksPerFile:	277	if self.blockIndex == self.blocksPerFile:
	1189	self.setNextFile()	278	self.setNextFile()
	1190		279
	1191	self.getData()	280	self.getData()
	1192		281
	1193	return	282	return
	1194		283
	1195	@MPDecorator	284	@MPDecorator
	1196	class ~~Parameter~~Writer(Operation):	285	class HDFWriter(Operation):
	1197	'''	286	"""Operation to write HDF5 files.
	1198	HDF5 Writer, stores parameters data in HDF5 format files	287
	1199		288	The HDF5 file contains by default two groups Data and Metadata where
	1200	path: path where the files will be stored	289	you can save any `dataOut` attribute specified by `dataList` and `metadataList`
	1201	blocksPerFile: number of blocks that will be saved in per HDF5 format file	290	parameters, data attributes are normaly time dependent where the metadata
	1202	mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)	291	are not.
	1203	metadataList: list of attributes that will be stored as metadata	292	It is possible to customize the structure of the HDF5 file with the
	1204	dataList: list of attributes that will be stores as data	293	optional description parameter see the examples.
	1205	'''	294
			295	Parameters:
			296	-----------
			297	path : str
			298	Path where files will be saved.
			299	blocksPerFile : int
			300	Number of blocks per file
			301	metadataList : list
			302	List of the dataOut attributes that will be saved as metadata
			303	dataList : int
			304	List of the dataOut attributes that will be saved as data
			305	setType : bool
			306	If True the name of the files corresponds to the timestamp of the data
			307	description : dict, optional
			308	Dictionary with the desired description of the HDF5 file
			309
			310	Examples
			311	--------
			312
			313	desc = {
			314	'data_output': {'winds': ['z', 'w', 'v']},
			315	'utctime': 'timestamps',
			316	'heightList': 'heights'
			317	}
			318	desc = {
			319	'data_output': ['z', 'w', 'v'],
			320	'utctime': 'timestamps',
			321	'heightList': 'heights'
			322	}
			323	desc = {
			324	'Data': {
			325	'data_output': 'winds',
			326	'utctime': 'timestamps'
			327	},
			328	'Metadata': {
			329	'heightList': 'heights'
			330	}
			331	}
			332
			333	writer = proc_unit.addOperation(name='HDFWriter')
			334	writer.addParameter(name='path', value='/path/to/file')
			335	writer.addParameter(name='blocksPerFile', value='32')
			336	writer.addParameter(name='metadataList', value='heightList,timeZone')
			337	writer.addParameter(name='dataList',value='data_output,utctime')
			338	# writer.addParameter(name='description',value=json.dumps(desc))
	1206		339
			340	"""
	1207		341
	1208	ext = ".hdf5"	342	ext = ".hdf5"
	1209	optchar = "D"	343	optchar = "D"
	1210	metaoptchar = "M"
	1211	metaFile = None
	1212	filename = None	344	filename = None
	1213	path = None	345	path = None
	1214	setFile = None	346	setFile = None
	1215	fp = None	347	fp = None
	1216	grp = None
	1217	ds = None
	1218	firsttime = True	348	firsttime = True
	1219	#Configurations	349	#Configurations
	1220	blocksPerFile = None	350	blocksPerFile = None
	1221	blockIndex = None	351	blockIndex = None
	1222	dataOut = None	352	dataOut = None
	1223	#Data Arrays	353	#Data Arrays
	1224	dataList = None	354	dataList = None
	1225	metadataList = None	355	metadataList = None
	1226	dsList = None #List of dictionaries with dataset properties
	1227	tableDim = None
	1228	dtype = [('name', 'S20'),('nDim', 'i')]
	1229	currentDay = None	356	currentDay = None
	1230	lastTime = None	357	lastTime = None
	1231		358
	1232	def __init__(self):	359	def __init__(self):
	1233		360
	1234	Operation.__init__(self)	361	Operation.__init__(self)
	1235	return	362	return
	1236		363
	1237	def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None):	364	def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
	1238	self.path = path	365	self.path = path
	1239	self.blocksPerFile = blocksPerFile	366	self.blocksPerFile = blocksPerFile
	1240	self.metadataList = metadataList	367	self.metadataList = metadataList
	1241	self.dataList = dataList	368	self.dataList = dataList
	1242	self.setType = setType	369	self.setType = setType
			370	self.description = description
			371
			372	for s in ['type', 'timeZone', 'useLocalTime']:
			373	if s not in self.metadataList:
			374	self.metadataList.append(s)
	1243		375
	1244	tableList = []	376	tableList = []
	1245	dsList = []	377	dsList = []
	1246		378
	1247	for i in range(len(self.dataList)):	379	for i in range(len(self.dataList)):
	1248	dsDict = {}	380	dsDict = {}
	1249	dataAux = getattr(self.dataOut, self.dataList[i])	381	dataAux = getattr(self.dataOut, self.dataList[i])
	1250	dsDict['variable'] = self.dataList[i]	382	dsDict['variable'] = self.dataList[i]
	1251		383
	1252	if dataAux is None:	384	if dataAux is None:
	1253	continue	385	continue
	1254	elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):	386	elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
	1255	dsDict['nDim'] = 0	387	dsDict['nDim'] = 0
	1256	else:	388	else:
	1257	dsDict['nDim'] = len(dataAux.shape)	389	dsDict['nDim'] = len(dataAux.shape)
	1258	dsDict['shape'] = dataAux.shape	390	dsDict['shape'] = dataAux.shape
	1259	dsDict['dsNumber'] = dataAux.shape[0]	391	dsDict['dsNumber'] = dataAux.shape[0]
			392	dsDict['dtype'] = dataAux.dtype
	1260		393
	1261	dsList.append(dsDict)	394	dsList.append(dsDict)
	1262	tableList.append((self.dataList[i], dsDict['nDim']))
	1263		395
	1264	self.dsList = dsList	396	self.dsList = dsList
	1265	self.tableDim = numpy.array(tableList, dtype=self.dtype)
	1266	self.currentDay = self.dataOut.datatime.date()	397	self.currentDay = self.dataOut.datatime.date()
	1267		398
	1268	def timeFlag(self):	399	def timeFlag(self):
	1269	currentTime = self.dataOut.utctime	400	currentTime = self.dataOut.utctime
	1270	timeTuple = time.localtime(currentTime)	401	timeTuple = time.localtime(currentTime)
	1271	dataDay = timeTuple.tm_yday	402	dataDay = timeTuple.tm_yday
	1272		403
	1273	if self.lastTime is None:	404	if self.lastTime is None:
	1274	self.lastTime = currentTime	405	self.lastTime = currentTime
	1275	self.currentDay = dataDay	406	self.currentDay = dataDay
	1276	return False	407	return False
	1277		408
	1278	timeDiff = currentTime - self.lastTime	409	timeDiff = currentTime - self.lastTime
	1279		410
	1280	#Si el dia es diferente o si la diferencia entre un dato y otro supera la hora	411	#Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
	1281	if dataDay != self.currentDay:	412	if dataDay != self.currentDay:
	1282	self.currentDay = dataDay	413	self.currentDay = dataDay
	1283	return True	414	return True
	1284	elif timeDiff > 36060:	415	elif timeDiff > 36060:
	1285	self.lastTime = currentTime	416	self.lastTime = currentTime
	1286	return True	417	return True
	1287	else:	418	else:
	1288	self.lastTime = currentTime	419	self.lastTime = currentTime
	1289	return False	420	return False
	1290		421
	1291	def run(self, dataOut, path, blocksPerFile=10, metadataList=~~None~~, ~~dataList~~=~~None~~, ~~setType~~=~~None~~):	422	def run(self, dataOut, path, blocksPerFile=10, metadataList=[],
			423	dataList=[], setType=None, description={}):
	1292		424
	1293	self.dataOut = dataOut	425	self.dataOut = dataOut
	1294	if not(self.isConfig):	426	if not(self.isConfig):
	1295	self.setup(path=path, blocksPerFile=blocksPerFile,	427	self.setup(path=path, blocksPerFile=blocksPerFile,
	1296	metadataList=metadataList, dataList=dataList,	428	metadataList=metadataList, dataList=dataList,
	1297	setType=setType)	429	setType=setType, description=description)
	1298		430
	1299	self.isConfig = True	431	self.isConfig = True
	1300	self.setNextFile()	432	self.setNextFile()
	1301		433
	1302	self.putData()	434	self.putData()
	1303	return	435	return
	1304		436
	1305	def setNextFile(self):	437	def setNextFile(self):
	1306		438
	1307	ext = self.ext	439	ext = self.ext
	1308	path = self.path	440	path = self.path
	1309	setFile = self.setFile	441	setFile = self.setFile
	1310		442
	1311	timeTuple = time.localtime(self.dataOut.utctime)	443	timeTuple = time.localtime(self.dataOut.utctime)
	1312	subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)	444	subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
	1313	fullpath = os.path.join(path, subfolder)	445	fullpath = os.path.join(path, subfolder)
	1314		446
	1315	if os.path.exists(fullpath):	447	if os.path.exists(fullpath):
	1316	filesList = os.listdir(fullpath)	448	filesList = os.listdir(fullpath)
	1317	filesList = [k for k in filesList if k.startswith(self.optchar)]	449	filesList = [k for k in filesList if k.startswith(self.optchar)]
	1318	if len( filesList ) > 0:	450	if len( filesList ) > 0:
	1319	filesList = sorted(filesList, key=str.lower)	451	filesList = sorted(filesList, key=str.lower)
	1320	filen = filesList[-1]	452	filen = filesList[-1]
	1321	# el filename debera tener el siguiente formato	453	# el filename debera tener el siguiente formato
	1322	# 0 1234 567 89A BCDE (hex)	454	# 0 1234 567 89A BCDE (hex)
	1323	# x YYYY DDD SSS .ext	455	# x YYYY DDD SSS .ext
	1324	if isNumber(filen[8:11]):	456	if isNumber(filen[8:11]):
	1325	setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file	457	setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
	1326	else:	458	else:
	1327	setFile = -1	459	setFile = -1
	1328	else:	460	else:
	1329	setFile = -1 #inicializo mi contador de seteo	461	setFile = -1 #inicializo mi contador de seteo
	1330	else:	462	else:
	1331	os.makedirs(fullpath)	463	os.makedirs(fullpath)
	1332	setFile = -1 #inicializo mi contador de seteo	464	setFile = -1 #inicializo mi contador de seteo
	1333		465
	1334	if self.setType is None:	466	if self.setType is None:
	1335	setFile += 1	467	setFile += 1
	1336	file = '%s%4.4d%3.3d%03d%s' % (self.optchar,	468	file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
	1337	timeTuple.tm_year,	469	timeTuple.tm_year,
	1338	timeTuple.tm_yday,	470	timeTuple.tm_yday,
	1339	setFile,	471	setFile,
	1340	ext )	472	ext )
	1341	else:	473	else:
	1342	setFile = timeTuple.tm_hour*60+timeTuple.tm_min	474	setFile = timeTuple.tm_hour*60+timeTuple.tm_min
	1343	file = '%s%4.4d%3.3d%04d%s' % (self.optchar,	475	file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
	1344	timeTuple.tm_year,	476	timeTuple.tm_year,
	1345	timeTuple.tm_yday,	477	timeTuple.tm_yday,
	1346	setFile,	478	setFile,
	1347	ext )	479	ext )
	1348		480
	1349	self.filename = os.path.join( path, subfolder, file )	481	self.filename = os.path.join( path, subfolder, file )
	1350		482
	1351	#Setting HDF5 File	483	#Setting HDF5 File
	1352	self.fp = h5py.File(self.filename, 'w')	484	self.fp = h5py.File(self.filename, 'w')
	1353	#write metadata	485	#write metadata
	1354	self.writeMetadata(self.fp)	486	self.writeMetadata(self.fp)
	1355	#Write data	487	#Write data
	1356	self.writeData(self.fp)	488	self.writeData(self.fp)
	1357		489
			490	def getLabel(self, name, x=None):
			491
			492	if x is None:
			493	if 'Data' in self.description:
			494	data = self.description['Data']
			495	if 'Metadata' in self.description:
			496	data.update(self.description['Metadata'])
			497	else:
			498	data = self.description
			499	if name in data:
			500	if isinstance(data[name], str):
			501	return data[name]
			502	elif isinstance(data[name], list):
			503	return None
			504	elif isinstance(data[name], dict):
			505	for key, value in data[name].items():
			506	return key
			507	return name
			508	else:
			509	if 'Metadata' in self.description:
			510	meta = self.description['Metadata']
			511	else:
			512	meta = self.description
			513	if name in meta:
			514	if isinstance(meta[name], list):
			515	return meta[name][x]
			516	elif isinstance(meta[name], dict):
			517	for key, value in meta[name].items():
			518	return value[x]
			519	return 'channel{:02d}'.format(x)
			520
	1358	def writeMetadata(self, fp):	521	def writeMetadata(self, fp):
	1359		522
	1360	grp = fp.create_group("Metadata")	523	if self.description:
	1361	grp.create_dataset('variables', data=self.tableDim, dtype=self.dtype)	524	if 'Metadata' in self.description:
			525	grp = fp.create_group('Metadata')
			526	else:
			527	grp = fp
			528	else:
			529	grp = fp.create_group('Metadata')
	1362		530
	1363	for i in range(len(self.metadataList)):	531	for i in range(len(self.metadataList)):
	1364	if not hasattr(self.dataOut, self.metadataList[i]):	532	if not hasattr(self.dataOut, self.metadataList[i]):
	1365	log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)	533	log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
	1366	continue	534	continue
	1367	value = getattr(self.dataOut, self.metadataList[i])	535	value = getattr(self.dataOut, self.metadataList[i])
	1368	grp.create_dataset(self.metadataList[i], data=value)	536	if isinstance(value, bool):
			537	if value is True:
			538	value = 1
			539	else:
			540	value = 0
			541	grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
	1369	return	542	return
	1370		543
	1371	def writeData(self, fp):	544	def writeData(self, fp):
	1372		545
	1373	grp = fp.create_group("Data")	546	if self.description:
			547	if 'Data' in self.description:
			548	grp = fp.create_group('Data')
			549	else:
			550	grp = fp
			551	else:
			552	grp = fp.create_group('Data')
			553
	1374	dtsets = []	554	dtsets = []
	1375	data = []	555	data = []
	1376		556
	1377	for dsInfo in self.dsList:	557	for dsInfo in self.dsList:
	1378	if dsInfo['nDim'] == 0:	558	if dsInfo['nDim'] == 0:
	1379	ds = grp.create_dataset(	559	ds = grp.create_dataset(
	1380	dsInfo['variable'],	560	self.getLabel(dsInfo['variable']),
	1381	(self.blocksPerFile, ),	561	(self.blocksPerFile, ),
	1382	chunks=True,	562	chunks=True,
	1383	dtype=numpy.float64)	563	dtype=numpy.float64)
	1384	dtsets.append(ds)	564	dtsets.append(ds)
	1385	data.append((dsInfo['variable'], -1))	565	data.append((dsInfo['variable'], -1))
	1386	else:	566	else:
	1387	~~sgrp~~ = ~~grp~~.~~create_group~~(dsInfo['variable'])	567	label = self.getLabel(dsInfo['variable'])
			568	if label is not None:
			569	sgrp = grp.create_group(label)
			570	else:
			571	sgrp = grp
	1388	for i in range(dsInfo['dsNumber']):	572	for i in range(dsInfo['dsNumber']):
	1389	ds = sgrp.create_dataset(	573	ds = sgrp.create_dataset(
	1390	'~~table{:02d}~~'.~~format~~(i),	574	self.getLabel(dsInfo['variable'], i),
	1391	(self.blocksPerFile, ) + dsInfo['shape'][1:],	575	(self.blocksPerFile, ) + dsInfo['shape'][1:],
	1392	chunks=True)	576	chunks=True,
			577	dtype=dsInfo['dtype'])
	1393	dtsets.append(ds)	578	dtsets.append(ds)
	1394	data.append((dsInfo['variable'], i))	579	data.append((dsInfo['variable'], i))
	1395	fp.flush()	580	fp.flush()
	1396		581
	1397	log.log('Creating file: {}'.format(fp.filename), self.name)	582	log.log('Creating file: {}'.format(fp.filename), self.name)
	1398		583
	1399	self.ds = dtsets	584	self.ds = dtsets
	1400	self.data = data	585	self.data = data
	1401	self.firsttime = True	586	self.firsttime = True
	1402	self.blockIndex = 0	587	self.blockIndex = 0
	1403	return	588	return
	1404		589
	1405	def putData(self):	590	def putData(self):
	1406		591
	1407	if (self.blockIndex == self.blocksPerFile) or self.timeFlag():	592	if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
	1408	self.closeFile()	593	self.closeFile()
	1409	self.setNextFile()	594	self.setNextFile()
	1410		595
	1411	for i, ds in enumerate(self.ds):	596	for i, ds in enumerate(self.ds):
	1412	attr, ch = self.data[i]	597	attr, ch = self.data[i]
	1413	if ch == -1:	598	if ch == -1:
	1414	ds[self.blockIndex] = getattr(self.dataOut, attr)	599	ds[self.blockIndex] = getattr(self.dataOut, attr)
	1415	else:	600	else:
	1416	ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]	601	ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
	1417		602
	1418	self.fp.flush()	603	self.fp.flush()
	1419	self.blockIndex += 1	604	self.blockIndex += 1
	1420	log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)	605	log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
	1421		606
	1422	return	607	return
	1423		608
	1424	def closeFile(self):	609	def closeFile(self):
	1425		610
	1426	if self.blockIndex != self.blocksPerFile:	611	if self.blockIndex != self.blocksPerFile:
	1427	for ds in self.ds:	612	for ds in self.ds:
	1428	ds.resize(self.blockIndex, axis=0)	613	ds.resize(self.blockIndex, axis=0)
	1429		614
	1430	self.fp.flush()	615	self.fp.flush()
	1431	self.fp.close()	616	self.fp.close()
	1432		617
	1433	def close(self):	618	def close(self):
	1434		619
	1435	self.closeFile()	620	self.closeFile()

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages