schain Commit - r684:ab5af7e4405a · Jicamarca Repository

-Using ValueError raises instead of IOError...

Miguel Valdez -

r684:ab5af7e4405a

parent child

Context file:

r684:ab5af7e4405a

Collapse all files

schainpy/model/data/jrodata.py +3 -6

             '''
             $Author: murco $
             $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
             '''
             import copy
             import numpy
             import datetime
             from jroheaderIO import SystemHeader, RadarControllerHeader
             def getNumpyDtype(dataTypeCode):
                 if dataTypeCode == 0:
                     numpyDtype = numpy.dtype([('real','<i1'),('imag','<i1')])
                 elif dataTypeCode == 1:
                     numpyDtype = numpy.dtype([('real','<i2'),('imag','<i2')])
                 elif dataTypeCode == 2:
                     numpyDtype = numpy.dtype([('real','<i4'),('imag','<i4')])
                 elif dataTypeCode == 3:
                     numpyDtype = numpy.dtype([('real','<i8'),('imag','<i8')])
                 elif dataTypeCode == 4:
                     numpyDtype = numpy.dtype([('real','<f4'),('imag','<f4')])
                 elif dataTypeCode == 5:
                     numpyDtype = numpy.dtype([('real','<f8'),('imag','<f8')])
                 else:
                     raise ValueError, 'dataTypeCode was not defined'
                 return numpyDtype
             def getDataTypeCode(numpyDtype):
                 if numpyDtype == numpy.dtype([('real','<i1'),('imag','<i1')]):
                     datatype = 0
                 elif numpyDtype == numpy.dtype([('real','<i2'),('imag','<i2')]):
                     datatype = 1
                 elif numpyDtype == numpy.dtype([('real','<i4'),('imag','<i4')]):
                     datatype = 2
                 elif numpyDtype == numpy.dtype([('real','<i8'),('imag','<i8')]):
                     datatype = 3
                 elif numpyDtype == numpy.dtype([('real','<f4'),('imag','<f4')]):
                     datatype = 4
                 elif numpyDtype == numpy.dtype([('real','<f8'),('imag','<f8')]):
                     datatype = 5
                 else:
                     datatype = None
                 return datatype
             def hildebrand_sekhon(data, navg):
                 """
                 This method is for the objective determination of the noise level in Doppler spectra. This
                 implementation technique is based on the fact that the standard deviation of the spectral
                 densities is equal to the mean spectral density for white Gaussian noise
                 Inputs:
                     Data    :    heights
                     navg    :    numbers of averages
                 Return:
                     -1        :    any error
                     anoise    :    noise's level
                 """
                 sortdata = numpy.sort(data,axis=None)
                 lenOfData = len(sortdata)
                 nums_min = lenOfData/10
                 if (lenOfData/10) > 2:
                     nums_min = lenOfData/10
                 else:
                     nums_min = 2
                 sump = 0.
                 sumq = 0.
                 j = 0
                 cont = 1
                 while((cont==1)and(j<lenOfData)):
                     sump += sortdata[j]
                     sumq += sortdata[j]**2
                     if j > nums_min:
                         rtest = float(j)/(j-1) + 1.0/navg
                         if ((sumq*j) > (rtest*sump**2)):
                             j = j - 1
                             sump  = sump - sortdata[j]
                             sumq =  sumq - sortdata[j]**2
                             cont = 0
                     j += 1
                 lnoise = sump /j
                 stdv = numpy.sqrt((sumq - lnoise**2)/(j - 1))
                 return lnoise
             class Beam:
                 def __init__(self):
                     self.codeList = []
                     self.azimuthList = []
                     self.zenithList = []
             class GenericData(object):
                 flagNoData = True
                 def __init__(self):
-                    raise ValueError, "This class has not been implemented"
+                    raise NotImplementedError
                 def copy(self, inputObj=None):
                     if inputObj == None:
                         return copy.deepcopy(self)
                     for key in inputObj.__dict__.keys():
                         self.__dict__[key] = inputObj.__dict__[key]
                 def deepcopy(self):
                     return copy.deepcopy(self)
                 def isEmpty(self):
                     return self.flagNoData
             class JROData(GenericData):
             #    m_BasicHeader = BasicHeader()
             #    m_ProcessingHeader = ProcessingHeader()
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
             #    data = None
                 type = None
                 datatype = None #dtype but in string
             #     dtype = None
             #    nChannels = None
             #    nHeights = None
                 nProfiles = None
                 heightList = None
                 channelList = None
                 flagDiscontinuousBlock = False
                 useLocalTime = False
                 utctime = None
                 timeZone = None
                 dstFlag = None
                 errorCount = None
                 blocksize = None
             #     nCode = None
             #
             #     nBaud = None
             #
             #     code = None
                 flagDecodeData = False #asumo q la data no esta decodificada
                 flagDeflipData = False #asumo q la data no esta sin flip
                 flagShiftFFT = False
             #     ippSeconds = None
             #     timeInterval = None
                 nCohInt = None
             #     noise = None
                 windowOfFilter = 1
                 #Speed of ligth
                 C = 3e8
                 frequency = 49.92e6
                 realtime = False
                 beacon_heiIndexList = None
                 last_block = None
                 blocknow = None
                 azimuth = None
                 zenith = None
                 beam = Beam()
                 profileIndex = None
                 def __init__(self):
-                    raise ValueError, "This class has not been implemented"
+                    raise NotImplementedError
                 def getNoise(self):
-                    raise ValueError, "Not implemented"
+                    raise NotImplementedError
                 def getNChannels(self):
                     return len(self.channelList)
                 def getChannelIndexList(self):
                     return range(self.nChannels)
                 def getNHeights(self):
                     return len(self.heightList)
                 def getHeiRange(self, extrapoints=0):
                     heis = self.heightList
             #        deltah = self.heightList[1] - self.heightList[0]
             #
             #        heis.append(self.heightList[-1])
                     return heis
                 def getltctime(self):
                     if self.useLocalTime:
                         return self.utctime - self.timeZone*60
                     return self.utctime
                 def getDatatime(self):
                     datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
                     return datatimeValue
                 def getTimeRange(self):
                     datatime = []
                     datatime.append(self.ltctime)
                     datatime.append(self.ltctime + self.timeInterval+60)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getFmax(self):
                     PRF = 1./(self.ippSeconds * self.nCohInt)
                     fmax = PRF/2.
                     return fmax
                 def getVmax(self):
                     _lambda = self.C/self.frequency
                     vmax = self.getFmax() * _lambda
                     return vmax
                 def get_ippSeconds(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.ippSeconds
                 def set_ippSeconds(self, ippSeconds):
                     '''
                     '''
                     self.radarControllerHeaderObj.ippSeconds = ippSeconds
                     return
                 def get_dtype(self):
                     '''
                     '''
                     return getNumpyDtype(self.datatype)
                 def set_dtype(self, numpyDtype):
                     '''
                     '''
                     self.datatype = getDataTypeCode(numpyDtype)
                 def get_code(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.code
                 def set_code(self, code):
                     '''
                     '''
                     self.radarControllerHeaderObj.code = code
                     return
                 def get_ncode(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.nCode
                 def set_ncode(self, nCode):
                     '''
                     '''
                     self.radarControllerHeaderObj.nCode = nCode
                     return
                 def get_nbaud(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.nBaud
                 def set_nbaud(self, nBaud):
                     '''
                     '''
                     self.radarControllerHeaderObj.nBaud = nBaud
                     return
-            #     def getTimeInterval(self):
-            #         raise IOError, "This method should be implemented inside each Class"
                 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
                 channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
                 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
                 #noise = property(getNoise, "I'm the 'nHeights' property.")
                 datatime = property(getDatatime, "I'm the 'datatime' property")
                 ltctime = property(getltctime, "I'm the 'ltctime' property")
                 ippSeconds = property(get_ippSeconds, set_ippSeconds)
                 dtype = property(get_dtype, set_dtype)
             #     timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
                 code = property(get_code, set_code)
                 nCode = property(get_ncode, set_ncode)
                 nBaud = property(get_nbaud, set_nbaud)
             class Voltage(JROData):
                 #data es un numpy array de 2 dmensiones (canales, alturas)
                 data = None
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.useLocalTime = True
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Voltage"
                     self.data = None
             #         self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.timeZone = None
                     self.dstFlag = None
                     self.errorCount = None
                     self.nCohInt = None
                     self.blocksize = None
                     self.flagDecodeData = False #asumo q la data no esta decodificada
                     self.flagDeflipData = False #asumo q la data no esta sin flip
                     self.flagShiftFFT = False
                     self.flagDataAsBlock = False    #Asumo que la data es leida perfil a perfil
                     self.profileIndex = 0
                 def getNoisebyHildebrand(self, channel = None):
                     """
                     Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                     Return:
                         noiselevel
                     """
                     if channel != None:
                         data = self.data[channel]
                         nChannels = 1
                     else:
                         data = self.data
                         nChannels = self.nChannels
                     noise = numpy.zeros(nChannels)
                     power = data * numpy.conjugate(data)
                     for thisChannel in range(nChannels):
                         if nChannels == 1:
                             daux = power[:].real
                         else:
                             daux = power[thisChannel,:].real
                         noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
                     return noise
                 def getNoise(self, type = 1, channel = None):
                     if type == 1:
                         noise = self.getNoisebyHildebrand(channel)
                     return 10*numpy.log10(noise)
                 def getPower(self, channel = None):
                     if channel != None:
                         data = self.data[channel]
                     else:
                         data = self.data
                     power = data * numpy.conjugate(data)
                     return 10*numpy.log10(power.real)
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt
                     return timeInterval
                 noise = property(getNoise, "I'm the 'nHeights' property.")
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
             class Spectra(JROData):
                 #data es un numpy array de 2 dmensiones (canales, perfiles, alturas)
                 data_spc = None
                 #data es un numpy array de 2 dmensiones (canales, pares, alturas)
                 data_cspc = None
                 #data es un numpy array de 2 dmensiones (canales, alturas)
                 data_dc = None
                 nFFTPoints = None
             #     nPairs = None
                 pairsList = None
                 nIncohInt = None
                 wavelength = None #Necesario para cacular el rango de velocidad desde la frecuencia
                 nCohInt = None #se requiere para determinar el valor de timeInterval
                 ippFactor = None
                 profileIndex = 0
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.useLocalTime = True
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Spectra"
             #        self.data = None
             #         self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.pairsList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.nCohInt = None
                     self.nIncohInt = None
                     self.blocksize = None
                     self.nFFTPoints = None
                     self.wavelength = None
                     self.flagDecodeData = False #asumo q la data no esta decodificada
                     self.flagDeflipData = False #asumo q la data no esta sin flip
                     self.flagShiftFFT = False
                     self.ippFactor = 1
                     #self.noise = None
                     self.beacon_heiIndexList = []
                     self.noise_estimation = None
                 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                     """
                     Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                     Return:
                         noiselevel
                     """
                     noise = numpy.zeros(self.nChannels)
                     for channel in range(self.nChannels):
                         daux = self.data_spc[channel,xmin_index:xmax_index,ymin_index:ymax_index]
                         noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
                     return noise
                 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                     if self.noise_estimation != None:
                         return self.noise_estimation #this was estimated by getNoise Operation defined in jroproc_spectra.py
                     else:
                         noise = self.getNoisebyHildebrand(xmin_index, xmax_index, ymin_index, ymax_index)
                         return noise
                 def getFreqRange(self, extrapoints=0):
                     deltafreq = self.getFmax() / (self.nFFTPoints*self.ippFactor)
                     freqrange = deltafreq*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltafreq/2
                     return freqrange
                 def getVelRange(self, extrapoints=0):
                     deltav = self.getVmax() / (self.nFFTPoints*self.ippFactor)
                     velrange = deltav*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltav/2
                     return velrange
                 def getNPairs(self):
                     return len(self.pairsList)
                 def getPairsIndexList(self):
                     return range(self.nPairs)
                 def getNormFactor(self):
                     pwcode = 1
                     if self.flagDecodeData:
                         pwcode = numpy.sum(self.code[0]**2)
                     #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
                     normFactor = self.nProfiles*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
                     return normFactor
                 def getFlagCspc(self):
                     if self.data_cspc is None:
                         return True
                     return False
                 def getFlagDc(self):
                     if self.data_dc is None:
                         return True
                     return False
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles
                     return timeInterval
                 def setValue(self, value):
                     print "This property should not be initialized"
                     return
                 nPairs = property(getNPairs, setValue, "I'm the 'nPairs' property.")
                 pairsIndexList = property(getPairsIndexList, setValue, "I'm the 'pairsIndexList' property.")
                 normFactor = property(getNormFactor, setValue, "I'm the 'getNormFactor' property.")
                 flag_cspc = property(getFlagCspc, setValue)
                 flag_dc = property(getFlagDc, setValue)
                 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
                 timeInterval = property(getTimeInterval, setValue, "I'm the 'timeInterval' property")
             class SpectraHeis(Spectra):
                 data_spc = None
                 data_cspc = None
                 data_dc = None
                 nFFTPoints = None
             #     nPairs = None
                 pairsList = None
                 nCohInt = None
                 nIncohInt = None
                 def __init__(self):
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "SpectraHeis"
             #         self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
             #         self.nPairs = 0
                     self.utctime = None
                     self.blocksize = None
                     self.profileIndex = 0
                     self.nCohInt = 1
                     self.nIncohInt = 1
                 def getNormFactor(self):
                     pwcode = 1
                     if self.flagDecodeData:
                         pwcode = numpy.sum(self.code[0]**2)
                     normFactor = self.nIncohInt*self.nCohInt*pwcode
                     return normFactor
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     return timeInterval
                 normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
             class Fits(JROData):
                 heightList = None
                 channelList = None
                 flagNoData = True
                 flagDiscontinuousBlock = False
                 useLocalTime = False
                 utctime = None
                 timeZone = None
             #     ippSeconds = None
             #     timeInterval = None
                 nCohInt = None
                 nIncohInt = None
                 noise = None
                 windowOfFilter = 1
                 #Speed of ligth
                 C = 3e8
                 frequency = 49.92e6
                 realtime = False
                 def __init__(self):
                     self.type = "Fits"
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #         self.channelIndexList = None
                     self.flagNoData = True
                     self.utctime = None
                     self.nCohInt = 1
                     self.nIncohInt = 1
                     self.useLocalTime = True
                     self.profileIndex = 0
             #         self.utctime = None
             #         self.timeZone = None
             #         self.ltctime = None
             #         self.timeInterval = None
             #         self.header = None
             #         self.data_header = None
             #         self.data = None
             #         self.datatime = None
             #         self.flagNoData = False
             #         self.expName = ''
             #         self.nChannels = None
             #         self.nSamples = None
             #         self.dataBlocksPerFile = None
             #         self.comments = ''
             #
                 def getltctime(self):
                     if self.useLocalTime:
                         return self.utctime - self.timeZone*60
                     return self.utctime
                 def getDatatime(self):
                     datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
                     return datatime
                 def getTimeRange(self):
                     datatime = []
                     datatime.append(self.ltctime)
                     datatime.append(self.ltctime + self.timeInterval)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getHeiRange(self):
                     heis = self.heightList
                     return heis
                 def getNHeights(self):
                     return len(self.heightList)
                 def getNChannels(self):
                     return len(self.channelList)
                 def getChannelIndexList(self):
                     return range(self.nChannels)
                 def getNoise(self, type = 1):
                     #noise = numpy.zeros(self.nChannels)
                     if type == 1:
                         noise = self.getNoisebyHildebrand()
                     if type == 2:
                         noise = self.getNoisebySort()
                     if type == 3:
                         noise = self.getNoisebyWindow()
                     return noise
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     return timeInterval
                 datatime = property(getDatatime, "I'm the 'datatime' property")
                 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
                 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
                 channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
                 noise = property(getNoise, "I'm the 'nHeights' property.")
                 ltctime = property(getltctime, "I'm the 'ltctime' property")
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
             class Correlation(JROData):
                 noise = None
                 SNR = None
                 pairsAutoCorr = None    #Pairs of Autocorrelation
                 #--------------------------------------------------
                 data_corr = None
                 data_volt = None
                 lagT = None # each element value is a profileIndex
                 lagR = None # each element value is in km
                 pairsList = None
                 calculateVelocity = None
                 nPoints = None
                 nAvg = None
                 bufferSize = None
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Correlation"
                     self.data = None
                     self.dtype = None
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.timeZone = None
                     self.dstFlag = None
                     self.errorCount = None
                     self.blocksize = None
                     self.flagDecodeData = False #asumo q la data no esta decodificada
                     self.flagDeflipData = False #asumo q la data no esta sin flip
                     self.pairsList = None
                     self.nPoints = None
                 def getLagTRange(self, extrapoints=0):
                     lagTRange = self.lagT
                     diff = lagTRange[1] - lagTRange[0]
                     extra = numpy.arange(1,extrapoints + 1)*diff + lagTRange[-1]
                     lagTRange = numpy.hstack((lagTRange, extra))
                     return lagTRange
                 def getLagRRange(self, extrapoints=0):
                     return self.lagR
                 def getPairsList(self):
                     return self.pairsList
                 def getCalculateVelocity(self):
                     return self.calculateVelocity
                 def getNPoints(self):
                     return self.nPoints
                 def getNAvg(self):
                     return self.nAvg
                 def getBufferSize(self):
                     return self.bufferSize
                 def getPairsAutoCorr(self):
                     pairsList = self.pairsList
                     pairsAutoCorr = numpy.zeros(self.nChannels, dtype = 'int')*numpy.nan
                     for l in range(len(pairsList)):
                         firstChannel = pairsList[l][0]
                         secondChannel = pairsList[l][1]
                         #Obteniendo pares de Autocorrelacion
                         if firstChannel == secondChannel:
                             pairsAutoCorr[firstChannel] = int(l)
                     pairsAutoCorr = pairsAutoCorr.astype(int)
                     return pairsAutoCorr
                 def getNoise(self, mode = 2):
                     indR = numpy.where(self.lagR == 0)[0][0]
                     indT = numpy.where(self.lagT == 0)[0][0]
                     jspectra0 = self.data_corr[:,:,indR,:]
                     jspectra = copy.copy(jspectra0)
                     num_chan = jspectra.shape[0]
                     num_hei = jspectra.shape[2]
                     freq_dc = jspectra.shape[1]/2
                     ind_vel = numpy.array([-2,-1,1,2]) + freq_dc
                     if ind_vel[0]<0:
                         ind_vel[range(0,1)] = ind_vel[range(0,1)] + self.num_prof
                     if mode == 1:
                         jspectra[:,freq_dc,:] = (jspectra[:,ind_vel[1],:] + jspectra[:,ind_vel[2],:])/2 #CORRECCION
                     if mode == 2:
                         vel = numpy.array([-2,-1,1,2])
                         xx = numpy.zeros([4,4])
                         for fil in range(4):
                             xx[fil,:] = vel[fil]**numpy.asarray(range(4))
                         xx_inv = numpy.linalg.inv(xx)
                         xx_aux = xx_inv[0,:]
                         for ich in range(num_chan):
                             yy = jspectra[ich,ind_vel,:]
                             jspectra[ich,freq_dc,:] = numpy.dot(xx_aux,yy)
                             junkid = jspectra[ich,freq_dc,:]<=0
                             cjunkid = sum(junkid)
                             if cjunkid.any():
                                 jspectra[ich,freq_dc,junkid.nonzero()] = (jspectra[ich,ind_vel[1],junkid] + jspectra[ich,ind_vel[2],junkid])/2
                     noise = jspectra0[:,freq_dc,:] - jspectra[:,freq_dc,:]
                     return noise
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nPoints
                     return timeInterval
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
             #     pairsList = property(getPairsList, "I'm the 'pairsList' property.")
             #     nPoints = property(getNPoints, "I'm the 'nPoints' property.")
                 calculateVelocity = property(getCalculateVelocity, "I'm the 'calculateVelocity' property.")
                 nAvg = property(getNAvg, "I'm the 'nAvg' property.")
                 bufferSize = property(getBufferSize, "I'm the 'bufferSize' property.")
             class Parameters(JROData):
                 #Information from previous data
                 inputUnit = None        #Type of data to be processed
                 operation = None        #Type of operation to parametrize
                 normFactor = None       #Normalization Factor
                 groupList = None        #List of Pairs, Groups, etc
                 #Parameters
                 data_param = None       #Parameters obtained
                 data_pre = None         #Data Pre Parametrization
                 data_SNR = None         #Signal to Noise Ratio
             #     heightRange = None      #Heights
                 abscissaList = None    #Abscissa, can be velocities, lags or time
                 noise = None            #Noise Potency
                 utctimeInit = None      #Initial UTC time
                 paramInterval = None    #Time interval to calculate Parameters in seconds
                 #Fitting
                 data_error = None       #Error of the estimation
                 constants = None
                 library = None
                 #Output signal
                 outputInterval = None     #Time interval to calculate output signal in seconds
                 data_output = None       #Out signal
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Parameters"
                 def getTimeRange1(self):
                     datatime = []
                     if self.useLocalTime:
                         time1 = self.utctimeInit - self.timeZone*60
                     else:
                         time1 = utctimeInit
             #         datatime.append(self.utctimeInit)
             #         datatime.append(self.utctimeInit + self.outputInterval)
                     datatime.append(time1)
                     datatime.append(time1 + self.outputInterval)
                     datatime = numpy.array(datatime)
                     return datatime

schainpy/model/data/jroheaderIO.py +36 -26

             '''
             $Author: murco $
             $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
             '''
             import numpy
             import copy
             import datetime
             SPEED_OF_LIGHT = 299792458
             SPEED_OF_LIGHT = 3e8
             BASIC_STRUCTURE = numpy.dtype([
                                       ('nSize','<u4'),
                                       ('nVersion','<u2'),
                                       ('nDataBlockId','<u4'),
                                       ('nUtime','<u4'),
                                       ('nMilsec','<u2'),
                                       ('nTimezone','<i2'),
                                       ('nDstflag','<i2'),
                                       ('nErrorCount','<u4')
                                       ])
             SYSTEM_STRUCTURE = numpy.dtype([
                                         ('nSize','<u4'),
                                         ('nNumSamples','<u4'),
                                         ('nNumProfiles','<u4'),
                                         ('nNumChannels','<u4'),
                                         ('nADCResolution','<u4'),
                                         ('nPCDIOBusWidth','<u4'),
                                         ])
             RADAR_STRUCTURE = numpy.dtype([
                                          ('nSize','<u4'),
                                          ('nExpType','<u4'),
                                          ('nNTx','<u4'),
                                          ('fIpp','<f4'),
                                          ('fTxA','<f4'),
                                          ('fTxB','<f4'),
                                          ('nNumWindows','<u4'),
                                          ('nNumTaus','<u4'),
                                          ('nCodeType','<u4'),
                                          ('nLine6Function','<u4'),
                                          ('nLine5Function','<u4'),
                                          ('fClock','<f4'),
                                          ('nPrePulseBefore','<u4'),
                                          ('nPrePulseAfter','<u4'),
                                          ('sRangeIPP','<a20'),
                                          ('sRangeTxA','<a20'),
                                          ('sRangeTxB','<a20'),
                                          ])
             SAMPLING_STRUCTURE = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')])
             PROCESSING_STRUCTURE = numpy.dtype([
                                            ('nSize','<u4'),
                                            ('nDataType','<u4'),
                                            ('nSizeOfDataBlock','<u4'),
                                            ('nProfilesperBlock','<u4'),
                                            ('nDataBlocksperFile','<u4'),
                                            ('nNumWindows','<u4'),
                                            ('nProcessFlags','<u4'),
                                            ('nCoherentIntegrations','<u4'),
                                            ('nIncoherentIntegrations','<u4'),
                                            ('nTotalSpectra','<u4')
                                            ])
             class Header(object):
                 def __init__(self):
-                    raise
+                    raise NotImplementedError
                 def copy(self):
                     return copy.deepcopy(self)
                 def read(self):
-                    raise ValueError
+                    raise NotImplementedError
                 def write(self):
-                    raise ValueError
+                    raise NotImplementedError
                 def printInfo(self):
                     message = "#"*50 + "\n"
                     message += self.__class__.__name__.upper() + "\n"
                     message += "#"*50 + "\n"
                     for key in self.__dict__.keys():
                         message += "%s = %s" %(key, self.__dict__[key]) + "\n"
                     print message
             class BasicHeader(Header):
                 size = None
                 version = None
                 dataBlock = None
                 utc = None
                 ltc = None
                 miliSecond = None
                 timeZone = None
                 dstFlag = None
                 errorCount = None
                 datatime = None
                 __LOCALTIME = None
                 def __init__(self, useLocalTime=True):
                     self.size = 24
                     self.version = 0
                     self.dataBlock = 0
                     self.utc = 0
                     self.miliSecond = 0
                     self.timeZone = 0
                     self.dstFlag = 0
                     self.errorCount = 0
                     self.useLocalTime = useLocalTime
                 def read(self, fp):
                     try:
                         header = numpy.fromfile(fp, BASIC_STRUCTURE,1)
-                        self.size = int(header['nSize'][0])
-                        self.version = int(header['nVersion'][0])
-                        self.dataBlock = int(header['nDataBlockId'][0])
-                        self.utc = int(header['nUtime'][0])
-                        self.miliSecond = int(header['nMilsec'][0])
-                        self.timeZone = int(header['nTimezone'][0])
-                        self.dstFlag = int(header['nDstflag'][0])
-                        self.errorCount = int(header['nErrorCount'][0])
                     except Exception, e:
                         print "BasicHeader: "
                         print e
                         return 0
+                    self.size = int(header['nSize'][0])
+                    self.version = int(header['nVersion'][0])
+                    self.dataBlock = int(header['nDataBlockId'][0])
+                    self.utc = int(header['nUtime'][0])
+                    self.miliSecond = int(header['nMilsec'][0])
+                    self.timeZone = int(header['nTimezone'][0])
+                    self.dstFlag = int(header['nDstflag'][0])
+                    self.errorCount = int(header['nErrorCount'][0])
                     return 1
                 def write(self, fp):
                     headerTuple = (self.size,self.version,self.dataBlock,self.utc,self.miliSecond,self.timeZone,self.dstFlag,self.errorCount)
                     header = numpy.array(headerTuple, BASIC_STRUCTURE)
                     header.tofile(fp)
                     return 1
                 def get_ltc(self):
                     return self.utc - self.timeZone*60
                 def set_ltc(self, value):
                     self.utc = value + self.timeZone*60
                 def get_datatime(self):
                     return datetime.datetime.utcfromtimestamp(self.ltc)
                 ltc = property(get_ltc, set_ltc)
                 datatime = property(get_datatime)
             class SystemHeader(Header):
                 size = None
                 nSamples = None
                 nProfiles = None
                 nChannels = None
                 adcResolution = None
                 pciDioBusWidth = None
                 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWith=0):
                     self.size = 24
                     self.nSamples = nSamples
                     self.nProfiles = nProfiles
                     self.nChannels = nChannels
                     self.adcResolution = adcResolution
                     self.pciDioBusWidth = pciDioBusWith
                 def read(self, fp):
                     startFp = fp.tell()
                     try:
                         header = numpy.fromfile(fp,SYSTEM_STRUCTURE,1)
-                        self.size = header['nSize'][0]
-                        self.nSamples = header['nNumSamples'][0]
-                        self.nProfiles = header['nNumProfiles'][0]
-                        self.nChannels = header['nNumChannels'][0]
-                        self.adcResolution = header['nADCResolution'][0]
-                        self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
                     except Exception, e:
                         print "SystemHeader: " + e
                         return 0
+                    self.size = header['nSize'][0]
+                    self.nSamples = header['nNumSamples'][0]
+                    self.nProfiles = header['nNumProfiles'][0]
+                    self.nChannels = header['nNumChannels'][0]
+                    self.adcResolution = header['nADCResolution'][0]
+                    self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
                     endFp = self.size + startFp
                     if fp.tell() != endFp:
-                        raise IOError, "System Header is not consistent"
+                        print "System Header is not consistent"
+                        return 0
                     return 1
                 def write(self, fp):
                     headerTuple = (self.size,self.nSamples,self.nProfiles,self.nChannels,self.adcResolution,self.pciDioBusWidth)
                     header = numpy.array(headerTuple,SYSTEM_STRUCTURE)
                     header.tofile(fp)
                     return 1
             class RadarControllerHeader(Header):
                 size = None
                 expType = None
                 nTx = None
                 ipp = None
                 txA = None
                 txB = None
                 nWindows = None
                 numTaus = None
                 codeType = None
                 line6Function = None
                 line5Function = None
                 fClock = None
                 prePulseBefore = None
                 prePulserAfter = None
                 rangeIpp = None
                 rangeTxA = None
                 rangeTxB = None
                 __size = None
                 def __init__(self, expType=2, nTx=1,
                              ippKm=None, txA=0, txB=0,
                              nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
                              numTaus=0, line6Function=0, line5Function=0, fClock=None,
                              prePulseBefore=0, prePulseAfter=0,
                              codeType=0, nCode=0, nBaud=0, code=None,
                              flip1=0, flip2=0):
                     self.size = 116
                     self.expType = expType
                     self.nTx = nTx
                     self.ipp = ippKm
                     self.txA = txA
                     self.txB = txB
                     self.rangeIpp = ippKm
                     self.rangeTxA = txA
                     self.rangeTxB = txB
                     self.nWindows = nWindows
                     self.numTaus = numTaus
                     self.codeType = codeType
                     self.line6Function = line6Function
                     self.line5Function = line5Function
                     self.fClock = fClock
                     self.prePulseBefore = prePulseBefore
                     self.prePulserAfter = prePulseAfter
                     self.nHeights = nHeights
                     self.firstHeight = firstHeight
                     self.deltaHeight = deltaHeight
                     self.samplesWin = nHeights
                     self.nCode = nCode
                     self.nBaud = nBaud
                     self.code = code
                     self.flip1 = flip1
                     self.flip2 = flip2
                     self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4
             #         self.dynamic = numpy.array([],numpy.dtype('byte'))
                     if self.fClock is None and self.deltaHeight is not None:
                         self.fClock = 0.15/(deltaHeight*1e-6)   #0.15Km / (height * 1u)
                 def read(self, fp):
                     startFp = fp.tell()
-                    header = numpy.fromfile(fp,RADAR_STRUCTURE,1)
+                    try:
+                        header = numpy.fromfile(fp,RADAR_STRUCTURE,1)
+                    except Exception, e:
+                        print "RadarControllerHeader: " + e
+                        return 0
                     size = int(header['nSize'][0])
                     self.expType = int(header['nExpType'][0])
                     self.nTx = int(header['nNTx'][0])
                     self.ipp = float(header['fIpp'][0])
                     self.txA = float(header['fTxA'][0])
                     self.txB = float(header['fTxB'][0])
                     self.nWindows = int(header['nNumWindows'][0])
                     self.numTaus = int(header['nNumTaus'][0])
                     self.codeType = int(header['nCodeType'][0])
                     self.line6Function = int(header['nLine6Function'][0])
                     self.line5Function = int(header['nLine5Function'][0])
                     self.fClock = float(header['fClock'][0])
                     self.prePulseBefore = int(header['nPrePulseBefore'][0])
                     self.prePulserAfter = int(header['nPrePulseAfter'][0])
                     self.rangeIpp = header['sRangeIPP'][0]
                     self.rangeTxA = header['sRangeTxA'][0]
                     self.rangeTxB = header['sRangeTxB'][0]
                     samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows)
                     self.nHeights = int(numpy.sum(samplingWindow['nsa']))
                     self.firstHeight = samplingWindow['h0']
                     self.deltaHeight = samplingWindow['dh']
                     self.samplesWin = samplingWindow['nsa']
                     self.Taus = numpy.fromfile(fp,'<f4',self.numTaus)
                     self.code_size = 0
                     if self.codeType != 0:
                         self.nCode = int(numpy.fromfile(fp,'<u4',1))
                         self.nBaud = int(numpy.fromfile(fp,'<u4',1))
                         code = numpy.empty([self.nCode,self.nBaud],dtype='i1')
                         for ic in range(self.nCode):
                             temp = numpy.fromfile(fp,'u4',int(numpy.ceil(self.nBaud/32.)))
                             for ib in range(self.nBaud-1,-1,-1):
                                 code[ic,ib] = temp[ib/32]%2
                                 temp[ib/32] = temp[ib/32]/2
                         self.code = 2.0*code - 1.0
                         self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4
             #             if self.line5Function == RCfunction.FLIP:
             #                 self.flip1 = numpy.fromfile(fp,'<u4',1)
             #
             #             if self.line6Function == RCfunction.FLIP:
             #                 self.flip2 = numpy.fromfile(fp,'<u4',1)
                     endFp = size + startFp
                     if fp.tell() != endFp:
-                        raise IOError, "Radar Controller Header is not consistent"
+                        print "Radar Controller Header is not consistent"
+                        return 0
                     return 1
                 def write(self, fp):
                     headerTuple = (self.size,
                                    self.expType,
                                    self.nTx,
                                    self.ipp,
                                    self.txA,
                                    self.txB,
                                    self.nWindows,
                                    self.numTaus,
                                    self.codeType,
                                    self.line6Function,
                                    self.line5Function,
                                    self.fClock,
                                    self.prePulseBefore,
                                    self.prePulserAfter,
                                    self.rangeIpp,
                                    self.rangeTxA,
                                    self.rangeTxB)
                     header = numpy.array(headerTuple,RADAR_STRUCTURE)
                     header.tofile(fp)
                     sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
                     samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE)
                     samplingWindow.tofile(fp)
                     if self.numTaus > 0:
                         self.Taus.tofile(fp)
                     if self.codeType !=0:
                         nCode = numpy.array(self.nCode, '<u4')
                         nCode.tofile(fp)
                         nBaud = numpy.array(self.nBaud, '<u4')
                         nBaud.tofile(fp)
                         code1 = (self.code + 1.0)/2.
                         for ic in range(self.nCode):
                             tempx = numpy.zeros(numpy.ceil(self.nBaud/32.))
                             start = 0
                             end = 32
                             for i in range(len(tempx)):
                                 code_selected = code1[ic,start:end]
                                 for j in range(len(code_selected)-1,-1,-1):
                                     if code_selected[j] == 1:
                                         tempx[i] = tempx[i] + 2**(len(code_selected)-1-j)
                                 start = start + 32
                                 end = end + 32
                             tempx = tempx.astype('u4')
                             tempx.tofile(fp)
             #         if self.line5Function == RCfunction.FLIP:
             #             self.flip1.tofile(fp)
             #
             #         if self.line6Function == RCfunction.FLIP:
             #             self.flip2.tofile(fp)
                     return 1
                 def get_ippSeconds(self):
                     '''
                     '''
                     ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT
                     return ippSeconds
                 def set_ippSeconds(self, ippSeconds):
                     '''
                     '''
                     self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0*1000)
                     return
                 def get_size(self):
                     self.__size = 116 + 12*self.nWindows + 4*self.numTaus
                     if self.codeType != 0:
                         self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
                     return self.__size
                 def set_size(self, value):
                     self.__size = value
                     return
                 ippSeconds = property(get_ippSeconds, set_ippSeconds)
                 size = property(get_size, set_size)
             class ProcessingHeader(Header):
             #     size = None
                 dtype = None
                 blockSize = None
                 profilesPerBlock = None
                 dataBlocksPerFile = None
                 nWindows = None
                 processFlags = None
                 nCohInt = None
                 nIncohInt = None
                 totalSpectra = None
                 flag_dc = None
                 flag_cspc = None
                 def __init__(self):
             #         self.size = 0
                     self.dtype = 0
                     self.blockSize = 0
                     self.profilesPerBlock = 0
                     self.dataBlocksPerFile = 0
                     self.nWindows = 0
                     self.processFlags = 0
                     self.nCohInt = 0
                     self.nIncohInt = 0
                     self.totalSpectra = 0
                     self.nHeights = 0
                     self.firstHeight = 0
                     self.deltaHeight = 0
                     self.samplesWin = 0
                     self.spectraComb = 0
                     self.nCode = None
                     self.code = None
                     self.nBaud = None
                     self.shif_fft = False
                     self.flag_dc = False
                     self.flag_cspc = False
                     self.flag_decode = False
                     self.flag_deflip = False
                 def read(self, fp):
                     startFp = fp.tell()
-                    header = numpy.fromfile(fp,PROCESSING_STRUCTURE,1)
+                    try:
+                        header = numpy.fromfile(fp,PROCESSING_STRUCTURE,1)
+                    except Exception, e:
+                        print "ProcessingHeader: " + e
+                        return 0
                     size = int(header['nSize'][0])
                     self.dtype = int(header['nDataType'][0])
                     self.blockSize = int(header['nSizeOfDataBlock'][0])
                     self.profilesPerBlock = int(header['nProfilesperBlock'][0])
                     self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
                     self.nWindows = int(header['nNumWindows'][0])
                     self.processFlags = header['nProcessFlags']
                     self.nCohInt = int(header['nCoherentIntegrations'][0])
                     self.nIncohInt = int(header['nIncoherentIntegrations'][0])
                     self.totalSpectra = int(header['nTotalSpectra'][0])
                     samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows)
                     self.nHeights = int(numpy.sum(samplingWindow['nsa']))
                     self.firstHeight = float(samplingWindow['h0'][0])
                     self.deltaHeight = float(samplingWindow['dh'][0])
                     self.samplesWin = samplingWindow['nsa'][0]
                     self.spectraComb = numpy.fromfile(fp,'u1',2*self.totalSpectra)
                     if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
                         self.nCode = int(numpy.fromfile(fp,'<u4',1))
                         self.nBaud = int(numpy.fromfile(fp,'<u4',1))
                         self.code = numpy.fromfile(fp,'<f4',self.nCode*self.nBaud).reshape(self.nCode,self.nBaud)
                     if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP):
                         exp_name_len = int(numpy.fromfile(fp,'<u4',1))
                         exp_name = numpy.fromfile(fp,'u1',exp_name_len+1)
                     if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
                         self.shif_fft = True
                     else:
                         self.shif_fft = False
                     if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
                         self.flag_dc = True
                     else:
                         self.flag_dc = False
                     if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA):
                         self.flag_decode = True
                     else:
                         self.flag_decode = False
                     if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA):
                         self.flag_deflip = True
                     else:
                         self.flag_deflip = False
                     nChannels = 0
                     nPairs = 0
                     pairList = []
                     for i in range( 0, self.totalSpectra*2, 2 ):
                         if self.spectraComb[i] == self.spectraComb[i+1]:
                             nChannels = nChannels + 1   #par de canales iguales
                         else:
                             nPairs = nPairs + 1 #par de canales diferentes
                             pairList.append( (self.spectraComb[i], self.spectraComb[i+1]) )
                     self.flag_cspc = False
                     if nPairs > 0:
                         self.flag_cspc = True
                     endFp = size + startFp
                     if fp.tell() != endFp:
-                        raise IOError, "Processing Header is not consistent"
+                        print "Processing Header is not consistent"
+                        return 0
                     return 1
                 def write(self, fp):
                     #Clear DEFINE_PROCESS_CODE
             #         self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE)
                     headerTuple = (self.size,
                                    self.dtype,
                                    self.blockSize,
                                    self.profilesPerBlock,
                                    self.dataBlocksPerFile,
                                    self.nWindows,
                                    self.processFlags,
                                    self.nCohInt,
                                    self.nIncohInt,
                                    self.totalSpectra)
                     header = numpy.array(headerTuple,PROCESSING_STRUCTURE)
                     header.tofile(fp)
                     if self.nWindows != 0:
                         sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
                         samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE)
                         samplingWindow.tofile(fp)
                     if self.totalSpectra != 0:
             #             spectraComb = numpy.array([],numpy.dtype('u1'))
                         spectraComb = self.spectraComb
                         spectraComb.tofile(fp)
                     if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
                         nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
                         nCode.tofile(fp)
                         nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
                         nBaud.tofile(fp)
                         code = self.code.reshape(self.nCode*self.nBaud)
                         code = code.astype(numpy.dtype('<f4'))
                         code.tofile(fp)
                     return 1
                 def get_size(self):
                     self.__size = 40 + 12*self.nWindows + 2*self.totalSpectra
                     if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
             #             self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
                         self.__size += 4 + 4 + 4 * self.nCode * self.nBaud
                     return self.__size
                 def set_size(self, value):
                     self.__size = value
                     return
                 size = property(get_size, set_size)
             class RCfunction:
                 NONE=0
                 FLIP=1
                 CODE=2
                 SAMPLING=3
                 LIN6DIV256=4
                 SYNCHRO=5
             class nCodeType:
                 NONE=0
                 USERDEFINE=1
                 BARKER2=2
                 BARKER3=3
                 BARKER4=4
                 BARKER5=5
                 BARKER7=6
                 BARKER11=7
                 BARKER13=8
                 AC128=9
                 COMPLEMENTARYCODE2=10
                 COMPLEMENTARYCODE4=11
                 COMPLEMENTARYCODE8=12
                 COMPLEMENTARYCODE16=13
                 COMPLEMENTARYCODE32=14
                 COMPLEMENTARYCODE64=15
                 COMPLEMENTARYCODE128=16
                 CODE_BINARY28=17
             class PROCFLAG:
                 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
                 DECODE_DATA = numpy.uint32(0x00000002)
                 SPECTRA_CALC = numpy.uint32(0x00000004)
                 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
                 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
                 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
                 DATATYPE_CHAR = numpy.uint32(0x00000040)
                 DATATYPE_SHORT = numpy.uint32(0x00000080)
                 DATATYPE_LONG = numpy.uint32(0x00000100)
                 DATATYPE_INT64 = numpy.uint32(0x00000200)
                 DATATYPE_FLOAT = numpy.uint32(0x00000400)
                 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
                 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
                 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
                 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
                 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
                 DEFLIP_DATA = numpy.uint32(0x00010000)
                 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
                 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
                 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
                 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
                 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
                 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
                 EXP_NAME_ESP = numpy.uint32(0x00200000)
                 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
                 OPERATION_MASK = numpy.uint32(0x0000003F)
                 DATATYPE_MASK = numpy.uint32(0x00000FC0)
                 DATAARRANGE_MASK = numpy.uint32(0x00007000)
                 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
             dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
             dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
             dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
             dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
             dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
             dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
             NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
             PROCFLAG_DTYPE_LIST =  [PROCFLAG.DATATYPE_CHAR,
                                    PROCFLAG.DATATYPE_SHORT,
                                    PROCFLAG.DATATYPE_LONG,
                                    PROCFLAG.DATATYPE_INT64,
                                    PROCFLAG.DATATYPE_FLOAT,
                                    PROCFLAG.DATATYPE_DOUBLE]
             DTYPE_WIDTH = [1, 2, 4, 8, 4, 8]
             def get_dtype_index(numpy_dtype):
                 index = None
                 for i in range(len(NUMPY_DTYPE_LIST)):
                     if numpy_dtype == NUMPY_DTYPE_LIST[i]:
                         index = i
                         break
                 return index
             def get_numpy_dtype(index):
                 return NUMPY_DTYPE_LIST[index]
             def get_procflag_dtype(index):
                 return PROCFLAG_DTYPE_LIST[index]
             def get_dtype_width(index):
                 return DTYPE_WIDTH[index]

schainpy/model/io/jroIO_HDF5.py +2 -2

             import numpy
             import time
             import os
             import h5py
             import re
             from schainpy.model.data.jrodata import *
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             from schainpy.model.io.jroIO_base import *
             class HDF5Reader(ProcessingUnit):
                 ext = ".hdf5"
                 optchar = "D"
                 timezone = None
                 secStart = None
                 secEnd = None
                 fileIndex = None
                 blockIndex = None
                 blocksPerFile = None
                 path = None
                 #List of Files
                 filenameList = None
                 datetimeList = None
                 #Hdf5 File
                 fpMetadata = None
                 pathMeta = None
                 listMetaname = None
                 listMeta = None
                 listDataname = None
                 listData = None
                 listShapes = None
                 fp = None
                 #dataOut reconstruction
                 dataOut = None
                 nRecords = None
                 def __init__(self):
                     self.dataOut = self.__createObjByDefault()
                     return
                 def __createObjByDefault(self):
                     dataObj = Parameters()
                     return dataObj
                 def setup(self,path=None,
                                 startDate=None,
                                 endDate=None,
                                 startTime=datetime.time(0,0,0),
                                 endTime=datetime.time(23,59,59),
                                 walk=True,
                                 timezone='ut',
                                 all=0,
                                 online=False,
                                 ext=None):
                     if ext==None:
                         ext = self.ext
                     self.timezone = timezone
             #         self.all = all
             #         self.online = online
                     self.path = path
                     startDateTime = datetime.datetime.combine(startDate,startTime)
                     endDateTime = datetime.datetime.combine(endDate,endTime)
                     secStart = (startDateTime-datetime.datetime(1970,1,1)).total_seconds()
                     secEnd = (endDateTime-datetime.datetime(1970,1,1)).total_seconds()
                     self.secStart = secStart
                     self.secEnd = secEnd
                     if not(online):
                         #Busqueda de archivos offline
                         self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, secStart, secEnd, walk)
                     else:
                         self.__searchFilesOnline(path, walk)
                     if not(self.filenameList):
                         print "There is no files into the folder: %s"%(path)
                         sys.exit(-1)
             #         self.__getExpParameters()
                     self.fileIndex = -1
                     self.__setNextFileOffline()
                     self.__readMetadata()
                     self.blockIndex = 0
                     return
                 def __searchFilesOffline(self,
                                         path,
                                         startDate,
                                         endDate,
                                         ext,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         secStart = 0,
                                         secEnd = numpy.inf,
                                         walk=True):
             #         self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
             #
             #         self.__checkPath()
             #
             #         self.__findDataForDates()
             #
             #         self.__selectDataForTimes()
             #
             #         for i in range(len(self.filenameList)):
             #             print "%s" %(self.filenameList[i])
                     pathList = []
                     if not walk:
                         #pathList.append(path)
                         multi_path = path.split(',')
                         for single_path in multi_path:
                             pathList.append(single_path)
                     else:
                         #dirList = []
                         multi_path = path.split(',')
                         for single_path in multi_path:
                             dirList = []
                             for thisPath in os.listdir(single_path):
                                 if not os.path.isdir(os.path.join(single_path,thisPath)):
                                     continue
                                 if not isDoyFolder(thisPath):
                                     continue
                                 dirList.append(thisPath)
                             if not(dirList):
                                 return None, None
                             thisDate = startDate
                             while(thisDate <= endDate):
                                 year = thisDate.timetuple().tm_year
                                 doy = thisDate.timetuple().tm_yday
                                 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
                                 if len(matchlist) == 0:
                                     thisDate += datetime.timedelta(1)
                                     continue
                                 for match in matchlist:
                                     pathList.append(os.path.join(single_path,match))
                                 thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
                         return None, None
                     print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
                     filenameList = []
                     datetimeList = []
                     pathDict = {}
                     filenameList_to_sort = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         pathDict.setdefault(fileList[0])
                         pathDict[fileList[0]] = i
                         filenameList_to_sort.append(fileList[0])
                     filenameList_to_sort.sort()
                     for file in filenameList_to_sort:
                         thisPath = pathList[pathDict[file]]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             thisDatetime = self.__isFileinThisTime(filename, secStart, secEnd)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "Any file was found for the time range %s - %s" %(startTime, endTime)
                         return None, None
                     print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __isFileinThisTime(self, filename, startSeconds, endSeconds):
                     """
                     Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                     Inputs:
                         filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                         startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                         endTime            :    tiempo final del rango seleccionado en formato datetime.time
                     Return:
                         Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                         fecha especificado, de lo contrario retorna False.
                     Excepciones:
                         Si el archivo no existe o no puede ser abierto
                         Si la cabecera no puede ser leida.
                     """
                     try:
                         fp = fp = h5py.File(filename,'r')
                     except IOError:
-                        traceback.print_exc()
+                        print "File %s can't be opened" %(filename)
-                        raise IOError, "The file %s can't be opened" %(filename)
+                        return None
                     grp = fp['Data']
                     timeAux = grp['time']
                     time0 = timeAux[:][0].astype(numpy.float)   #Time Vector
                     fp.close()
                     if self.timezone == 'lt':
                         time0 -= 5*3600
                     boolTimer = numpy.logical_and(time0 >= startSeconds,time0 < endSeconds)
                     if not (numpy.any(boolTimer)):
                         return None
                     thisDatetime = datetime.datetime.utcfromtimestamp(time0[0])
                     return thisDatetime
                 def __checkPath(self):
                     if os.path.exists(self.path):
                         self.status = 1
                     else:
                         self.status = 0
                         print 'Path:%s does not exists'%self.path
                     return
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     idFile += 1
                     if not(idFile < len(self.filenameList)):
                         print "No more Files"
                         return 0
                     filename = self.filenameList[idFile]
                     filePointer = h5py.File(filename,'r')
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fp = filePointer
                     print "Setting the file: %s"%self.filename
                     self.__readMetadata()
                     self.__setBlockList()
             #         self.nRecords = self.fp['Data'].attrs['blocksPerFile']
                     self.nRecords = self.fp['Data'].attrs['nRecords']
                     self.blockIndex = 0
                     return 1
                 def __setBlockList(self):
                     '''
                     self.fp
                     self.startDateTime
                     self.endDateTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     filePointer = self.fp
                     secStart = self.secStart
                     secEnd = self.secEnd
                     grp = filePointer['Data']
                     timeVector = grp['time'].value.astype(numpy.float)[0]
                     if self.timezone == 'lt':
                         timeVector -= 5*3600
                     ind = numpy.where(numpy.logical_and(timeVector >= secStart , timeVector < secEnd))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     self.pathMeta
                     self.listShapes
                     self.listMetaname
                     self.listMeta
                     '''
                     grp = self.fp['Data']
                     pathMeta = os.path.join(self.path, grp.attrs['metadata'])
                     if pathMeta == self.pathMeta:
                         return
                     else:
                         self.pathMeta = pathMeta
                     filePointer = h5py.File(self.pathMeta,'r')
                     groupPointer = filePointer['Metadata']
                     listMetaname = []
                     listMetadata = []
                     for item in groupPointer.items():
                         name = item[0]
                         if name=='array dimensions':
                             table = groupPointer[name][:]
                             listShapes = {}
                             for shapes in table:
                                 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4]])
                         else:
                             data = groupPointer[name].value
                             listMetaname.append(name)
                             listMetadata.append(data)
                             if name=='type':
                                 self.__initDataOut(data)
                     filePointer.close()
                     self.listShapes = listShapes
                     self.listMetaname = listMetaname
                     self.listMeta = listMetadata
                     return
                 def __readData(self):
                     grp = self.fp['Data']
                     listdataname = []
                     listdata = []
                     for item in grp.items():
                         name = item[0]
                         if name == 'time':
                             listdataname.append('utctime')
                             timeAux = grp[name].value.astype(numpy.float)[0]
                             listdata.append(timeAux)
                             continue
                         listdataname.append(name)
                         array = self.__setDataArray(self.nRecords, grp[name],self.listShapes[name])
                         listdata.append(array)
                     self.listDataname = listdataname
                     self.listData = listdata
                     return
                 def __setDataArray(self, nRecords, dataset, shapes):
                     nChannels = shapes[0]    #Dimension 0
                     nPoints = shapes[1]      #Dimension 1, number of Points or Parameters
                     nSamples = shapes[2]     #Dimension 2, number of samples or ranges
                     mode = shapes[3]
             #         if nPoints>1:
             #             arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
             #         else:
             #             arrayData = numpy.zeros((nRecords,nChannels,nSamples))
             #
             #         chn = 'channel'
             #
             #         for i in range(nChannels):
             #
             #             data = dataset[chn + str(i)].value
             #
             #             if nPoints>1:
             #                 data = numpy.rollaxis(data,2)
             #
             #             arrayData[:,i,:] = data
                     arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
                     doSqueeze = False
                     if mode == 0:
                         strds = 'channel'
                         nDatas = nChannels
                         newShapes = (nRecords,nPoints,nSamples)
                         if nPoints == 1:
                             doSqueeze = True
                             axisSqueeze = 2
                     else:
                         strds = 'param'
                         nDatas = nPoints
                         newShapes = (nRecords,nChannels,nSamples)
                         if nChannels == 1:
                             doSqueeze = True
                             axisSqueeze = 1
                     for i in range(nDatas):
                         data = dataset[strds + str(i)].value
                         data = data.reshape(newShapes)
                         if mode == 0:
                             arrayData[:,i,:,:] = data
                         else:
                             arrayData[:,:,i,:] = data
                     if doSqueeze:
                             arrayData = numpy.squeeze(arrayData, axis=axisSqueeze)
                     return arrayData
                 def __initDataOut(self, type):
             #         if type =='Parameters':
             #             self.dataOut = Parameters()
             #         elif type =='Spectra':
             #             self.dataOut = Spectra()
             #         elif type =='Voltage':
             #             self.dataOut = Voltage()
             #         elif type =='Correlation':
             #             self.dataOut = Correlation()
                     return
                 def __setDataOut(self):
                     listMeta = self.listMeta
                     listMetaname = self.listMetaname
                     listDataname = self.listDataname
                     listData = self.listData
                     blockIndex = self.blockIndex
                     blockList = self.blockList
                     for i in range(len(listMeta)):
                         setattr(self.dataOut,listMetaname[i],listMeta[i])
                     for j in range(len(listData)):
                         if listDataname[j]=='utctime':
             #                 setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex]])
                             setattr(self.dataOut,'utctimeInit',listData[j][blockList[blockIndex]])
                             continue
                         setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex],:])
                     return self.dataOut.data_param
                 def getData(self):
             #         if self.flagNoMoreFiles:
             #             self.dataOut.flagNoData = True
             #             print 'Process finished'
             #             return 0
             #
                     if self.blockIndex==self.blocksPerFile:
                          if not( self.__setNextFileOffline() ):
                             self.dataOut.flagNoData = True
                             return 0
             #
             #         if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
             #             self.dataOut.flagNoData = True
             #             return 0
                     self.__readData()
                     self.__setDataOut()
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
             #             self.setObjProperties()
                         self.isConfig = True
                     self.getData()
                     return
             class HDF5Writer(Operation):
                 ext = ".hdf5"
                 optchar = "D"
                 metaoptchar = "M"
                 metaFile = None
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 grp = None
                 ds = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 arrayDim = None
                 tableDim = None
             #     dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
                 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
                 mode = None
                 nDatas = None    #Number of datasets to be stored per array
                 nDims = None  #Number Dimensions in each dataset
                 nDimsForDs = None
                 def __init__(self):
                     Operation.__init__(self)
                     self.isConfig = False
                     return
                 def setup(self, dataOut, **kwargs):
                     self.path = kwargs['path']
                     if kwargs.has_key('ext'):
                         self.ext = kwargs['ext']
                     if kwargs.has_key('blocksPerFile'):
                         self.blocksPerFile = kwargs['blocksPerFile']
                     else:
                         self.blocksPerFile = 10
                     self.metadataList = kwargs['metadataList']
                     self.dataList = kwargs['dataList']
                     self.dataOut = dataOut
                     if kwargs.has_key('mode'):
                         mode = kwargs['mode']
                         if type(mode) == int:
                             mode = numpy.zeros(len(self.dataList)) + mode
                     else:
                         mode = numpy.zeros(len(self.dataList))
                     self.mode = mode
                     arrayDim = numpy.zeros((len(self.dataList),5))
                     #Table dimensions
                     dtype0 = self.dtype
                     tableList = []
                     for i in range(len(self.dataList)):
                         dataAux = getattr(self.dataOut, self.dataList[i])
                         if type(dataAux)==float or type(dataAux)==int:
                             arrayDim[i,0] = 1
                         else:
                             arrayDim0 = dataAux.shape
                             arrayDim[i,0] = len(arrayDim0)
                             arrayDim[i,4] = mode[i]
                             if len(arrayDim0) == 3:
                                 arrayDim[i,1:-1] = numpy.array(arrayDim0)
                             elif len(arrayDim0) == 2:
                                 arrayDim[i,2:-1] = numpy.array(arrayDim0) #nHeights
                             elif len(arrayDim0) == 1:
                                 arrayDim[i,3] = arrayDim0
                             elif len(arrayDim0) == 0:
                                 arrayDim[i,0] = 1
                                 arrayDim[i,3] = 1
                         table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
                         tableList.append(table)
                     self.arrayDim = arrayDim
                     self.tableDim = numpy.array(tableList, dtype = dtype0)
                     self.blockIndex = 0
                     return
                 def putMetadata(self):
                     fp = self.createMetadataFile()
                     self.writeMetadata(fp)
                     fp.close()
                     return
                 def createMetadataFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = ''
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         filesList = sorted( filesList, key=str.lower )
                         if len( filesList ) > 0:
                             filesList = [k for k in filesList if 'M' in k]
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     setFile += 1
                     file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, file )
                     self.metaFile = file
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     return fp
                 def writeMetadata(self, fp):
                     grp = fp.create_group("Metadata")
                     grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
                     for i in range(len(self.metadataList)):
                         grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
                     return
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     mode = self.mode
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if os.path.exists(fullpath):
                         filesList = os.listdir( fullpath )
                         filesList = [k for k in filesList if 'D' in k]
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     setFile += 1
                     file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, file )
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     grp = fp.create_group("Data")
                     grp.attrs['metadata'] = self.metaFile
             #         grp.attrs['blocksPerFile'] = 0
                     ds = []
                     data = []
                     nDimsForDs = []
                     nDatas = numpy.zeros(len(self.dataList))
                     nDims = self.arrayDim[:,0]
                     nDim1 = self.arrayDim[:,2]
                     nDim0 = self.arrayDim[:,3]
                     for i in range(len(self.dataList)):
                         if nDims[i]==1:
             #                 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
                             ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
                             ds.append(ds0)
                             data.append([])
                             nDimsForDs.append(nDims[i])
                         else:
                             if mode[i]==0:
                                 strMode = "channel"
                                 nDatas[i] = self.arrayDim[i,1]
                             else:
                                 strMode = "param"
                                 nDatas[i] = self.arrayDim[i,2]
                             if nDims[i]==2:
                                 nDatas[i] = self.arrayDim[i,2]
                             grp0 = grp.create_group(self.dataList[i])
                             for j in range(int(nDatas[i])):
                                 tableName = strMode + str(j)
                                 if nDims[i] == 3:
                                     ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True)
                                 else:
                                     ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True)
                                 ds.append(ds0)
                                 data.append([])
                                 nDimsForDs.append(nDims[i])
                     self.nDatas = nDatas
                     self.nDims = nDims
                     self.nDimsForDs = nDimsForDs
                     #Saving variables
                     print 'Writing the file: %s'%filename
                     self.filename = filename
                     self.fp = fp
                     self.grp = grp
                     self.grp.attrs.modify('nRecords', 1)
                     self.ds = ds
                     self.data = data
                     self.setFile = setFile
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if not self.firsttime:
                         self.readBlock()
                     if self.blockIndex == self.blocksPerFile:
                         self.setNextFile()
                     self.setBlock()
                     self.writeBlock()
                     self.fp.flush()
                     self.fp.close()
                     return
                 def readBlock(self):
                     '''
                     data Array configured
                     self.data
                     '''
                     ds = self.ds
                             #Setting HDF5 File
                     fp = h5py.File(self.filename,'r+')
                     grp = fp["Data"]
                     ind = 0
             #         grp.attrs['blocksPerFile'] = 0
                     for i in range(len(self.dataList)):
                         if self.nDims[i]==1:
                             ds0 = grp[self.dataList[i]]
                             ds[ind] = ds0
                             ind += 1
                         else:
                             if self.mode[i]==0:
                                 strMode = "channel"
                             else:
                                 strMode = "param"
                             grp0 = grp[self.dataList[i]]
                             for j in range(int(self.nDatas[i])):
                                 tableName = strMode + str(j)
                                 ds0 = grp0[tableName]
                                 ds[ind] = ds0
                                 ind += 1
                     self.fp = fp
                     self.grp = grp
                     self.ds = ds
                     return
                 def setBlock(self):
                     '''
                     data Array configured
                     self.data
                     '''
                     #Creating Arrays
                     data = self.data
                     nDatas = self.nDatas
                     nDims = self.nDims
                     mode = self.mode
                     ind = 0
                     for i in range(len(self.dataList)):
                         dataAux = getattr(self.dataOut,self.dataList[i])
                         if nDims[i] == 1:
             #                 data[ind] = numpy.array([str(dataAux)]).reshape((1,1))
                             data[ind] = dataAux
             #                 if not self.firsttime:
             #                     data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
                             ind += 1
                         else:
                             for j in range(int(nDatas[i])):
                                 if (mode[i] == 0) or (nDims[i] == 2):       #In case division per channel or Dimensions is only 1
                                     data[ind] = dataAux[j,:]
                                 else:
                                     data[ind] = dataAux[:,j,:]
             #                     if nDims[i] == 3:
             #                         data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
             #                         if not self.firsttime:
             #                             data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
             #                     else:
             #                         data[ind] = data[ind].reshape((1,data[ind].shape[0]))
             #                         if not self.firsttime:
             #                             data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
                                 ind += 1
                     self.data = data
                     return
                 def writeBlock(self):
                     '''
                     Saves the block in the HDF5 file
                     '''
                     for i in range(len(self.ds)):
                         if self.firsttime:
             #                 self.ds[i].resize(self.data[i].shape)
             #                 self.ds[i][self.blockIndex,:] = self.data[i]
                             if type(self.data[i]) == numpy.ndarray:
                                 nDims1 = len(self.ds[i].shape)
                                 if nDims1 == 3:
                                     self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
                                     self.ds[i].resize(self.data[i].shape)
                             self.ds[i][:] = self.data[i]
                         else:
                             if self.nDimsForDs[i] == 1:
                                 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
                                 self.ds[i][0,-1] = self.data[i]
                             elif self.nDimsForDs[i] == 2:
                                 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
                                 self.ds[i][self.blockIndex,:] = self.data[i]
                             elif self.nDimsForDs[i] == 3:
                                 dataShape = self.data[i].shape
                                 dsShape = self.ds[i].shape
                                 if dataShape[0]==dsShape[0]:
                                     self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
                                     self.ds[i][:,:,-1] = self.data[i]
                                 else:
                                     self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1],self.ds[i].shape[2]))
                                     self.ds[i][dsShape[0]:,:,0] = self.data[i]
             #                 self.ds[i].append(self.data[i])
             #                 self.fp.flush()
             #         if not self.firsttime:
             #             self.fp.root.Data._v_attrs.nRecords = self.blockIndex
             #         if self.firsttime:
             #             self.fp.close()
             #             self.readBlock2()
                     self.blockIndex += 1
                     self.firsttime = False
                     return
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                         self.putMetadata()
                         self.setNextFile()
                     self.putData()
                     return

schainpy/model/io/jroIO_base.py +69 -51

             '''
             Created on Jul 2, 2014
             @author: roj-idl71
             '''
             import os
             import sys
             import glob
             import time
             import numpy
             import fnmatch
             import time, datetime
             #import h5py
             import traceback
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
             LOCALTIME = True
             def isNumber(cad):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float( cad )
                     return True
                 except:
                     return False
             def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename,'rb')
                 except IOError:
-                    traceback.print_exc()
+                    print "The file %s can't be opened" %(filename)
-                    raise IOError, "The file %s can't be opened" %(filename)
+                    return 0
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename,'rb')
                 except IOError:
-                    traceback.print_exc()
+                    print "The file %s can't be opened" %(filename)
-                    raise IOError, "The file %s can't be opened" %(filename)
+                    return None
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = basicHeaderObj.datatime
                 thisDate = thisDatetime.date()
                 thisTime = thisDatetime.time()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return None
                 #General case
                 #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                 #-----------o----------------------------o-----------
                 #       startTime                     endTime
                 if endTime >= startTime:
                     if (thisTime < startTime) or (thisTime > endTime):
                         return None
                     return thisDatetime
                 #If endTime < startTime then endTime belongs to the next day
                 #<<<<<<<<<<<o                            o>>>>>>>>>>>
                 #-----------o----------------------------o-----------
                 #        endTime                    startTime
                 if (thisDate == startDate) and (thisTime < startTime):
                     return None
                 if (thisDate == endDate) and (thisTime > endTime):
                     return None
                 if (thisTime < startTime) and (thisTime > endTime):
                     return None
                 return thisDatetime
             def isFolderInDateRange(folder, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     folder            :    nombre completo del directorio.
                                            Su formato deberia ser "/path_root/?YYYYDDD"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el directorio no tiene el formato adecuado
                 """
                 basename = os.path.basename(folder)
                 if not isRadarFolder(basename):
-                    raise IOError, "The folder %s has not the rigth format" %folder
+                    print "The folder %s has not the rigth format" %folder
+                    return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFolder(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def isFileInDateRange(filename, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                                            Su formato deberia ser "?YYYYDDDsss"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                                                 sss     :    set
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no tiene el formato adecuado
                 """
                 basename = os.path.basename(filename)
                 if not isRadarFile(basename):
-                    raise IOError, "The filename %s has not the rigth format" %filename
+                    print "The filename %s has not the rigth format" %filename
+                    return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFile(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def getFileFromSet(path, ext, set):
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     try:
                         year = int(thisFile[1:5])
                         doy  = int(thisFile[5:8])
                     except:
                         continue
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
                 if len(myfile)!= 0:
                     return myfile[0]
                 else:
                     filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
                     print 'the filename %s does not exist'%filename
                     print '...going to the last file: '
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     year = thisFile[1:5]
                     if not isNumber(year):
                         continue
                     doy = thisFile[5:8]
                     if not isNumber(doy):
                         continue
                     year = int(year)
                     doy = int(doy)
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def checkForRealPath(path, foldercounter, year, doy, set, ext):
                 """
                 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
                 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
                 el path exacto de un determinado file.
                 Example    :
                     nombre correcto del file es  .../.../D2009307/P2009307367.ext
                     Entonces la funcion prueba con las siguientes combinaciones
                         .../.../y2009307367.ext
                         .../.../Y2009307367.ext
                         .../.../x2009307/y2009307367.ext
                         .../.../x2009307/Y2009307367.ext
                         .../.../X2009307/y2009307367.ext
                         .../.../X2009307/Y2009307367.ext
                     siendo para este caso, la ultima combinacion de letras, identica al file buscado
                 Return:
                     Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
                     caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
                     para el filename
                 """
                 fullfilename = None
                 find_flag = False
                 filename = None
                 prefixDirList = [None,'d','D']
                 if ext.lower() == ".r": #voltage
                     prefixFileList = ['d','D']
                 elif ext.lower() == ".pdata": #spectra
                     prefixFileList = ['p','P']
                 else:
                     return None, filename
                 #barrido por las combinaciones posibles
                 for prefixDir in prefixDirList:
                     thispath = path
                     if prefixDir != None:
                         #formo el nombre del directorio xYYYYDDD (x=d o x=D)
                         if foldercounter == 0:
                             thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
                         else:
                             thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
                     for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
                         filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
                         fullfilename = os.path.join( thispath, filename ) #formo el path completo
                         if os.path.exists( fullfilename ): #verifico que exista
                             find_flag = True
                             break
                     if find_flag:
                         break
                 if not(find_flag):
                     return None, filename
                 return fullfilename, filename
             def isRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
             def isRadarFile(file):
                     try:
                         year = int(file[1:5])
                         doy = int(file[5:8])
                         set = int(file[8:11])
                     except:
                         return 0
                     return 1
             def getDateFromRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
                 return thisDate
             def getDateFromRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
                 return thisDate
             class JRODataIO:
                 c = 3E8
                 isConfig = False
                 basicHeaderObj = None
                 systemHeaderObj = None
                 radarControllerHeaderObj = None
                 processingHeaderObj = None
                 dtype = None
                 pathList = []
                 filenameList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagDiscontinuousBlock = 0
                 flagIsNewBlock = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
             #     ippSeconds = None
                 fileSizeByHeader = None
                 fileIndex = None
                 profileIndex = None
                 blockIndex = None
                 nTotalBlocks = None
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
                 blocksize = None
                 getByBlock = False
                 def __init__(self):
-                    raise ValueError, "Not implemented"
+                    raise NotImplementedError
                 def run(self):
-                    raise ValueError, "Not implemented"
+                    raise NotImplementedError
                 def getDtypeWidth(self):
                     dtype_index = get_dtype_index(self.dtype)
                     dtype_width = get_dtype_width(dtype_index)
                     return dtype_width
             class JRODataReader(JRODataIO):
                 online = 0
                 realtime = 0
                 nReadBlocks = 0
                 delay  = 10   #number of seconds waiting a new file
                 nTries  = 3  #quantity tries
                 nFiles = 3   #number of files for searching
                 path = None
                 foldercounter = 0
                 flagNoMoreFiles = 0
                 datetimeList = []
                 __isFirstTimeOnline = 1
                 __printInfo = True
                 profileIndex = None
                 nTxs = 1
                 txIndex = None
                 def __init__(self):
                     """
+                    This class is used to find data files
-                    """
-            #         raise NotImplementedError, "This method has not been implemented"
+                    Example:
+                        reader = JRODataReader()
+                        fileList = reader.findDataFiles()
+                    """
+                    pass
                 def createObjByDefault(self):
                     """
                     """
-                    raise NotImplementedError, "This method has not been implemented"
+                    raise NotImplementedError
                 def getBlockDimension(self):
-                    raise NotImplementedError, "No implemented"
+                    raise NotImplementedError
                 def __searchFilesOffLine(self,
                                         path,
                                         startDate=None,
                                         endDate=None,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.r',
                                         walk=True):
                     self.filenameList = []
                     self.datetimeList = []
                     pathList = []
                     dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
                     if dateList == []:
-                        print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
+            #             print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
-                                                                    datetime.datetime.combine(startDate,startTime).ctime(),
+            #                                                         datetime.datetime.combine(startDate,startTime).ctime(),
-                                                                    datetime.datetime.combine(endDate,endTime).ctime())
+            #                                                         datetime.datetime.combine(endDate,endTime).ctime())
                         return None, None
                     if len(dateList) > 1:
                         print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
                     else:
                         print "[Reading] data was found for the date %s" %(dateList[0])
                     filenameList = []
                     datetimeList = []
                     for thisPath in pathList:
             #             thisPath = pathList[pathDict[file]]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             if not isFileInDateRange(filename, startDate, endDate):
                                 continue
                             thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         return None, None
                     print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
                     """
                     Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
                     devuelve el archivo encontrado ademas de otros datos.
                     Input:
                         path           :    carpeta donde estan contenidos los files que contiene data
                         expLabel        :     Nombre del subexperimento (subfolder)
                         ext              :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
                         directory   :    eL directorio donde esta el file encontrado
                         filename    :    el ultimo file de una determinada carpeta
                         year        :    el anho
                         doy         :    el numero de dia del anho
                         set         :    el set del archivo
                     """
                     dirList = []
                     if not walk:
                         fullpath = path
                         foldercounter = 0
                     else:
                         #Filtra solo los directorios
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isRadarFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None, None, None, None, None
                         dirList = sorted( dirList, key=str.lower )
                         doypath = dirList[-1]
                         foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
                         fullpath = os.path.join(path, doypath, expLabel)
                     print "[Reading] %s folder was found: " %(fullpath )
                     if set == None:
                         filename = getlastFileFromPath(fullpath, ext)
                     else:
                         filename = getFileFromSet(fullpath, ext, set)
                     if not(filename):
                         return None, None, None, None, None, None
                     print "[Reading] %s file was found" %(filename)
                     if not(self.__verifyFile(os.path.join(fullpath, filename))):
                         return None, None, None, None, None, None
                     year = int( filename[1:5] )
                     doy  = int( filename[5:8] )
                     set  = int( filename[8:11] )
                     return fullpath, foldercounter, filename, year, doy, set
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
             #                 print "[Reading] No more Files"
                             return 0
                         filename = self.filenameList[idFile]
                         if not(self.__verifyFile(filename)):
                             continue
                         fileSize = os.path.getsize(filename)
                         fp = open(filename,'rb')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fp = fp
             #         print "[Reading] Setting the file: %s"%self.filename
                     return 1
                 def __setNextFileOnline(self):
                     """
                     Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
                     no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
                     siguientes.
                     Affected:
                         self.flagIsNewFile
                         self.filename
                         self.fileSize
                         self.fp
                         self.set
                         self.flagNoMoreFiles
                     Return:
 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
 : si el file fue abierto con exito y esta listo a ser leido
                     Excepciones:
                         Si un determinado file no puede ser abierto
                     """
                     nFiles = 0
                     fileOk_flag = False
                     firstTime_flag = True
                     self.set += 1
                     if self.set > 999:
                         self.set = 0
                         self.foldercounter += 1
                     #busca el 1er file disponible
                     fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                     if fullfilename:
                         if self.__verifyFile(fullfilename, False):
                             fileOk_flag = True
                     #si no encuentra un file entonces espera y vuelve a buscar
                     if not(fileOk_flag):
                         for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
                             if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
                                 tries = self.nTries
                             else:
                                 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
                             for nTries in range( tries ):
                                 if firstTime_flag:
                                     print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
                                     sleep( self.delay )
                                 else:
                                     print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
                                 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                                 if fullfilename:
                                     if self.__verifyFile(fullfilename):
                                         fileOk_flag = True
                                         break
                             if fileOk_flag:
                                 break
                             firstTime_flag = False
                             print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
                             self.set += 1
                             if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
                                 self.set = 0
                                 self.doy += 1
                                 self.foldercounter = 0
                     if fileOk_flag:
                         self.fileSize = os.path.getsize( fullfilename )
                         self.filename = fullfilename
                         self.flagIsNewFile = 1
                         if self.fp != None: self.fp.close()
                         self.fp = open(fullfilename, 'rb')
                         self.flagNoMoreFiles = 0
             #             print '[Reading] Setting the file: %s' % fullfilename
                     else:
                         self.fileSize = 0
                         self.filename = None
                         self.flagIsNewFile = 0
                         self.fp = None
                         self.flagNoMoreFiles = 1
             #             print '[Reading] No more files to read'
                     return fileOk_flag
                 def setNextFile(self):
                     if self.fp != None:
                         self.fp.close()
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         print '[Reading] No more files to read'
                         return 0
                     print '[Reading] Setting the file: %s' % self.filename
                     self.__readFirstHeader()
                     self.nReadBlocks = 0
                     return 1
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.basicHeaderObj.read(self.fp)
                             return 1
                         if self.fileSize == self.fileSizeByHeader:
             #                self.flagEoF = True
                             return 0
                         print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         sleep( self.delay )
                     return 0
                 def waitDataBlock(self,pointer_location):
                     currentPointer = pointer_location
                     neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             return 1
                         print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         sleep( self.delay )
                     return 0
                 def __jumpToLastBlock(self):
                     if not(self.__isFirstTimeOnline):
                         return
                     csize = self.fileSize - self.fp.tell()
                     blocksize = self.processingHeaderObj.blockSize
                     #salta el primer bloque de datos
                     if csize > self.processingHeaderObj.blockSize:
                         self.fp.seek(self.fp.tell() + blocksize)
                     else:
                         return
                     csize = self.fileSize - self.fp.tell()
                     neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     while True:
                         if self.fp.tell()<self.fileSize:
                             self.fp.seek(self.fp.tell() + neededsize)
                         else:
                             self.fp.seek(self.fp.tell() - neededsize)
                             break
             #        csize = self.fileSize - self.fp.tell()
             #        neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
             #        factor = int(csize/neededsize)
             #        if factor > 0:
             #            self.fp.seek(self.fp.tell() + factor*neededsize)
                     self.flagIsNewFile = 0
                     self.__isFirstTimeOnline = 0
                 def __setNewBlock(self):
                     if self.fp == None:
                         return 0
             #         if self.online:
             #             self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if self.realtime:
                         self.flagDiscontinuousBlock = 1
                         if not(self.setNextFile()):
                             return 0
                         else:
                             return 1
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.basicHeaderObj.read(self.fp)
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if self.__waitNewBlock():
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
                     self.lastUTTime = self.basicHeaderObj.utc
                     self.flagDiscontinuousBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagDiscontinuousBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     self.getBasicHeader()
                     print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
                                                               self.processingHeaderObj.dataBlocksPerFile,
                                                               self.dataOut.datatime.ctime())
                     return 1
                 def __readFirstHeader(self):
                     self.basicHeaderObj.read(self.fp)
                     self.systemHeaderObj.read(self.fp)
                     self.radarControllerHeaderObj.read(self.fp)
                     self.processingHeaderObj.read(self.fp)
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
                     else:
                         raise ValueError, 'Data type was not defined'
                     self.dtype = datatype_str
                     #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
             #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
             #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def __verifyFile(self, filename, msgFlag=True):
                     msg = None
                     try:
                         fp = open(filename, 'rb')
-                        currentPosition = fp.tell()
                     except IOError:
-                        traceback.print_exc()
                         if msgFlag:
-                            print "[Reading] The file %s can't be opened" % (filename)
+                            print "[Reading] File %s can't be opened" % (filename)
                         return False
+                    currentPosition = fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                     if neededSize == 0:
                         basicHeaderObj = BasicHeader(LOCALTIME)
                         systemHeaderObj = SystemHeader()
                         radarControllerHeaderObj = RadarControllerHeader()
                         processingHeaderObj = ProcessingHeader()
-                        try:
+                        if not( basicHeaderObj.read(fp) ):
-                            if not( basicHeaderObj.read(fp) ): raise IOError
+                            fp.close()
-                            if not( systemHeaderObj.read(fp) ): raise IOError
+                            return False
-                            if not( radarControllerHeaderObj.read(fp) ): raise IOError
-                            if not( processingHeaderObj.read(fp) ): raise IOError
+                        if not( systemHeaderObj.read(fp) ):
-            #                 data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
-                            neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
-                        except IOError:
-                            traceback.print_exc()
-            #                 sys.exit(0)
-                            if msgFlag:
-                                print "[Reading] The file %s is empty or it hasn't enough data" % filename
                             fp.close()
                             return False
+                        if not( radarControllerHeaderObj.read(fp) ):
+                            fp.close()
+                            return False
+                        if not( processingHeaderObj.read(fp) ):
+                            fp.close()
+                            return False
+                        neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
                     else:
                         msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
                     fp.close()
                     fileSize = os.path.getsize(filename)
                     currentSize = fileSize - currentPosition
                     if currentSize < neededSize:
                         if msgFlag and (msg != None):
-                            print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
+                            print msg
                         return False
                     return True
                 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
                     dateList = []
                     pathList = []
                     multi_path = path.split(',')
                     if not walk:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             fileList = glob.glob1(single_path, "*"+ext)
                             for thisFile in fileList:
                                 if not os.path.isfile(os.path.join(single_path, thisFile)):
                                     continue
                                 if not isRadarFile(thisFile):
                                     continue
                                 if not isFileInDateRange(thisFile, startDate, endDate):
                                     continue
                                 thisDate = getDateFromRadarFile(thisFile)
                                 if thisDate in dateList:
                                     continue
                                 dateList.append(thisDate)
                                 pathList.append(single_path)
                     else:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             dirList = []
                             for thisPath in os.listdir(single_path):
                                 if not os.path.isdir(os.path.join(single_path,thisPath)):
                                     continue
                                 if not isRadarFolder(thisPath):
                                     continue
                                 if not isFolderInDateRange(thisPath, startDate, endDate):
                                     continue
                                 dirList.append(thisPath)
                             if not dirList:
                                 continue
                             for thisDir in dirList:
                                 datapath = os.path.join(single_path, thisDir, expLabel)
                                 fileList = glob.glob1(datapath, "*"+ext)
                                 if len(fileList) < 1:
                                     continue
                                 thisDate = getDateFromRadarFolder(thisDir)
                                 pathList.append(datapath)
                                 dateList.append(thisDate)
                     dateList.sort()
+                    if walk:
+                        pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
+                    else:
+                        pattern_path = multi_path[0]
+                    if not dateList:
+                        print "[Reading] No *%s files in %s from %s to %s" %(ext, pattern_path, startDate, endDate)
                     if include_path:
                         return dateList, pathList
                     return dateList
                 def setup(self,
                             path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=None,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True,
                             getblock = False,
                             nTxs = 1,
                             realtime=False):
                     if path == None:
                         raise ValueError, "[Reading] The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if online:
                         print "[Reading] Searching files in online mode..."
                         for nTries in range( self.nTries ):
                             fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
                             if fullpath:
                                 break
                             print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
                             sleep( self.delay )
                         if not(fullpath):
                             print "[Reading] There 'isn't any valid file in %s" % path
                             return
                         self.year = year
                         self.doy  = doy
                         self.set  = set - 1
                         self.path = path
                         self.foldercounter = foldercounter
                         last_set = None
                     else:
                         print "[Reading] Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
-                            print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
+            #                 print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
-                                                                    datetime.datetime.combine(startDate,startTime).ctime(),
+            #                                                         datetime.datetime.combine(startDate,startTime).ctime(),
-                                                                    datetime.datetime.combine(endDate,endTime).ctime())
+            #                                                         datetime.datetime.combine(endDate,endTime).ctime())
             #                 sys.exit(-1)
                             self.fileIndex = -1
                             self.pathList = []
                             self.filenameList = []
                             return
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                         file_name = os.path.basename(filenameList[-1])
                         basename, ext = os.path.splitext(file_name)
                         last_set = int(basename[-3:])
                     self.online = online
                     self.realtime = realtime
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     self.getByBlock = getblock
                     self.nTxs = int(nTxs)
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "[Reading] No files"
                             self.fileIndex = -1
                             self.pathList = []
                             self.filenameList = []
                             return
             #         self.getBasicHeader()
                     if last_set != None:
                         self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
                     return
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                     self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
                 def getFirstHeader(self):
-                    raise ValueError, "This method has not been implemented"
+                    raise NotImplementedError
                 def getData(self):
-                    raise ValueError, "This method has not been implemented"
+                    raise NotImplementedError
                 def hasNotDataInBuffer(self):
-                    raise ValueError, "This method has not been implemented"
+                    raise NotImplementedError
                 def readBlock(self):
-                    raise ValueError, "This method has not been implemented"
+                    raise NotImplementedError
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
                 def printTotalBlocks(self):
                     print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
                 def printNumberOfBlock(self):
                     if self.flagIsNewBlock:
                         print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
                                                                   self.processingHeaderObj.dataBlocksPerFile,
                                                                   self.dataOut.datatime.ctime())
                 def printInfo(self):
                     if self.__printInfo == False:
                         return
                     self.basicHeaderObj.printInfo()
                     self.systemHeaderObj.printInfo()
                     self.radarControllerHeaderObj.printInfo()
                     self.processingHeaderObj.printInfo()
                     self.__printInfo = False
                 def run(self, **kwargs):
                     if not(self.isConfig):
             #            self.dataOut = dataOut
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             class JRODataWriter(JRODataIO):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 blockIndex = 0
                 path = None
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 fileDate = None
                 def __init__(self, dataOut=None):
-                    raise ValueError, "Not implemented"
+                    raise NotImplementedError
                 def hasAllDataInBuffer(self):
-                    raise ValueError, "Not implemented"
+                    raise NotImplementedError
                 def setBlockDimension(self):
-                    raise ValueError, "Not implemented"
+                    raise NotImplementedError
                 def writeBlock(self):
-                    raise ValueError, "No implemented"
+                    raise NotImplementedError
                 def putData(self):
-                    raise ValueError, "No implemented"
+                    raise NotImplementedError
                 def getProcessFlags(self):
                     processFlags = 0
                     dtype_index = get_dtype_index(self.dtype)
                     procflag_dtype = get_procflag_dtype(dtype_index)
                     processFlags += procflag_dtype
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code is not None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nCohInt > 1:
                         processFlags += PROCFLAG.COHERENT_INTEGRATION
                     if self.dataOut.type == "Spectra":
                         if self.dataOut.nIncohInt > 1:
                             processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                         if self.dataOut.data_dc is not None:
                             processFlags += PROCFLAG.SAVE_CHANNELS_DC
                         if self.dataOut.flagShiftFFT:
                             processFlags += PROCFLAG.SHIFT_FFT_DATA
                     return processFlags
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize #bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond  = (self.dataOut.utctime - utc)* 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
-                    raise ValueError, "No implemented"
+                    raise NotImplementedError
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     print "[Writing] Block No. %d/%d" %(self.blockIndex,
                                                         self.processingHeaderObj.dataBlocksPerFile)
                     return 1
                 def setNextFile(self):
                     """
                     Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     setFile = self.setFile
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     setFile += 1
                     #If this is a new day it resets some values
                     if self.dataOut.datatime.date() > self.fileDate:
                         setFile = 0
                         self.nTotalBlocks = 0
                     filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
                     filename = os.path.join( path, subfolder, filen )
                     fp = open( filename,'wb' )
                     self.blockIndex = 0
                     #guardando atributos
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.fileDate = self.dataOut.datatime.date()
                     self.setFirstHeader()
                     print '[Writing] Opening file: %s'%self.filename
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path                :    directory where data will be saved
                         profilesPerBlock    :    number of profiles per block
                         set                 :    initial file set
                         datatype            :    An integer number that defines data type:
 : int8  (1 byte)
 : int16 (2 bytes)
 : int32 (4 bytes)
 : int64 (8 bytes)
 : float32 (4 bytes)
 : double64 (8 bytes)
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     self.ext = ext.lower()
                     self.path = path
                     if set is None:
                         self.setFile = -1
                     else:
                         self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     self.fileDate = self.dataOut.datatime.date()
                     #By default
                     self.dtype = self.dataOut.dtype
                     if datatype is not None:
                         self.dtype = get_numpy_dtype(datatype)
                     if not(self.setNextFile()):
                         print "[Writing] There isn't a next file"
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()

schainpy/model/io/jroIO_heispectra.py +4 -3

             '''
             Created on Jul 3, 2014
             @author: roj-idl71
             '''
             import os, sys
             import time, datetime
             import numpy
             import fnmatch
             import glob
             from time import sleep
             try:
                 import pyfits
             except ImportError, e:
                 print "Fits data cannot be used. Install pyfits module"
             from xml.etree.ElementTree import ElementTree
             from jroIO_base import isRadarFolder, isNumber
             from schainpy.model.data.jrodata import Fits
             from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
             class PyFits(object):
                 name=None
                 format=None
                 array =None
                 data =None
                 thdulist=None
                 prihdr=None
                 hdu=None
                 def __init__(self):
                     pass
                 def setColF(self,name,format,array):
                     self.name=name
                     self.format=format
                     self.array=array
                     a1=numpy.array([self.array],dtype=numpy.float32)
                     self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
                     return self.col1
             #    def setColP(self,name,format,data):
             #        self.name=name
             #        self.format=format
             #        self.data=data
             #        a2=numpy.array([self.data],dtype=numpy.float32)
             #        self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
             #        return self.col2
                 def writeData(self,name,format,data):
                     self.name=name
                     self.format=format
                     self.data=data
                     a2=numpy.array([self.data],dtype=numpy.float32)
                     self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
                     return self.col2
                 def cFImage(self,idblock,year,month,day,hour,minute,second):
                     self.hdu= pyfits.PrimaryHDU(idblock)
                     self.hdu.header.set("Year",year)
                     self.hdu.header.set("Month",month)
                     self.hdu.header.set("Day",day)
                     self.hdu.header.set("Hour",hour)
                     self.hdu.header.set("Minute",minute)
                     self.hdu.header.set("Second",second)
                     return self.hdu
                 def Ctable(self,colList):
                     self.cols=pyfits.ColDefs(colList)
                     self.tbhdu = pyfits.new_table(self.cols)
                     return self.tbhdu
                 def CFile(self,hdu,tbhdu):
                     self.thdulist=pyfits.HDUList([hdu,tbhdu])
                 def wFile(self,filename):
                     if os.path.isfile(filename):
                         os.remove(filename)
                     self.thdulist.writeto(filename)
             class ParameterConf:
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.name = ''
                     self.value = ''
                 def readXml(self, parmElement):
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                 def getElementName(self):
                     return self.ELEMENTNAME
             class Metadata(object):
                 def __init__(self, filename):
                     self.parmConfObjList = []
                     self.readXml(filename)
                 def readXml(self, filename):
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     self.projectElement = ElementTree().parse(filename)
                     self.project = self.projectElement.tag
                     parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         self.parmConfObjList.append(parmConfObj)
             class FitsWriter(Operation):
                 def __init__(self):
                     self.isConfig = False
                     self.dataBlocksPerFile = None
                     self.blockIndex = 0
                     self.flagIsNewFile = 1
                     self.fitsObj = None
                     self.optchar = 'P'
                     self.ext = '.fits'
                     self.setFile = 0
                 def setFitsHeader(self, dataOut, metadatafile=None):
                     header_data = pyfits.PrimaryHDU()
                     header_data.header['EXPNAME'] = "RADAR DATA"
                     header_data.header['DATATYPE'] = "SPECTRA"
                     header_data.header['COMMENT'] = ""
                     if metadatafile:
                         metadata4fits = Metadata(metadatafile)
                         for parameter in metadata4fits.parmConfObjList:
                             parm_name = parameter.name
                             parm_value = parameter.value
                             header_data.header[parm_name] = parm_value
                     header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
                     header_data.header['CHANNELLIST'] = str(dataOut.channelList)
                     header_data.header['NCHANNELS'] = dataOut.nChannels
                     #header_data.header['HEIGHTS'] = dataOut.heightList
                     header_data.header['NHEIGHTS'] = dataOut.nHeights
                     header_data.header['IPPSECONDS'] = dataOut.ippSeconds
                     header_data.header['NCOHINT'] = dataOut.nCohInt
                     header_data.header['NINCOHINT'] = dataOut.nIncohInt
                     header_data.header['TIMEZONE'] = dataOut.timeZone
                     header_data.header['NBLOCK'] = self.blockIndex
                     header_data.writeto(self.filename)
                     self.addExtension(dataOut.heightList,'HEIGHTLIST')
                 def setup(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None):
                     self.path = path
                     self.dataOut = dataOut
                     self.metadatafile = metadatafile
                     self.dataBlocksPerFile = dataBlocksPerFile
                 def open(self):
                     self.fitsObj = pyfits.open(self.filename, mode='update')
                 def addExtension(self, data, tagname):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=tagname)
                     #extension.header['TAG'] = tagname
                     self.fitsObj.append(extension)
                     self.write()
                 def addData(self, data):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
                     extension.header['UTCTIME'] = self.dataOut.utctime
                     self.fitsObj.append(extension)
                     self.blockIndex += 1
                     self.fitsObj[0].header['NBLOCK'] = self.blockIndex
                     self.write()
                 def write(self):
                     self.fitsObj.flush(verbose=True)
                     self.fitsObj.close()
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     thisFile = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, thisFile )
                     self.blockIndex = 0
                     self.filename = filename
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     print 'Writing the file: %s'%self.filename
                     self.setFitsHeader(self.dataOut, self.metadatafile)
                     return 1
                 def writeBlock(self):
                     self.addData(self.dataOut.data_spc)
                     self.flagIsNewFile = 0
                 def __setNewBlock(self):
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.dataBlocksPerFile:
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def putData(self):
                     if self.flagIsNewFile:
                         self.setNextFile()
                     self.writeNextBlock()
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class FitsReader(ProcessingUnit):
             #     __TIMEZONE = time.timezone
                 expName = None
                 datetimestr = None
                 utc = None
                 nChannels = None
                 nSamples = None
                 dataBlocksPerFile = None
                 comments = None
                 lastUTTime = None
                 header_dict = None
                 data = None
                 data_header_dict = None
                 def __init__(self):
                     self.isConfig = False
                     self.ext = '.fits'
                     self.setFile = 0
                     self.flagNoMoreFiles = 0
                     self.flagIsNewFile = 1
                     self.flagDiscontinuousBlock = None
                     self.fileIndex = None
                     self.filename = None
                     self.fileSize = None
                     self.fitsObj = None
                     self.timeZone = None
                     self.nReadBlocks = 0
                     self.nTotalBlocks = 0
                     self.dataOut = self.createObjByDefault()
                     self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
                     self.blockIndex = 1
                 def createObjByDefault(self):
                     dataObj = Fits()
                     return dataObj
                 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
                     try:
                         fitsObj = pyfits.open(filename,'readonly')
                     except:
-                        raise IOError, "The file %s can't be opened" %(filename)
+                        print "File %s can't be opened" %(filename)
+                        return None
                     header = fitsObj[0].header
                     struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
                     utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
                     ltc = utc
                     if useLocalTime:
                         ltc -= time.timezone
                     thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
                     thisTime = thisDatetime.time()
                     if not ((startTime <= thisTime) and (endTime > thisTime)):
                         return None
                     return thisDatetime
                 def __setNextFileOnline(self):
-                    raise ValueError, "No implemented"
+                    raise NotImplementedError
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print "No more Files"
                             return 0
                         filename = self.filenameList[idFile]
             #            if not(self.__verifyFile(filename)):
             #                continue
                         fileSize = os.path.getsize(filename)
                         fitsObj = pyfits.open(filename,'readonly')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fitsObj = fitsObj
                     self.blockIndex = 0
                     print "Setting the file: %s"%self.filename
                     return 1
                 def __setValuesFromHeader(self):
                     self.dataOut.header = self.header_dict
                     self.dataOut.expName = self.expName
                     self.dataOut.timeZone = self.timeZone
                     self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
                     self.dataOut.comments = self.comments
             #         self.dataOut.timeInterval = self.timeInterval
                     self.dataOut.channelList = self.channelList
                     self.dataOut.heightList = self.heightList
                     self.dataOut.nCohInt = self.nCohInt
                     self.dataOut.nIncohInt = self.nIncohInt
                     self.dataOut.ippSeconds = self.ippSeconds
                 def readHeader(self):
                     headerObj = self.fitsObj[0]
                     self.header_dict = headerObj.header
                     if 'EXPNAME' in headerObj.header.keys():
                         self.expName = headerObj.header['EXPNAME']
                     if 'DATATYPE' in headerObj.header.keys():
                         self.dataType = headerObj.header['DATATYPE']
                     self.datetimestr = headerObj.header['DATETIME']
                     channelList = headerObj.header['CHANNELLIST']
                     channelList = channelList.split('[')
                     channelList = channelList[1].split(']')
                     channelList = channelList[0].split(',')
                     channelList = [int(ch) for ch in channelList]
                     self.channelList = channelList
                     self.nChannels = headerObj.header['NCHANNELS']
                     self.nHeights = headerObj.header['NHEIGHTS']
                     self.ippSeconds = headerObj.header['IPPSECONDS']
                     self.nCohInt = headerObj.header['NCOHINT']
                     self.nIncohInt = headerObj.header['NINCOHINT']
                     self.dataBlocksPerFile = headerObj.header['NBLOCK']
                     self.timeZone = headerObj.header['TIMEZONE']
             #         self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     if 'COMMENT' in headerObj.header.keys():
                         self.comments = headerObj.header['COMMENT']
                     self.readHeightList()
                 def readHeightList(self):
                     self.blockIndex = self.blockIndex + 1
                     obj = self.fitsObj[self.blockIndex]
                     self.heightList = obj.data
                     self.blockIndex = self.blockIndex + 1
                 def readExtension(self):
                     obj = self.fitsObj[self.blockIndex]
                     self.heightList = obj.data
                     self.blockIndex = self.blockIndex + 1
                 def setNextFile(self):
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         return 0
                     self.readHeader()
                     self.__setValuesFromHeader()
                     self.nReadBlocks = 0
             #         self.blockIndex = 1
                     return 1
                 def __searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.fits',
                                         walk=True):
                     pathList = []
                     if not walk:
                         pathList.append(path)
                     else:
                         dirList = []
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isRadarFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None
                         thisDate = startDate
                         while(thisDate <= endDate):
                             year = thisDate.timetuple().tm_year
                             doy = thisDate.timetuple().tm_yday
                             matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
                             if len(matchlist) == 0:
                                 thisDate += datetime.timedelta(1)
                                 continue
                             for match in matchlist:
                                 pathList.append(os.path.join(path,match,expLabel))
                             thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
                         return None, None
                     print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
                     filenameList = []
                     datetimeList = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for thisFile in fileList:
                             filename = os.path.join(thisPath,thisFile)
                             thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "Any file was found for the time range %s - %s" %(startTime, endTime)
                         return None, None
                     print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def setup(self, path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=0,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True):
                     if path == None:
                         raise ValueError, "The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if not(online):
                         print "Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
                             print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime())
                             sys.exit(-1)
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                     self.online = online
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "No files"
                         sys.exit(-1)
                 def readBlock(self):
                     dataObj = self.fitsObj[self.blockIndex]
                     self.data = dataObj.data
                     self.data_header_dict = dataObj.header
                     self.utc = self.data_header_dict['UTCTIME']
                     self.flagIsNewFile = 0
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def __jumpToLastBlock(self):
-                    raise ValueError, "No implemented"
+                    raise NotImplementedError
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.__rdBasicHeader()
                             return 1
                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         sleep( self.delay )
                     return 0
                 def __setNewBlock(self):
                     if self.online:
                         self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         return 1
                     self.lastUTTime = self.utc
                     if self.online:
                         if self.__waitNewBlock():
                             return 1
                     if self.nReadBlocks < self.dataBlocksPerFile:
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.utc - self.lastUTTime
                     self.flagDiscontinuousBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagDiscontinuousBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def printInfo(self):
                     pass
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagDiscontinuousBlock = 0
                     self.flagIsNewBlock = 0
                     if not(self.readNextBlock()):
                         return 0
                     if self.data is None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.dataOut.data = self.data
                     self.dataOut.data_header = self.data_header_dict
                     self.dataOut.utctime = self.utc
             #         self.dataOut.header = self.header_dict
             #         self.dataOut.expName = self.expName
             #         self.dataOut.nChannels = self.nChannels
             #         self.dataOut.timeZone = self.timeZone
             #         self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
             #         self.dataOut.comments = self.comments
             # #         self.dataOut.timeInterval = self.timeInterval
             #         self.dataOut.channelList = self.channelList
             #         self.dataOut.heightList = self.heightList
                     self.dataOut.flagNoData = False
                     return self.dataOut.data
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             class SpectraHeisWriter(Operation):
             #    set = None
                 setFile = None
                 idblock = None
                 doypath = None
                 subfolder = None
                 def __init__(self):
                     self.wrObj = PyFits()
             #        self.dataOut = dataOut
                     self.nTotalBlocks=0
             #        self.set = None
                     self.setFile = None
                     self.idblock = 0
                     self.wrpath = None
                     self.doypath = None
                     self.subfolder = None
                     self.isConfig = False
                 def isNumber(str):
                     """
                     Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                     Excepciones:
                     Si un determinado string no puede ser convertido a numero
                     Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                     Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                     """
                     try:
                         float( str )
                         return True
                     except:
                         return False
                 def setup(self, dataOut, wrpath):
                     if not(os.path.exists(wrpath)):
                         os.mkdir(wrpath)
                     self.wrpath = wrpath
             #        self.setFile = 0
                     self.dataOut = dataOut
                 def putData(self):
                     name= time.localtime( self.dataOut.utctime)
                     ext=".fits"
                     if self.doypath == None:
                        self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
                        self.doypath = os.path.join( self.wrpath, self.subfolder )
                        os.mkdir(self.doypath)
                     if self.setFile == None:
             #           self.set = self.dataOut.set
                        self.setFile = 0
             #        if self.set != self.dataOut.set:
             ##            self.set = self.dataOut.set
             #            self.setFile = 0
                     #make the filename
                     thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
                     filename = os.path.join(self.wrpath,self.subfolder, thisFile)
                     idblock = numpy.array([self.idblock],dtype="int64")
                     header=self.wrObj.cFImage(idblock=idblock,
                                             year=time.gmtime(self.dataOut.utctime).tm_year,
                                             month=time.gmtime(self.dataOut.utctime).tm_mon,
                                             day=time.gmtime(self.dataOut.utctime).tm_mday,
                                             hour=time.gmtime(self.dataOut.utctime).tm_hour,
                                             minute=time.gmtime(self.dataOut.utctime).tm_min,
                                             second=time.gmtime(self.dataOut.utctime).tm_sec)
                     c=3E8
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
                     colList = []
                     colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
                     colList.append(colFreq)
                     nchannel=self.dataOut.nChannels
                     for i in range(nchannel):
                         col = self.wrObj.writeData(name="PCh"+str(i+1),
                                                      format=str(self.dataOut.nFFTPoints)+'E',
                                                       data=10*numpy.log10(self.dataOut.data_spc[i,:]))
                         colList.append(col)
                     data=self.wrObj.Ctable(colList=colList)
                     self.wrObj.CFile(header,data)
                     self.wrObj.wFile(filename)
                     #update the setFile
                     self.setFile += 1
                     self.idblock += 1
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()

schainpy/model/io/jroIO_usrp.py +5 -5

             '''
             Created on Jul 3, 2014
             @author: roj-idl71
             '''
             import os
             import datetime
             import numpy
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             try:
                 import digital_rf_hdf5
             except:
                 print 'You should install "digital_rf_hdf5" module if you want to read USRP data'
             class USRPReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self)
                     self.dataOut = Voltage()
                     self.__printInfo = True
                     self.__flagDiscontinuousBlock = False
                     self.__bufferIndex = 9999999
                     self.__ippKm = None
                     self.__codeType = 0
                     self.__nCode = None
                     self.__nBaud = None
                     self.__code = None
                 def __getCurrentSecond(self):
                     return self.__thisUnixSample/self.__sample_rate
                 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
                 def __setFileHeader(self):
                     '''
                     In this method will be initialized every parameter of dataOut object (header, no data)
                     '''
                     nProfiles = self.__sample_rate  #Number of profiles by second
                     self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
                                                                                   txA=0,
                                                                                   txB=0,
                                                                                   nWindows=1,
                                                                                   nHeights=self.__nSamples,
                                                                                   firstHeight=self.__firstHeigth,
                                                                                   deltaHeight=self.__deltaHeigth,
                                                                                   codeType=self.__codeType,
                                                                                   nCode=self.__nCode, nBaud=self.__nBaud,
                                                                                   code = self.__code)
                     self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                 nProfiles=nProfiles,
                                                                 nChannels=len(self.__channelList),
                                                                 adcResolution=14)
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
             #        self.dataOut.nChannels = 0
             #        self.dataOut.nHeights = 0
                     self.dataOut.nProfiles = nProfiles
                     self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
                     self.dataOut.channelList = self.__channelList
                     self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
             #        self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     #Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     self.dataOut.timeZone = self.__timezone/60  #timezone like jroheader, difference in minutes between UTC and localtime
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     self.dataOut.nCohInt = 1
                     self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
                     self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
                     #Time interval between profiles
                     #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.__online
                 def findDatafiles(self, path, startDate=None, endDate=None):
                     if not os.path.isdir(path):
                         return []
                     try:
                         digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
                     except:
                         digitalReadObj = digital_rf_hdf5.read_hdf5(path)
                     channelNameList = digitalReadObj.get_channels()
                     if not channelNameList:
                         return []
                     metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
                     sample_rate = metadata_dict['sample_rate'][0]
                     this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
                     try:
                         timezone = this_metadata_file['timezone'].value
                     except:
                         timezone = 0
                     startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(channelNameList[0])/sample_rate - timezone
                     startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
                     endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
                     if not startDate:
                         startDate = startDatetime.date()
                     if not endDate:
                         endDate = endDatatime.date()
                     dateList = []
                     thisDatetime = startDatetime
                     while(thisDatetime<=endDatatime):
                         thisDate = thisDatetime.date()
                         if thisDate < startDate:
                             continue
                         if thisDate > endDate:
                             break
                         dateList.append(thisDate)
                         thisDatetime += datetime.timedelta(1)
                     return dateList
                 def setup(self, path = None,
                                 startDate = None,
                                 endDate = None,
                                 startTime = datetime.time(0,0,0),
                                 endTime = datetime.time(23,59,59),
                                 channelList = None,
                                 nSamples = None,
                                 ippKm = 60,
                                 online = False,
                                 delay = 60,
                                 buffer_size = None,
                                 nbuffer = 1024,
                                 **kwargs):
                     '''
                     In this method we should set all initial parameters.
                     Inputs:
                         path
                         startDate
                         endDate
                         startTime
                         endTime
                         set
                         expLabel
                         ext
                         online
                         delay
                     '''
                     if not buffer_size:
                         buffer_size = nbuffer
                     if not os.path.isdir(path):
-                        raise ValueError, "[Reading] This path %s does not exist" %path
+                        raise ValueError, "[Reading] Directory %s does not exist" %path
                     try:
                         self.digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
                     except:
                         self.digitalReadObj = digital_rf_hdf5.read_hdf5(path)
                     channelNameList = self.digitalReadObj.get_channels()
                     if not channelNameList:
-                        raise IOError, "[Reading] The path doesn,t have any files .. "
+                        raise ValueError, "[Reading] Directory %s does not have any files" %path
                     if not channelList:
                         channelList = range(len(channelNameList))
                     ##########  Reading metadata ######################
                     metadata_dict = self.digitalReadObj.get_rf_file_metadata(channelNameList[channelList[0]])
                     self.__sample_rate = metadata_dict['sample_rate'][0]
                     self.__samples_per_file = metadata_dict['samples_per_file'][0]
                     self.__deltaHeigth = 1e6*0.15/self.__sample_rate
                     this_metadata_file = self.digitalReadObj.get_metadata(channelNameList[channelList[0]])
                     self.__frequency = this_metadata_file['center_frequencies'].value
                     try:
                         self.__timezone = this_metadata_file['timezone'].value
                     except:
                         self.__timezone = 0
                     self.__firstHeigth = 0
                     try:
                         codeType = this_metadata_file['codeType'].value
                     except:
                         codeType = 0
-                    nCode = 0
+                    nCode = 1
-                    nBaud = 0
+                    nBaud = 1
-                    code = None
+                    code = [1]
                     if codeType:
                         nCode = this_metadata_file['nCode'].value
                         nBaud = this_metadata_file['nBaud'].value
                         code = this_metadata_file['code'].value
                     if not ippKm:
                         try:
                             #seconds to km
                             ippKm = 1e6*0.15*this_metadata_file['ipp'].value
                         except:
                             ippKm = None
                     ####################################################
                     startUTCSecond = None
                     endUTCSecond = None
                     if startDate:
                         startDatetime = datetime.datetime.combine(startDate, startTime)
                         startUTCSecond = (startDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
                     if endDate:
                         endDatetime = datetime.datetime.combine(endDate, endTime)
                         endUTCSecond = (endDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
                     start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
                     if not startUTCSecond:
                         startUTCSecond = start_index/self.__sample_rate
                     if start_index > startUTCSecond*self.__sample_rate:
                         startUTCSecond = start_index/self.__sample_rate
                     if not endUTCSecond:
                         endUTCSecond = end_index/self.__sample_rate
                     if end_index < endUTCSecond*self.__sample_rate:
                         endUTCSecond = end_index/self.__sample_rate
                     if not nSamples:
                         if not ippKm:
                             raise ValueError, "[Reading] nSamples or ippKm should be defined"
                         nSamples = ippKm / (1e6*0.15/self.__sample_rate)
                     channelBoundList = []
                     channelNameListFiltered = []
                     for thisIndexChannel in channelList:
                         thisChannelName =  channelNameList[thisIndexChannel]
                         start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
                         channelBoundList.append((start_index, end_index))
                         channelNameListFiltered.append(thisChannelName)
                     self.profileIndex = 0
                     self.__delay = delay
                     self.__ippKm = ippKm
                     self.__codeType = codeType
                     self.__nCode = nCode
                     self.__nBaud = nBaud
                     self.__code = code
                     self.__datapath = path
                     self.__online = online
                     self.__channelList = channelList
                     self.__channelNameList = channelNameListFiltered
                     self.__channelBoundList = channelBoundList
                     self.__nSamples = nSamples
                     self.__samples_to_read = buffer_size*nSamples
                     self.__nChannels = len(self.__channelList)
                     self.__startUTCSecond = startUTCSecond
                     self.__endUTCSecond = endUTCSecond
                     self.__timeInterval = 1.0 * self.__samples_to_read/self.__sample_rate #Time interval
                     if online:
             #             self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
                         startUTCSecond = numpy.floor(endUTCSecond)
                     self.__thisUnixSample = int(startUTCSecond*self.__sample_rate) - self.__samples_to_read
                     self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype = numpy.complex)
                     self.__setFileHeader()
                     self.isConfig = True
                     print "[Reading] USRP Data was found from %s to %s " %(
                                                                   datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                                                                   datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                                                                   )
                     print "[Reading] Starting process from %s to %s" %(datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
                                                                        datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
                                                                        )
                 def __reload(self):
                     if not self.__online:
                         return
             #         print
             #         print "%s not in range [%s, %s]" %(
             #                                           datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
             #                                           datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
             #                                           datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
             #                                           )
                     print "[Reading] reloading metadata ..."
                     try:
                         self.digitalReadObj.reload(complete_update=True)
                     except:
                         self.digitalReadObj.reload()
                     start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
                     if start_index > self.__startUTCSecond*self.__sample_rate:
                         self.__startUTCSecond = 1.0*start_index/self.__sample_rate
                     if end_index > self.__endUTCSecond*self.__sample_rate:
                         self.__endUTCSecond = 1.0*end_index/self.__sample_rate
                         print
                         print "[Reading] New timerange found [%s, %s] " %(
                                                                   datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                                                                   datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                                                                   )
                         return True
                     return False
                 def __readNextBlock(self, seconds=30, volt_scale = 218776):
                     '''
                     '''
                     #Set the next data
                     self.__flagDiscontinuousBlock = False
                     self.__thisUnixSample += self.__samples_to_read
                     if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
                         print "[Reading] There are no more data into selected timerange"
                         self.__reload()
                         if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
                             self.__thisUnixSample -=  self.__samples_to_read
                             return False
                     indexChannel = 0
                     dataOk = False
                     for thisChannelName in self.__channelNameList:
                         try:
                             result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
                                                                           self.__samples_to_read,
                                                                           thisChannelName)
                         except IOError, e:
                             #read next profile
                             self.__flagDiscontinuousBlock = True
                             print "[Reading] %s" %datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e
                             break
                         if result.shape[0] != self.__samples_to_read:
                             self.__flagDiscontinuousBlock = True
                             print "[Reading] %s: Too few samples were found, just %d/%d  samples" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                                             result.shape[0],
                                                                                             self.__samples_to_read)
                             break
                         self.__data_buffer[indexChannel,:] = result*volt_scale
                         indexChannel += 1
                         dataOk = True
                     self.__utctime = self.__thisUnixSample/self.__sample_rate
                     if not dataOk:
                         return False
                     print "[Reading] %s: %d samples <> %f sec" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                      self.__samples_to_read,
                                                                      self.__timeInterval)
                     self.__bufferIndex = 0
                     return True
                 def __isBufferEmpty(self):
                     if self.__bufferIndex <= self.__samples_to_read - self.__nSamples:
                         return False
                     return True
                 def getData(self, seconds=30, nTries=5):
                     '''
                     This method gets the data from files and put the data into the dataOut object
                     In addition, increase el the buffer counter in one.
                     Return:
                         data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                      buffer. Si no hay mas archivos a leer retorna None.
                     Affected:
                         self.dataOut
                         self.profileIndex
                         self.flagDiscontinuousBlock
                         self.flagIsNewBlock
                     '''
                     err_counter = 0
                     self.dataOut.flagNoData = True
                     if self.__isBufferEmpty():
                         self.__flagDiscontinuousBlock = False
                         while True:
                             if self.__readNextBlock():
                                 break
                             if self.__thisUnixSample > self.__endUTCSecond*self.__sample_rate:
                                 return False
                             if self.__flagDiscontinuousBlock:
                                 print '[Reading] discontinuous block found ... continue with the next block'
                                 continue
                             if not self.__online:
                                 return False
                             err_counter += 1
                             if err_counter > nTries:
                                 return False
                             print '[Reading] waiting %d seconds to read a new block' %seconds
                             sleep(seconds)
                     self.dataOut.data = self.__data_buffer[:,self.__bufferIndex:self.__bufferIndex+self.__nSamples]
                     self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex)/self.__sample_rate
                     self.dataOut.flagNoData = False
                     self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
                     self.__bufferIndex += self.__nSamples
                     self.profileIndex += 1
                     return True
                 def printInfo(self):
                     '''
                     '''
                     if self.__printInfo == False:
                         return
             #         self.systemHeaderObj.printInfo()
             #         self.radarControllerHeaderObj.printInfo()
                     self.__printInfo = False
                 def printNumberOfBlock(self):
                     '''
                     '''
                     print self.profileIndex
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                     self.getData(seconds=self.__delay)
                     return
             class USRPWriter(Operation):
                 '''
                 classdocs
                 '''
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.dataOut = None
                 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
                     '''
                     In this method we should set all initial parameters.
                     Input:
                         dataIn        :        Input data will also be outputa data
                     '''
                     self.dataOut = dataIn
                     self.isConfig = True
                     return
                 def run(self, dataIn, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     Inputs:
                         dataIn        :        object with the data
                     '''
                     if not self.isConfig:
                         self.setup(dataIn, **kwargs)
             if __name__ == '__main__':
                 readObj = USRPReader()
                 while True:
                     readObj.run(path='/Volumes/DATA/haystack/passive_radar/')
             #         readObj.printInfo()
                     readObj.printNumberOfBlock()
   No newline at end of file

schainpy/model/io/jroIO_usrp_api.py +1 -1

             '''
             Created on Jul 15, 2014
             @author: roj-idl71
             '''
             import time
             import threading
             import cPickle
             # try:
             #     from gevent import sleep
             # except:
             from time import sleep
             SERIALIZER = cPickle
             # from schainpy.serializer import DynamicSerializer
             from schainpy.model.io.jroIO_usrp import USRPReader
             from schainpy.serializer.DataTranslate import obj2Serial
             class USRPReaderAPI(USRPReader, threading.Thread):
             #     __isBufferEmpty = True
                 __DATAKEYLIST = ['data','utctime','flagNoData']
                 def __init__(self, serializer='msgpack'):
                     threading.Thread.__init__(self)
                     USRPReader.__init__(self)
             #         self.__serializerObj = DynamicSerializer.DynamicSerializer('msgpack')
                     self.__mySerial = None
                     self.__isBufferEmpty = True
                     self.setSerializer(serializer)
                 def setSerializer(self, serializer):
                     self.__serializer = serializer
                 def getSerializer(self):
                     return self.__serializer
                 def getProfileIndex(self):
                     return self.profileIndex
                 def getSerialMetaData(self):
                     if self.__isBufferEmpty:
                         ini = time.time()
                         while True:
                             if not self.__isBufferEmpty:
                                 break
                             if time.time() - ini > 20:
                                 break
                             sleep(1e-12)
             #             if not self.getData():
             #                 self.__isBufferEmpty = False
             #                 return None
                     if self.dataOut.flagNoData:
                         return None
                     myMetadataSerial = obj2Serial(self.dataOut,
                                                   serializer = self.__serializer)
                     return myMetadataSerial
                 def getSerialData(self):
                     if self.__isBufferEmpty:
                         ini = time.time()
                         while True:
                             if not self.__isBufferEmpty:
                                 break
                             if time.time() - ini > 20:
                                 break
                             sleep(1e-12)
             #             if not self.getData():
             #                 self.__isBufferEmpty = False
             #                 return None
                     if self.dataOut.flagNoData:
                         return None
                     self.__isBufferEmpty = True
                     return self.__mySerial
                 def run(self):
                     '''
                     This method will be called once when start() is called
                     '''
                     if not self.isConfig:
-                        raise IOError, 'setup() method has to be called before start()'
+                        raise RuntimeError, 'setup() method has to be called before start()'
                     while True:
                         if not self.__isBufferEmpty:
                             sleep(1e-12)
                             continue
                         if not self.getData():
                             break
                         print ".",
                         self.__mySerial = obj2Serial(self.dataOut,
                                                      keyList = self.__DATAKEYLIST,
                                                      serializer = self.__serializer)
                         self.__isBufferEmpty = False
             #             print self.profileIndex
             #             print 'wait 1 second'
             #             sleep(0.1)
                     return

schainpy/model/proc/jroproc_base.py +8 -8

             '''
             $Author: murco $
             $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
             '''
             class ProcessingUnit(object):
                 """
                 Esta es la clase base para el procesamiento de datos.
                 Contiene el metodo "call" para llamar operaciones. Las operaciones pueden ser:
                     - Metodos internos (callMethod)
                     - Objetos del tipo Operation (callObject). Antes de ser llamados, estos objetos
                       tienen que ser agreagados con el metodo "add".
                 """
                 # objeto de datos de entrada (Voltage, Spectra o Correlation)
                 dataIn = None
                 dataInList = []
                 # objeto de datos de entrada (Voltage, Spectra o Correlation)
                 dataOut = None
                 operations2RunDict = None
                 isConfig = False
                 def __init__(self):
                     self.dataIn = None
                     self.dataInList = []
                     self.dataOut = None
                     self.operations2RunDict = {}
                     self.isConfig = False
                 def addOperation(self, opObj, objId):
                     """
                     Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
                     identificador asociado a este objeto.
                     Input:
                         object    :    objeto de la clase "Operation"
                     Return:
                         objId    :    identificador del objeto, necesario para ejecutar la operacion
                     """
                     self.operations2RunDict[objId] = opObj
                     return objId
                 def getOperationObj(self, objId):
                     if objId not in self.operations2RunDict.keys():
                         return None
                     return self.operations2RunDict[objId]
                 def operation(self, **kwargs):
                     """
                     Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
                     atributos del objeto dataOut
                     Input:
                         **kwargs    :    Diccionario de argumentos de la funcion a ejecutar
                     """
-                    raise ValueError, "ImplementedError"
+                    raise NotImplementedError
                 def callMethod(self, name, **kwargs):
                     """
                     Ejecuta el metodo con el nombre "name" y con argumentos **kwargs de la propia clase.
                     Input:
                         name        :    nombre del metodo a ejecutar
                         **kwargs     :    diccionario con los nombres y valores de la funcion a ejecutar.
                     """
                     #Checking the inputs
                     if name == 'run':
                         if not self.checkInputs():
                             self.dataOut.flagNoData = True
                             return False
                     else:
                         #Si no es un metodo RUN la entrada es la misma dataOut (interna)
                         if self.dataOut.isEmpty():
                             return False
                     #Getting the pointer to method
                     methodToCall = getattr(self, name)
                     #Executing the self method
                     methodToCall(**kwargs)
                     #Checkin the outputs
             #         if name == 'run':
             #             pass
             #         else:
             #             pass
             #
             #         if name != 'run':
             #             return True
                     if self.dataOut is None:
                         return False
                     if self.dataOut.isEmpty():
                         return False
                     return True
                 def callObject(self, objId, **kwargs):
                     """
                     Ejecuta la operacion asociada al identificador del objeto "objId"
                     Input:
                         objId        :    identificador del objeto a ejecutar
                         **kwargs    :    diccionario con los nombres y valores de la funcion a ejecutar.
                     Return:
                         None
                     """
                     if self.dataOut.isEmpty():
                         return False
                     externalProcObj = self.operations2RunDict[objId]
                     externalProcObj.run(self.dataOut, **kwargs)
                     return True
                 def call(self, opType, opName=None, opId=None, **kwargs):
                     """
                     Return True si ejecuta la operacion interna nombrada "opName" o la operacion externa
                     identificada con el id "opId"; con los argumentos "**kwargs".
                     False si la operacion no se ha ejecutado.
                     Input:
                         opType    :    Puede ser "self" o "external"
                             La operacion puede ser de dos tipos (callMethod or callObject):
 . Un metodo propio de esta clase:
                                 opType = "self"
 . El metodo "run" de un objeto del tipo Operation o de un derivado de ella:
                                 opType = "other" or "external".
                         opName    : Si la operacion es interna (opType = 'self'), entonces el "opName" sera
                                     usada para llamar a un metodo interno de la clase Processing
                         opId    :    Si la operacion es externa (opType = 'other'), entonces el "opId" sera
                                     usada para llamar al metodo "run" de la clase Operation registrada con ese Id
                     Exception:
                            Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
                            "addOperation" e identificado con el valor "opId"  = el id de la operacion.
                            De lo contrario retornara un error del tipo IOError
                     """
                     if opType == 'self':
                         if not opName:
-                            raise IOError, "opName parameter should be defined"
+                            raise ValueError, "opName parameter should be defined"
                         sts = self.callMethod(opName, **kwargs)
                     if opType == 'other' or opType == 'external':
                         if not opId:
-                            raise IOError, "opId parameter should be defined"
+                            raise ValueError, "opId parameter should be defined"
                         if opId not in self.operations2RunDict.keys():
-                            raise IOError, "This id operation have not been registered"
+                            raise ValueError, "Id operation has not been registered"
                         sts = self.callObject(opId, **kwargs)
                     return sts
                 def setInput(self, dataIn):
                     self.dataIn = dataIn
                     self.dataInList.append(dataIn)
                 def getOutputObj(self):
                     return self.dataOut
                 def checkInputs(self):
                     for thisDataIn in self.dataInList:
                         if thisDataIn.isEmpty():
                             return False
                     return True
                 def setup(self):
-                    raise ValueError, "Not implemented"
+                    raise NotImplementedError
                 def run(self):
-                    raise ValueError, "Not implemented"
+                    raise NotImplementedError
                 def close(self):
                     #Close every thread, queue or any other object here is it is neccesary.
                     return
             class Operation(object):
                 """
                 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
                 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
                 acumulacion dentro de esta clase
                 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
                 """
                 __buffer = None
                 isConfig = False
                 def __init__(self):
                     self.__buffer = None
                     self.isConfig = False
                 def setup(self):
                     self.isConfig = True
-                    raise ValueError, "Not implemented"
+                    raise NotImplementedError
                 def run(self, dataIn, **kwargs):
                     """
                     Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
                     atributos del objeto dataIn.
                     Input:
                         dataIn    :    objeto del tipo JROData
                     Return:
                         None
                     Affected:
                         __buffer    :    buffer de recepcion de datos.
                     """
                     if not self.isConfig:
                         self.setup(**kwargs)
-                    raise ValueError, "ImplementedError"
+                    raise NotImplementedError
                 def close(self):
                     pass

schainpy/model/proc/jroproc_spectra.py 0 -5

             import numpy
             import math
             from jroproc_base import ProcessingUnit, Operation
             from schainpy.model.data.jrodata import Spectra
             from schainpy.model.data.jrodata import hildebrand_sekhon
             class SpectraProc(ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.buffer = None
                     self.firstdatatime = None
                     self.profIndex = 0
                     self.dataOut = Spectra()
                     self.id_min = None
                     self.id_max = None
                 def __updateSpecFromVoltage(self):
                     self.dataOut.timeZone = self.dataIn.timeZone
                     self.dataOut.dstFlag = self.dataIn.dstFlag
                     self.dataOut.errorCount = self.dataIn.errorCount
                     self.dataOut.useLocalTime = self.dataIn.useLocalTime
                     self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
                     self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
                     self.dataOut.channelList = self.dataIn.channelList
                     self.dataOut.heightList = self.dataIn.heightList
                     self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
                     self.dataOut.nBaud = self.dataIn.nBaud
                     self.dataOut.nCode = self.dataIn.nCode
                     self.dataOut.code = self.dataIn.code
                     self.dataOut.nProfiles = self.dataOut.nFFTPoints
                     self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
                     self.dataOut.utctime = self.firstdatatime
                     self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
                     self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.nCohInt = self.dataIn.nCohInt
                     self.dataOut.nIncohInt = 1
                     self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
                     self.dataOut.frequency = self.dataIn.frequency
                     self.dataOut.realtime = self.dataIn.realtime
                     self.dataOut.azimuth = self.dataIn.azimuth
                     self.dataOut.zenith = self.dataIn.zenith
                     self.dataOut.beam.codeList = self.dataIn.beam.codeList
                     self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
                     self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
                 def __getFft(self):
                     """
                     Convierte valores de Voltaje a Spectra
                     Affected:
                         self.dataOut.data_spc
                         self.dataOut.data_cspc
                         self.dataOut.data_dc
                         self.dataOut.heightList
                         self.profIndex
                         self.buffer
                         self.dataOut.flagNoData
                     """
                     fft_volt = numpy.fft.fft(self.buffer,n=self.dataOut.nFFTPoints,axis=1)
                     fft_volt = fft_volt.astype(numpy.dtype('complex'))
                     dc = fft_volt[:,0,:]
                     #calculo de self-spectra
                     fft_volt = numpy.fft.fftshift(fft_volt,axes=(1,))
                     spc = fft_volt * numpy.conjugate(fft_volt)
                     spc = spc.real
                     blocksize = 0
                     blocksize += dc.size
                     blocksize += spc.size
                     cspc = None
                     pairIndex = 0
                     if self.dataOut.pairsList != None:
                         #calculo de cross-spectra
                         cspc = numpy.zeros((self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
                         for pair in self.dataOut.pairsList:
                             if pair[0] not in self.dataOut.channelList:
                                 raise ValueError, "Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" %(str(pair), str(self.dataOut.channelList))
                             if pair[1] not in self.dataOut.channelList:
                                 raise ValueError, "Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" %(str(pair), str(self.dataOut.channelList))
                             cspc[pairIndex,:,:] = fft_volt[pair[0],:,:] * numpy.conjugate(fft_volt[pair[1],:,:])
                             pairIndex += 1
                         blocksize += cspc.size
                     self.dataOut.data_spc = spc
                     self.dataOut.data_cspc = cspc
                     self.dataOut.data_dc = dc
                     self.dataOut.blockSize = blocksize
                     self.dataOut.flagShiftFFT = True
                 def run(self, nProfiles=None, nFFTPoints=None, pairsList=[], ippFactor=None):
                     self.dataOut.flagNoData = True
                     if self.dataIn.type == "Spectra":
                         self.dataOut.copy(self.dataIn)
                         return True
                     if self.dataIn.type == "Voltage":
                         if nFFTPoints == None:
                             raise ValueError, "This SpectraProc.run() need nFFTPoints input variable"
                         if nProfiles == None:
                             nProfiles = nFFTPoints
-            #                 raise ValueError, "This SpectraProc.run() need nProfiles input variable"
                         if ippFactor == None:
                             ippFactor = 1
                         self.dataOut.ippFactor = ippFactor
                         self.dataOut.nFFTPoints = nFFTPoints
                         self.dataOut.pairsList = pairsList
                         if self.buffer is None:
                             self.buffer = numpy.zeros( (self.dataIn.nChannels,
                                                         nProfiles,
                                                         self.dataIn.nHeights),
                                                       dtype='complex')
                         if self.dataIn.flagDataAsBlock:
                             if self.dataIn.nProfiles == nProfiles:
                                 self.buffer = self.dataIn.data.copy()
                                 self.profIndex = nProfiles
                             elif self.dataIn.nProfiles < nProfiles:
                                 if self.profIndex == 0:
                                     self.id_min = 0
                                     self.id_max = self.dataIn.nProfiles
                                 self.buffer[:,self.id_min:self.id_max,:] = self.dataIn.data
                                 self.profIndex += self.dataIn.nProfiles
                                 self.id_min += self.dataIn.data.shape[1]
                                 self.id_max += self.dataIn.data.shape[1]
                             else:
                                 raise ValueError, "The type object %s has %d profiles, it should be equal to %d profiles"%(self.dataIn.type,self.dataIn.data.shape[1],nProfiles)
                                 self.dataOut.flagNoData = True
                                 return 0
                         else:
                             self.buffer[:,self.profIndex,:] = self.dataIn.data.copy()
                             self.profIndex += 1
                         if self.firstdatatime == None:
                             self.firstdatatime = self.dataIn.utctime
                         if self.profIndex == nProfiles:
                             self.__updateSpecFromVoltage()
                             self.__getFft()
                             self.dataOut.flagNoData = False
                             self.firstdatatime = None
                             self.profIndex = 0
                         return True
                     raise ValueError, "The type of input object '%s' is not valid"%(self.dataIn.type)
                 def __selectPairs(self, channelList=None):
                     if channelList == None:
                         return
                     pairsIndexListSelected = []
                     for pairIndex in self.dataOut.pairsIndexList:
                         #First pair
                         if self.dataOut.pairsList[pairIndex][0] not in channelList:
                             continue
                         #Second pair
                         if self.dataOut.pairsList[pairIndex][1] not in channelList:
                             continue
                         pairsIndexListSelected.append(pairIndex)
                     if not pairsIndexListSelected:
                         self.dataOut.data_cspc = None
                         self.dataOut.pairsList = []
                         return
                     self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndexListSelected]
                     self.dataOut.pairsList = [self.dataOut.pairsList[i] for i in pairsIndexListSelected]
                     return
                 def selectChannels(self, channelList):
                     channelIndexList = []
                     for channel in channelList:
                         if channel not in self.dataOut.channelList:
                             raise ValueError, "Error selecting channels: The value %d in channelList is not valid.\nAvailable channels = %s" %(channel, str(self.dataOut.channelList))
                         index = self.dataOut.channelList.index(channel)
                         channelIndexList.append(index)
                     self.selectChannelsByIndex(channelIndexList)
                 def selectChannelsByIndex(self, channelIndexList):
                     """
                     Selecciona un bloque de datos en base a canales segun el channelIndexList
                     Input:
                         channelIndexList    :    lista sencilla de canales a seleccionar por ej. [2,3,7]
                     Affected:
                         self.dataOut.data_spc
                         self.dataOut.channelIndexList
                         self.dataOut.nChannels
                     Return:
                         None
                     """
                     for channelIndex in channelIndexList:
                         if channelIndex not in self.dataOut.channelIndexList:
                             raise ValueError, "Error selecting channels: The value %d in channelIndexList is not valid.\nAvailable channel indexes = " %(channelIndex, self.dataOut.channelIndexList)
             #         nChannels = len(channelIndexList)
                     data_spc = self.dataOut.data_spc[channelIndexList,:]
                     data_dc = self.dataOut.data_dc[channelIndexList,:]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_dc = data_dc
                     self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
             #        self.dataOut.nChannels = nChannels
                     self.__selectPairs(self.dataOut.channelList)
                     return 1
                 def selectHeights(self, minHei, maxHei):
                     """
                     Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
                     minHei <= height <= maxHei
                     Input:
                         minHei    :    valor minimo de altura a considerar
                         maxHei    :    valor maximo de altura a considerar
                     Affected:
                         Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if (minHei > maxHei):
                         raise ValueError, "Error selecting heights: Height range (%d,%d) is not valid" % (minHei, maxHei)
                     if (minHei < self.dataOut.heightList[0]):
                         minHei = self.dataOut.heightList[0]
                     if (maxHei > self.dataOut.heightList[-1]):
                         maxHei = self.dataOut.heightList[-1]
-            #            raise ValueError, "some value in (%d,%d) is not valid" % (minHei, maxHei)
                     minIndex = 0
                     maxIndex = 0
                     heights = self.dataOut.heightList
                     inda = numpy.where(heights >= minHei)
                     indb = numpy.where(heights <= maxHei)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(heights)
                     self.selectHeightsByIndex(minIndex, maxIndex)
                     return 1
                 def getBeaconSignal(self, tauindex = 0, channelindex = 0, hei_ref=None):
                     newheis = numpy.where(self.dataOut.heightList>self.dataOut.radarControllerHeaderObj.Taus[tauindex])
                     if hei_ref != None:
                         newheis = numpy.where(self.dataOut.heightList>hei_ref)
                     minIndex = min(newheis[0])
                     maxIndex = max(newheis[0])
                     data_spc = self.dataOut.data_spc[:,:,minIndex:maxIndex+1]
                     heightList = self.dataOut.heightList[minIndex:maxIndex+1]
                     # determina indices
                     nheis = int(self.dataOut.radarControllerHeaderObj.txB/(self.dataOut.heightList[1]-self.dataOut.heightList[0]))
                     avg_dB = 10*numpy.log10(numpy.sum(data_spc[channelindex,:,:],axis=0))
                     beacon_dB = numpy.sort(avg_dB)[-nheis:]
                     beacon_heiIndexList = []
                     for val in avg_dB.tolist():
                         if val >= beacon_dB[0]:
                             beacon_heiIndexList.append(avg_dB.tolist().index(val))
                     #data_spc = data_spc[:,:,beacon_heiIndexList]
                     data_cspc = None
                     if self.dataOut.data_cspc is not None:
                         data_cspc = self.dataOut.data_cspc[:,:,minIndex:maxIndex+1]
                         #data_cspc = data_cspc[:,:,beacon_heiIndexList]
                     data_dc = None
                     if self.dataOut.data_dc is not None:
                         data_dc = self.dataOut.data_dc[:,minIndex:maxIndex+1]
                         #data_dc = data_dc[:,beacon_heiIndexList]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_cspc = data_cspc
                     self.dataOut.data_dc = data_dc
                     self.dataOut.heightList = heightList
                     self.dataOut.beacon_heiIndexList = beacon_heiIndexList
                     return 1
                 def selectHeightsByIndex(self, minIndex, maxIndex):
                     """
                     Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
                     minIndex <= index <= maxIndex
                     Input:
                         minIndex    :    valor de indice minimo de altura a considerar
                         maxIndex    :    valor de indice maximo de altura a considerar
                     Affected:
                         self.dataOut.data_spc
                         self.dataOut.data_cspc
                         self.dataOut.data_dc
                         self.dataOut.heightList
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError, "Error selecting heights by index: Index range in (%d,%d) is not valid" % (minIndex, maxIndex)
                     if (maxIndex >= self.dataOut.nHeights):
                         maxIndex = self.dataOut.nHeights-1
-            #            raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
-            #         nHeights = maxIndex - minIndex + 1
                     #Spectra
                     data_spc = self.dataOut.data_spc[:,:,minIndex:maxIndex+1]
                     data_cspc = None
                     if self.dataOut.data_cspc is not None:
                         data_cspc = self.dataOut.data_cspc[:,:,minIndex:maxIndex+1]
                     data_dc = None
                     if self.dataOut.data_dc is not None:
                         data_dc = self.dataOut.data_dc[:,minIndex:maxIndex+1]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_cspc = data_cspc
                     self.dataOut.data_dc = data_dc
                     self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex+1]
                     return 1
                 def removeDC(self, mode = 2):
                     jspectra = self.dataOut.data_spc
                     jcspectra = self.dataOut.data_cspc
                     num_chan = jspectra.shape[0]
                     num_hei = jspectra.shape[2]
                     if jcspectra is not None:
                         jcspectraExist = True
                         num_pairs = jcspectra.shape[0]
                     else:   jcspectraExist = False
                     freq_dc = jspectra.shape[1]/2
                     ind_vel = numpy.array([-2,-1,1,2]) + freq_dc
                     if ind_vel[0]<0:
                         ind_vel[range(0,1)] = ind_vel[range(0,1)] + self.num_prof
                     if mode == 1:
                         jspectra[:,freq_dc,:] = (jspectra[:,ind_vel[1],:] + jspectra[:,ind_vel[2],:])/2 #CORRECCION
                         if jcspectraExist:
                             jcspectra[:,freq_dc,:] = (jcspectra[:,ind_vel[1],:] + jcspectra[:,ind_vel[2],:])/2
                     if mode == 2:
                         vel = numpy.array([-2,-1,1,2])
                         xx = numpy.zeros([4,4])
                         for fil in range(4):
                             xx[fil,:] = vel[fil]**numpy.asarray(range(4))
                         xx_inv = numpy.linalg.inv(xx)
                         xx_aux = xx_inv[0,:]
                         for ich in range(num_chan):
                             yy = jspectra[ich,ind_vel,:]
                             jspectra[ich,freq_dc,:] = numpy.dot(xx_aux,yy)
                             junkid = jspectra[ich,freq_dc,:]<=0
                             cjunkid = sum(junkid)
                             if cjunkid.any():
                                 jspectra[ich,freq_dc,junkid.nonzero()] = (jspectra[ich,ind_vel[1],junkid] + jspectra[ich,ind_vel[2],junkid])/2
                         if jcspectraExist:
                             for ip in range(num_pairs):
                                 yy = jcspectra[ip,ind_vel,:]
                                 jcspectra[ip,freq_dc,:] = numpy.dot(xx_aux,yy)
                     self.dataOut.data_spc = jspectra
                     self.dataOut.data_cspc = jcspectra
                     return 1
                 def removeInterference(self,  interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None):
                     jspectra = self.dataOut.data_spc
                     jcspectra = self.dataOut.data_cspc
                     jnoise = self.dataOut.getNoise()
                     num_incoh = self.dataOut.nIncohInt
                     num_channel  = jspectra.shape[0]
                     num_prof  = jspectra.shape[1]
                     num_hei   = jspectra.shape[2]
                     #hei_interf
                     if hei_interf is None:
                         count_hei = num_hei/2   #Como es entero no importa
                         hei_interf = numpy.asmatrix(range(count_hei)) + num_hei - count_hei
                         hei_interf = numpy.asarray(hei_interf)[0]
                     #nhei_interf
                     if (nhei_interf == None):
                         nhei_interf = 5
                     if (nhei_interf < 1):
                         nhei_interf = 1
                     if (nhei_interf > count_hei):
                         nhei_interf = count_hei
                     if (offhei_interf == None):
                         offhei_interf = 0
                     ind_hei = range(num_hei)
             #         mask_prof = numpy.asarray(range(num_prof - 2)) + 1
             #         mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
                     mask_prof = numpy.asarray(range(num_prof))
                     num_mask_prof = mask_prof.size
                     comp_mask_prof = [0, num_prof/2]
                     #noise_exist:    Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
                     if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
                         jnoise = numpy.nan
                     noise_exist = jnoise[0] < numpy.Inf
                     #Subrutina de Remocion de la Interferencia
                     for ich in range(num_channel):
                         #Se ordena los espectros segun su potencia (menor a mayor)
                         power = jspectra[ich,mask_prof,:]
                         power = power[:,hei_interf]
                         power = power.sum(axis = 0)
                         psort = power.ravel().argsort()
                         #Se estima la interferencia promedio en los Espectros de Potencia empleando
                         junkspc_interf = jspectra[ich,:,hei_interf[psort[range(offhei_interf, nhei_interf + offhei_interf)]]]
                         if noise_exist:
                         #    tmp_noise = jnoise[ich] / num_prof
                             tmp_noise = jnoise[ich]
                         junkspc_interf = junkspc_interf - tmp_noise
                         #junkspc_interf[:,comp_mask_prof] = 0
                         jspc_interf = junkspc_interf.sum(axis = 0) / nhei_interf
                         jspc_interf = jspc_interf.transpose()
                         #Calculando el espectro de interferencia promedio
                         noiseid =  numpy.where(jspc_interf <= tmp_noise/ math.sqrt(num_incoh))
                         noiseid = noiseid[0]
                         cnoiseid = noiseid.size
                         interfid = numpy.where(jspc_interf > tmp_noise/ math.sqrt(num_incoh))
                         interfid = interfid[0]
                         cinterfid = interfid.size
                         if (cnoiseid > 0):   jspc_interf[noiseid] = 0
                         #Expandiendo los perfiles a limpiar
                         if (cinterfid > 0):
                             new_interfid = (numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof)%num_prof
                             new_interfid = numpy.asarray(new_interfid)
                             new_interfid = {x for x in new_interfid}
                             new_interfid = numpy.array(list(new_interfid))
                             new_cinterfid = new_interfid.size
                         else: new_cinterfid = 0
                         for ip in range(new_cinterfid):
                             ind = junkspc_interf[:,new_interfid[ip]].ravel().argsort()
                             jspc_interf[new_interfid[ip]] = junkspc_interf[ind[nhei_interf/2],new_interfid[ip]]
                         jspectra[ich,:,ind_hei] = jspectra[ich,:,ind_hei] - jspc_interf #Corregir indices
                         #Removiendo la interferencia del punto de mayor interferencia
                         ListAux = jspc_interf[mask_prof].tolist()
                         maxid = ListAux.index(max(ListAux))
                         if cinterfid > 0:
                             for ip in range(cinterfid*(interf == 2) - 1):
                                 ind = (jspectra[ich,interfid[ip],:] < tmp_noise*(1 + 1/math.sqrt(num_incoh))).nonzero()
                                 cind = len(ind)
                                 if (cind > 0):
                                     jspectra[ich,interfid[ip],ind] = tmp_noise*(1 + (numpy.random.uniform(cind) - 0.5)/math.sqrt(num_incoh))
                             ind = numpy.array([-2,-1,1,2])
                             xx = numpy.zeros([4,4])
                             for id1 in range(4):
                                 xx[:,id1] = ind[id1]**numpy.asarray(range(4))
                             xx_inv = numpy.linalg.inv(xx)
                             xx = xx_inv[:,0]
                             ind = (ind + maxid + num_mask_prof)%num_mask_prof
                             yy = jspectra[ich,mask_prof[ind],:]
                             jspectra[ich,mask_prof[maxid],:] = numpy.dot(yy.transpose(),xx)
                         indAux = (jspectra[ich,:,:] < tmp_noise*(1-1/math.sqrt(num_incoh))).nonzero()
                         jspectra[ich,indAux[0],indAux[1]] = tmp_noise * (1 - 1/math.sqrt(num_incoh))
                     #Remocion de Interferencia en el Cross Spectra
                     if jcspectra is None: return jspectra, jcspectra
                     num_pairs = jcspectra.size/(num_prof*num_hei)
                     jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
                     for ip in range(num_pairs):
                         #-------------------------------------------
                         cspower = numpy.abs(jcspectra[ip,mask_prof,:])
                         cspower = cspower[:,hei_interf]
                         cspower = cspower.sum(axis = 0)
                         cspsort = cspower.ravel().argsort()
                         junkcspc_interf = jcspectra[ip,:,hei_interf[cspsort[range(offhei_interf, nhei_interf + offhei_interf)]]]
                         junkcspc_interf = junkcspc_interf.transpose()
                         jcspc_interf = junkcspc_interf.sum(axis = 1)/nhei_interf
                         ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
                         median_real = numpy.median(numpy.real(junkcspc_interf[mask_prof[ind[range(3*num_prof/4)]],:]))
                         median_imag = numpy.median(numpy.imag(junkcspc_interf[mask_prof[ind[range(3*num_prof/4)]],:]))
                         junkcspc_interf[comp_mask_prof,:] = numpy.complex(median_real, median_imag)
                         for iprof in range(num_prof):
                             ind = numpy.abs(junkcspc_interf[iprof,:]).ravel().argsort()
                             jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf/2]]
                         #Removiendo la Interferencia
                         jcspectra[ip,:,ind_hei] = jcspectra[ip,:,ind_hei] - jcspc_interf
                         ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
                         maxid = ListAux.index(max(ListAux))
                         ind = numpy.array([-2,-1,1,2])
                         xx = numpy.zeros([4,4])
                         for id1 in range(4):
                             xx[:,id1] = ind[id1]**numpy.asarray(range(4))
                         xx_inv = numpy.linalg.inv(xx)
                         xx = xx_inv[:,0]
                         ind = (ind + maxid + num_mask_prof)%num_mask_prof
                         yy = jcspectra[ip,mask_prof[ind],:]
                         jcspectra[ip,mask_prof[maxid],:] = numpy.dot(yy.transpose(),xx)
                     #Guardar Resultados
                     self.dataOut.data_spc = jspectra
                     self.dataOut.data_cspc = jcspectra
                     return 1
                 def setRadarFrequency(self, frequency=None):
                     if frequency != None:
                         self.dataOut.frequency = frequency
                     return 1
                 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
                     #validacion de rango
                     if minHei == None:
                         minHei = self.dataOut.heightList[0]
                     if maxHei == None:
                         maxHei = self.dataOut.heightList[-1]
                     if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
                         print 'minHei: %.2f is out of the heights range'%(minHei)
                         print 'minHei is setting to %.2f'%(self.dataOut.heightList[0])
                         minHei = self.dataOut.heightList[0]
                     if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
                         print 'maxHei: %.2f is out of the heights range'%(maxHei)
                         print 'maxHei is setting to %.2f'%(self.dataOut.heightList[-1])
                         maxHei = self.dataOut.heightList[-1]
                     # validacion de velocidades
                     velrange = self.dataOut.getVelRange(1)
                     if minVel == None:
                         minVel = velrange[0]
                     if maxVel == None:
                         maxVel = velrange[-1]
                     if (minVel < velrange[0]) or (minVel > maxVel):
                         print 'minVel: %.2f is out of the velocity range'%(minVel)
                         print 'minVel is setting to %.2f'%(velrange[0])
                         minVel = velrange[0]
                     if (maxVel > velrange[-1]) or (maxVel < minVel):
                         print 'maxVel: %.2f is out of the velocity range'%(maxVel)
                         print 'maxVel is setting to %.2f'%(velrange[-1])
                         maxVel = velrange[-1]
                     # seleccion de indices para rango
                     minIndex = 0
                     maxIndex = 0
                     heights = self.dataOut.heightList
                     inda = numpy.where(heights >= minHei)
                     indb = numpy.where(heights <= maxHei)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(heights)
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
                     if (maxIndex >= self.dataOut.nHeights):
                         maxIndex = self.dataOut.nHeights-1
                     # seleccion de indices para velocidades
                     indminvel = numpy.where(velrange >= minVel)
                     indmaxvel = numpy.where(velrange <= maxVel)
                     try:
                         minIndexVel = indminvel[0][0]
                     except:
                         minIndexVel = 0
                     try:
                         maxIndexVel = indmaxvel[0][-1]
                     except:
                         maxIndexVel = len(velrange)
                     #seleccion del espectro
                     data_spc = self.dataOut.data_spc[:,minIndexVel:maxIndexVel+1,minIndex:maxIndex+1]
                     #estimacion de ruido
                     noise = numpy.zeros(self.dataOut.nChannels)
                     for channel in range(self.dataOut.nChannels):
                         daux = data_spc[channel,:,:]
                         noise[channel] = hildebrand_sekhon(daux, self.dataOut.nIncohInt)
                     self.dataOut.noise_estimation = noise.copy()
                     return 1
             class IncohInt(Operation):
                 __profIndex = 0
                 __withOverapping  = False
                 __byTime = False
                 __initime = None
                 __lastdatatime = None
                 __integrationtime = None
                 __buffer_spc = None
                 __buffer_cspc = None
                 __buffer_dc = None
                 __dataReady = False
                 __timeInterval = None
                 n = None
                 def __init__(self):
                     Operation.__init__(self)
             #         self.isConfig = False
                 def setup(self, n=None, timeInterval=None, overlapping=False):
                     """
                     Set the parameters of the integration class.
                     Inputs:
                         n        :    Number of coherent integrations
                         timeInterval   :    Time of integration. If the parameter "n" is selected this one does not work
                         overlapping    :
                     """
                     self.__initime = None
                     self.__lastdatatime = 0
                     self.__buffer_spc = 0
                     self.__buffer_cspc = 0
                     self.__buffer_dc = 0
                     self.__profIndex = 0
                     self.__dataReady = False
                     self.__byTime = False
                     if n is None and timeInterval is None:
                         raise ValueError, "n or timeInterval should be specified ..."
                     if n is not None:
                         self.n = int(n)
                     else:
                         self.__integrationtime = int(timeInterval) #if (type(timeInterval)!=integer) -> change this line
                         self.n = None
                         self.__byTime = True
                 def putData(self, data_spc, data_cspc, data_dc):
                     """
                     Add a profile to the __buffer_spc and increase in one the __profileIndex
                     """
                     self.__buffer_spc += data_spc
                     if data_cspc is None:
                         self.__buffer_cspc = None
                     else:
                         self.__buffer_cspc += data_cspc
                     if data_dc is None:
                         self.__buffer_dc = None
                     else:
                         self.__buffer_dc += data_dc
                     self.__profIndex += 1
                     return
                 def pushData(self):
                     """
                     Return the sum of the last profiles and the profiles used in the sum.
                     Affected:
                     self.__profileIndex
                     """
                     data_spc = self.__buffer_spc
                     data_cspc = self.__buffer_cspc
                     data_dc = self.__buffer_dc
                     n = self.__profIndex
                     self.__buffer_spc = 0
                     self.__buffer_cspc = 0
                     self.__buffer_dc = 0
                     self.__profIndex = 0
                     return data_spc, data_cspc, data_dc, n
                 def byProfiles(self, *args):
                     self.__dataReady = False
                     avgdata_spc = None
                     avgdata_cspc = None
                     avgdata_dc = None
                     self.putData(*args)
                     if self.__profIndex == self.n:
                         avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata_spc, avgdata_cspc, avgdata_dc
                 def byTime(self, datatime, *args):
                     self.__dataReady = False
                     avgdata_spc = None
                     avgdata_cspc = None
                     avgdata_dc = None
                     self.putData(*args)
                     if (datatime - self.__initime) >= self.__integrationtime:
                         avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata_spc, avgdata_cspc, avgdata_dc
                 def integrate(self, datatime, *args):
                     if self.__profIndex == 0:
                         self.__initime = datatime
                     if self.__byTime:
                         avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(datatime, *args)
                     else:
                         avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
                     if not self.__dataReady:
                         return None, None, None, None
                     return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
                 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
                     if n==1:
                         return
                     dataOut.flagNoData = True
                     if not self.isConfig:
                         self.setup(n, timeInterval, overlapping)
                         self.isConfig = True
                     avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
                                                                                         dataOut.data_spc,
                                                                                         dataOut.data_cspc,
                                                                                         dataOut.data_dc)
                     if self.__dataReady:
                         dataOut.data_spc = avgdata_spc
                         dataOut.data_cspc = avgdata_cspc
                         dataOut.data_dc = avgdata_dc
                         dataOut.nIncohInt *= self.n
                         dataOut.utctime = avgdatatime
                         dataOut.flagNoData = False

schainpy/model/proc/jroproc_voltage.py +6 -26

             import numpy
             from jroproc_base import ProcessingUnit, Operation
             from schainpy.model.data.jrodata import Voltage
             class VoltageProc(ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
             #         self.objectDict = {}
                     self.dataOut = Voltage()
                     self.flip = 1
                 def run(self):
                     if self.dataIn.type == 'AMISR':
                         self.__updateObjFromAmisrInput()
                     if self.dataIn.type == 'Voltage':
                         self.dataOut.copy(self.dataIn)
             #         self.dataOut.copy(self.dataIn)
                 def __updateObjFromAmisrInput(self):
                     self.dataOut.timeZone = self.dataIn.timeZone
                     self.dataOut.dstFlag = self.dataIn.dstFlag
                     self.dataOut.errorCount = self.dataIn.errorCount
                     self.dataOut.useLocalTime = self.dataIn.useLocalTime
                     self.dataOut.flagNoData = self.dataIn.flagNoData
                     self.dataOut.data = self.dataIn.data
                     self.dataOut.utctime = self.dataIn.utctime
                     self.dataOut.channelList = self.dataIn.channelList
             #         self.dataOut.timeInterval = self.dataIn.timeInterval
                     self.dataOut.heightList = self.dataIn.heightList
                     self.dataOut.nProfiles = self.dataIn.nProfiles
                     self.dataOut.nCohInt = self.dataIn.nCohInt
                     self.dataOut.ippSeconds = self.dataIn.ippSeconds
                     self.dataOut.frequency = self.dataIn.frequency
                     self.dataOut.azimuth = self.dataIn.azimuth
                     self.dataOut.zenith = self.dataIn.zenith
                     self.dataOut.beam.codeList = self.dataIn.beam.codeList
                     self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
                     self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
             #
             #        pass#
             #
             #    def init(self):
             #
             #
             #        if self.dataIn.type == 'AMISR':
             #            self.__updateObjFromAmisrInput()
             #
             #        if self.dataIn.type == 'Voltage':
             #            self.dataOut.copy(self.dataIn)
             #        # No necesita copiar en cada init() los atributos de dataIn
             #        # la copia deberia hacerse por cada nuevo bloque de datos
                 def selectChannels(self, channelList):
                     channelIndexList = []
                     for channel in channelList:
                         if channel not in self.dataOut.channelList:
                             raise ValueError, "Channel %d is not in %s" %(channel, str(self.dataOut.channelList))
                         index = self.dataOut.channelList.index(channel)
                         channelIndexList.append(index)
                     self.selectChannelsByIndex(channelIndexList)
                 def selectChannelsByIndex(self, channelIndexList):
                     """
                     Selecciona un bloque de datos en base a canales segun el channelIndexList
                     Input:
                         channelIndexList    :    lista sencilla de canales a seleccionar por ej. [2,3,7]
                     Affected:
                         self.dataOut.data
                         self.dataOut.channelIndexList
                         self.dataOut.nChannels
                         self.dataOut.m_ProcessingHeader.totalSpectra
                         self.dataOut.systemHeaderObj.numChannels
                         self.dataOut.m_ProcessingHeader.blockSize
                     Return:
                         None
                     """
                     for channelIndex in channelIndexList:
                         if channelIndex not in self.dataOut.channelIndexList:
                             print channelIndexList
                             raise ValueError, "The value %d in channelIndexList is not valid" %channelIndex
-            #         nChannels = len(channelIndexList)
                     if self.dataOut.flagDataAsBlock:
                         """
                         Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         data = self.dataOut.data[channelIndexList,:,:]
                     else:
                         data = self.dataOut.data[channelIndexList,:]
                     self.dataOut.data = data
                     self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
             #        self.dataOut.nChannels = nChannels
                     return 1
                 def selectHeights(self, minHei=None, maxHei=None):
                     """
                     Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
                     minHei <= height <= maxHei
                     Input:
                         minHei    :    valor minimo de altura a considerar
                         maxHei    :    valor maximo de altura a considerar
                     Affected:
                         Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if minHei == None:
                         minHei = self.dataOut.heightList[0]
                     if maxHei == None:
                         maxHei = self.dataOut.heightList[-1]
                     if (minHei < self.dataOut.heightList[0]):
                         minHei = self.dataOut.heightList[0]
-            #             raise ValueError, "height range [%d,%d] is not valid. Data height range is [%d, %d]" % (minHei,
-            #                                                                                                     maxHei,
-            #                                                                                                     self.dataOut.heightList[0],
-            #                                                                                                     self.dataOut.heightList[-1])
                     if (maxHei > self.dataOut.heightList[-1]):
                         maxHei = self.dataOut.heightList[-1]
-            #            raise ValueError, "some value in (%d,%d) is not valid" % (minHei, maxHei)
                     minIndex = 0
                     maxIndex = 0
                     heights = self.dataOut.heightList
                     inda = numpy.where(heights >= minHei)
                     indb = numpy.where(heights <= maxHei)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(heights)
                     self.selectHeightsByIndex(minIndex, maxIndex)
                     return 1
                 def selectHeightsByIndex(self, minIndex, maxIndex):
                     """
                     Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
                     minIndex <= index <= maxIndex
                     Input:
                         minIndex    :    valor de indice minimo de altura a considerar
                         maxIndex    :    valor de indice maximo de altura a considerar
                     Affected:
                         self.dataOut.data
                         self.dataOut.heightList
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if (minIndex < 0) or (minIndex > maxIndex):
-                        raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
+                        raise ValueError, "Height index range (%d,%d) is not valid" % (minIndex, maxIndex)
                     if (maxIndex >= self.dataOut.nHeights):
                         maxIndex = self.dataOut.nHeights
-            #            raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
-            #         nHeights = maxIndex - minIndex + 1
                     #voltage
                     if self.dataOut.flagDataAsBlock:
                         """
                         Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         data = self.dataOut.data[:,:, minIndex:maxIndex]
                     else:
                         data = self.dataOut.data[:, minIndex:maxIndex]
             #         firstHeight = self.dataOut.heightList[minIndex]
                     self.dataOut.data = data
                     self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
                     if self.dataOut.nHeights <= 1:
                         raise ValueError, "selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights)
                     return 1
                 def filterByHeights(self, window):
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     if window == None:
                         window = (self.dataOut.radarControllerHeaderObj.txA/self.dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
                     newdelta = deltaHeight * window
                     r = self.dataOut.nHeights % window
                     newheights = (self.dataOut.nHeights-r)/window
                     if newheights <= 1:
                         raise ValueError, "filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(self.dataOut.nHeights, window)
                     if self.dataOut.flagDataAsBlock:
                         """
                         Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         buffer = self.dataOut.data[:, :, 0:self.dataOut.nHeights-r]
                         buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nProfiles,self.dataOut.nHeights/window,window)
                         buffer = numpy.sum(buffer,3)
                     else:
                         buffer = self.dataOut.data[:,0:self.dataOut.nHeights-r]
                         buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nHeights/window,window)
                         buffer = numpy.sum(buffer,2)
                     self.dataOut.data = buffer
                     self.dataOut.heightList = self.dataOut.heightList[0] + numpy.arange( newheights )*newdelta
                     self.dataOut.windowOfFilter = window
                 def setH0(self, h0, deltaHeight = None):
                     if not deltaHeight:
                         deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     nHeights = self.dataOut.nHeights
                     newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
                     self.dataOut.heightList = newHeiRange
                 def deFlip(self, channelList = []):
                     data = self.dataOut.data.copy()
                     if self.dataOut.flagDataAsBlock:
                         flip = self.flip
                         profileList = range(self.dataOut.nProfiles)
                         if not channelList:
                             for thisProfile in profileList:
                                 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
                                 flip *= -1.0
                         else:
                             for thisChannel in channelList:
                                 if thisChannel not in self.dataOut.channelList:
                                     continue
                                 for thisProfile in profileList:
                                     data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
                                     flip *= -1.0
                         self.flip = flip
                     else:
                         if not channelList:
                             data[:,:] = data[:,:]*self.flip
                         else:
                             for thisChannel in channelList:
                                 if thisChannel not in self.dataOut.channelList:
                                     continue
                                 data[thisChannel,:] = data[thisChannel,:]*self.flip
                         self.flip *= -1.
                     self.dataOut.data = data
                 def setRadarFrequency(self, frequency=None):
                     if frequency != None:
                         self.dataOut.frequency = frequency
                     return 1
             class CohInt(Operation):
                 isConfig = False
                 __profIndex = 0
                 __withOverapping  = False
                 __byTime = False
                 __initime = None
                 __lastdatatime = None
                 __integrationtime = None
                 __buffer = None
                 __dataReady = False
                 n = None
                 def __init__(self):
                     Operation.__init__(self)
             #         self.isConfig = False
                 def setup(self, n=None, timeInterval=None, overlapping=False, byblock=False):
                     """
                     Set the parameters of the integration class.
                     Inputs:
                         n        :    Number of coherent integrations
                         timeInterval   :    Time of integration. If the parameter "n" is selected this one does not work
                         overlapping    :
                     """
                     self.__initime = None
                     self.__lastdatatime = 0
                     self.__buffer = None
                     self.__dataReady = False
                     self.byblock = byblock
                     if n == None and timeInterval == None:
                         raise ValueError, "n or timeInterval should be specified ..."
                     if n != None:
                         self.n = n
                         self.__byTime = False
                     else:
                         self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
                         self.n = 9999
                         self.__byTime = True
                     if overlapping:
                         self.__withOverapping = True
                         self.__buffer = None
                     else:
                         self.__withOverapping = False
                         self.__buffer = 0
                     self.__profIndex = 0
                 def putData(self, data):
                     """
                     Add a profile to the __buffer and increase in one the __profileIndex
                     """
                     if not self.__withOverapping:
                         self.__buffer += data.copy()
                         self.__profIndex += 1
                         return
                     #Overlapping data
                     nChannels, nHeis = data.shape
                     data = numpy.reshape(data, (1, nChannels, nHeis))
                     #If the buffer is empty then it takes the data value
                     if self.__buffer is None:
                         self.__buffer = data
                         self.__profIndex += 1
                         return
                     #If the buffer length is lower than n then stakcing the data value
                     if self.__profIndex < self.n:
                         self.__buffer = numpy.vstack((self.__buffer, data))
                         self.__profIndex += 1
                         return
                     #If the buffer length is equal to n then replacing the last buffer value with the data value
                     self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
                     self.__buffer[self.n-1] = data
                     self.__profIndex = self.n
                     return
                 def pushData(self):
                     """
                     Return the sum of the last profiles and the profiles used in the sum.
                     Affected:
                     self.__profileIndex
                     """
                     if not self.__withOverapping:
                         data = self.__buffer
                         n = self.__profIndex
                         self.__buffer = 0
                         self.__profIndex = 0
                         return data, n
                     #Integration with Overlapping
                     data = numpy.sum(self.__buffer, axis=0)
                     n = self.__profIndex
                     return data, n
                 def byProfiles(self, data):
                     self.__dataReady = False
                     avgdata = None
             #         n = None
                     self.putData(data)
                     if self.__profIndex == self.n:
                         avgdata, n = self.pushData()
                         self.__dataReady = True
                     return avgdata
                 def byTime(self, data, datatime):
                     self.__dataReady = False
                     avgdata = None
                     n = None
                     self.putData(data)
                     if (datatime - self.__initime) >= self.__integrationtime:
                         avgdata, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata
                 def integrate(self, data, datatime=None):
                     if self.__initime == None:
                         self.__initime = datatime
                     if self.__byTime:
                         avgdata = self.byTime(data, datatime)
                     else:
                         avgdata = self.byProfiles(data)
                     self.__lastdatatime = datatime
                     if avgdata is None:
                         return None, None
                     avgdatatime = self.__initime
                     deltatime = datatime -self.__lastdatatime
                     if not self.__withOverapping:
                         self.__initime = datatime
                     else:
                         self.__initime += deltatime
                     return avgdata, avgdatatime
                 def integrateByBlock(self, dataOut):
                     times = int(dataOut.data.shape[1]/self.n)
                     avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
                     id_min = 0
                     id_max = self.n
                     for i in range(times):
                         junk = dataOut.data[:,id_min:id_max,:]
                         avgdata[:,i,:] = junk.sum(axis=1)
                         id_min += self.n
                         id_max += self.n
                     timeInterval = dataOut.ippSeconds*self.n
                     avgdatatime = (times - 1) * timeInterval + dataOut.utctime
                     self.__dataReady = True
                     return avgdata, avgdatatime
                 def run(self, dataOut, **kwargs):
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     if dataOut.flagDataAsBlock:
                         """
                         Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         avgdata, avgdatatime = self.integrateByBlock(dataOut)
                     else:
                         avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
             #        dataOut.timeInterval *= n
                     dataOut.flagNoData = True
                     if self.__dataReady:
                         dataOut.data = avgdata
                         dataOut.nCohInt *= self.n
                         dataOut.utctime = avgdatatime
             #             dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
                         dataOut.flagNoData = False
             class Decoder(Operation):
                 isConfig = False
                 __profIndex = 0
                 code = None
                 nCode = None
                 nBaud = None
                 def __init__(self):
                     Operation.__init__(self)
                     self.times = None
                     self.osamp = None
             #         self.__setValues = False
                     self.isConfig = False
                 def setup(self, code, osamp, dataOut):
                     self.__profIndex = 0
                     self.code = code
                     self.nCode = len(code)
                     self.nBaud = len(code[0])
                     if (osamp != None) and (osamp >1):
                         self.osamp = osamp
                         self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
                         self.nBaud = self.nBaud*self.osamp
                     self.__nChannels = dataOut.nChannels
                     self.__nProfiles = dataOut.nProfiles
                     self.__nHeis = dataOut.nHeights
                     if self.__nHeis < self.nBaud:
-                        print 'IOError: Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud)
+                        raise ValueError, 'Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud)
-                        raise IOError, 'Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud)
                     #Frequency
                     __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
                     __codeBuffer[:,0:self.nBaud] = self.code
                     self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
                     if dataOut.flagDataAsBlock:
                         self.ndatadec = self.__nHeis #- self.nBaud + 1
                         self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
                     else:
                         #Time
                         self.ndatadec = self.__nHeis #- self.nBaud + 1
                         self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
                 def __convolutionInFreq(self, data):
                     fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
                     fft_data = numpy.fft.fft(data, axis=1)
                     conv = fft_data*fft_code
                     data = numpy.fft.ifft(conv,axis=1)
                     return data
                 def __convolutionInFreqOpt(self, data):
                     raise NotImplementedError
-            #         fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
-            #         data = cfunctions.decoder(fft_code, data)
-            #         datadec = data#[:,:]
-            #         return datadec
                 def __convolutionInTime(self, data):
                     code = self.code[self.__profIndex]
                     for i in range(self.__nChannels):
                         self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
                     return self.datadecTime
                 def __convolutionByBlockInTime(self, data):
                     repetitions = self.__nProfiles / self.nCode
                     junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
                     junk = junk.flatten()
                     code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
                     for i in range(self.__nChannels):
                         for j in range(self.__nProfiles):
                             self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
                     return self.datadecTime
                 def __convolutionByBlockInFreq(self, data):
                     fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
                     fft_data = numpy.fft.fft(data, axis=2)
                     conv = fft_data*fft_code
                     data = numpy.fft.ifft(conv,axis=2)
                     return data
                 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
                     if dataOut.flagDecodeData:
                         print "This data is already decoded, recoding again ..."
                     if not self.isConfig:
                         if code is None:
                             if dataOut.code is None:
-                                print "Code is not defined"
+                                raise ValueError, "Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type
-                                raise ValueError, "Code could not be read from %s object. Enter a value in Code parameter" %dataOut.type
                             code = dataOut.code
                         else:
                             code = numpy.array(code).reshape(nCode,nBaud)
                         self.setup(code, osamp, dataOut)
                         self.isConfig = True
                     if self.code is None:
                         print "Fail decoding: Code is not defined."
                         return
                     datadec = None
                     if dataOut.flagDataAsBlock:
                         """
                         Decoding when data have been read as block,
                         """
                         if mode == 0:
                             datadec = self.__convolutionByBlockInTime(dataOut.data)
                         if mode == 1:
                             datadec = self.__convolutionByBlockInFreq(dataOut.data)
                     else:
                         """
                         Decoding when data have been read profile by profile
                         """
                         if mode == 0:
                             datadec = self.__convolutionInTime(dataOut.data)
                         if mode == 1:
                             datadec = self.__convolutionInFreq(dataOut.data)
                         if mode == 2:
                             datadec = self.__convolutionInFreqOpt(dataOut.data)
                     if datadec is None:
                         raise ValueError, "Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode
                     dataOut.code = self.code
                     dataOut.nCode = self.nCode
                     dataOut.nBaud = self.nBaud
                     dataOut.data = datadec
                     dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
                     dataOut.flagDecodeData = True #asumo q la data esta decodificada
                     if self.__profIndex == self.nCode-1:
                         self.__profIndex = 0
                         return 1
                     self.__profIndex += 1
                     return 1
             #        dataOut.flagDeflipData = True #asumo q la data no esta sin flip
             class ProfileConcat(Operation):
                 isConfig = False
                 buffer = None
                 def __init__(self):
                     Operation.__init__(self)
                     self.profileIndex = 0
                 def reset(self):
                     self.buffer = numpy.zeros_like(self.buffer)
                     self.start_index = 0
                     self.times = 1
                 def setup(self, data, m, n=1):
                     self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
                     self.nHeights = data.nHeights
                     self.start_index = 0
                     self.times = 1
                 def concat(self, data):
                     self.buffer[:,self.start_index:self.profiles*self.times] = data.copy()
                     self.start_index = self.start_index + self.nHeights
                 def run(self, dataOut, m):
                     dataOut.flagNoData = True
                     if not self.isConfig:
                         self.setup(dataOut.data, m, 1)
                         self.isConfig = True
                     if dataOut.flagDataAsBlock:
                         raise ValueError, "ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False"
                     else:
                         self.concat(dataOut.data)
                         self.times += 1
                         if self.times > m:
                             dataOut.data = self.buffer
                             self.reset()
                             dataOut.flagNoData = False
                             # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
                             deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                             xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
                             dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
                             dataOut.ippSeconds *= m
             class ProfileSelector(Operation):
                 profileIndex = None
                 # Tamanho total de los perfiles
                 nProfiles = None
                 def __init__(self):
                     Operation.__init__(self)
                     self.profileIndex = 0
                 def incIndex(self):
                     self.profileIndex += 1
                     if self.profileIndex >= self.nProfiles:
                         self.profileIndex = 0
                 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
                     if profileIndex < minIndex:
                         return False
                     if profileIndex > maxIndex:
                         return False
                     return True
                 def isThisProfileInList(self, profileIndex, profileList):
                     if profileIndex not in profileList:
                         return False
                     return True
                 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
                     """
                     ProfileSelector:
                     Inputs:
                         profileList        :    Index of profiles selected. Example: profileList = (0,1,2,7,8)
                         profileRangeList    :    Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
                         rangeList            :    List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
                     """
                     dataOut.flagNoData = True
                     if dataOut.flagDataAsBlock:
                         """
                         data dimension  = [nChannels, nProfiles, nHeis]
                         """
                         if profileList != None:
                             dataOut.data = dataOut.data[:,profileList,:]
                             dataOut.nProfiles = len(profileList)
                             dataOut.profileIndex = dataOut.nProfiles - 1
                         if profileRangeList != None:
                             minIndex = profileRangeList[0]
                             maxIndex = profileRangeList[1]
                             dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
                             dataOut.nProfiles = maxIndex - minIndex + 1
                             dataOut.profileIndex = dataOut.nProfiles - 1
                         if rangeList != None:
                             raise ValueError, "Profile Selector: Invalid argument rangeList. Not implemented for getByBlock yet"
                         dataOut.flagNoData = False
                         return True
                     """
                     data dimension  = [nChannels, nHeis]
                     """
                     if nProfiles:
                         self.nProfiles = nProfiles
                     else:
                         self.nProfiles = dataOut.nProfiles
                     if profileList != None:
                         dataOut.nProfiles = len(profileList)
                         if self.isThisProfileInList(dataOut.profileIndex, profileList):
                             dataOut.flagNoData = False
                             dataOut.profileIndex = self.profileIndex
                             self.incIndex()
                         return True
                     if profileRangeList != None:
                         minIndex = profileRangeList[0]
                         maxIndex = profileRangeList[1]
                         dataOut.nProfiles = maxIndex - minIndex + 1
                         if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
                             dataOut.flagNoData = False
                             dataOut.profileIndex = self.profileIndex
                             self.incIndex()
                         return True
                     if rangeList != None:
                         nProfiles = 0
                         for thisRange in rangeList:
                             minIndex = thisRange[0]
                             maxIndex = thisRange[1]
                             nProfiles += maxIndex - minIndex + 1
                         dataOut.nProfiles = nProfiles
                         for thisRange in rangeList:
                             minIndex = thisRange[0]
                             maxIndex = thisRange[1]
                             if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
             #                         print "profileIndex = ", dataOut.profileIndex
                                 dataOut.flagNoData = False
                                 dataOut.profileIndex = self.profileIndex
                                 self.incIndex()
                                 break
                         return True
                     if beam != None: #beam is only for AMISR data
                         if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
                             dataOut.flagNoData = False
                             dataOut.profileIndex = self.profileIndex
                             self.incIndex()
                         return True
                     raise ValueError, "ProfileSelector needs profileList, profileRangeList or rangeList parameter"
                     return False
             class Reshaper(Operation):
                 def __init__(self):
                     Operation.__init__(self)
                     self.updateNewHeights = True
                 def run(self, dataOut, shape):
                     if not dataOut.flagDataAsBlock:
                         raise ValueError, "Reshaper can only be used when voltage have been read as Block, getBlock = True"
                     if len(shape) != 3:
                         raise ValueError, "shape len should be equal to 3, (nChannels, nProfiles, nHeis)"
                     shape_tuple = tuple(shape)
                     dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
                     dataOut.flagNoData = False
                     if self.updateNewHeights:
                         old_nheights = dataOut.nHeights
                         new_nheights = dataOut.data.shape[2]
                         factor = 1.0*new_nheights / old_nheights
                         deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                         xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * factor
                         dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
                         dataOut.nProfiles = dataOut.data.shape[1]
                         dataOut.ippSeconds *= factor
             #
             # import collections
             # from scipy.stats import mode
             #
             # class Synchronize(Operation):
             #
             #     isConfig = False
             #     __profIndex = 0
             #
             #     def __init__(self):
             #
             #         Operation.__init__(self)
             # #         self.isConfig = False
             #         self.__powBuffer = None
             #         self.__startIndex = 0
             #         self.__pulseFound = False
             #
             #     def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
             #
             #         #Read data
             #
             #         powerdB = dataOut.getPower(channel = channel)
             #         noisedB = dataOut.getNoise(channel = channel)[0]
             #
             #         self.__powBuffer.extend(powerdB.flatten())
             #
             #         dataArray = numpy.array(self.__powBuffer)
             #
             #         filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
             #
             #         maxValue = numpy.nanmax(filteredPower)
             #
             #         if maxValue < noisedB + 10:
             #             #No se encuentra ningun pulso de transmision
             #             return None
             #
             #         maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
             #
             #         if len(maxValuesIndex) < 2:
             #             #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
             #             return None
             #
             #         phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
             #
             #         #Seleccionar solo valores con un espaciamiento de nSamples
             #         pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
             #
             #         if len(pulseIndex) < 2:
             #             #Solo se encontro un pulso de transmision con ancho mayor a 1
             #             return None
             #
             #         spacing = pulseIndex[1:] - pulseIndex[:-1]
             #
             #         #remover senales que se distancien menos de 10 unidades o muestras
             #         #(No deberian existir IPP menor a 10 unidades)
             #
             #         realIndex = numpy.where(spacing > 10 )[0]
             #
             #         if len(realIndex) < 2:
             #             #Solo se encontro un pulso de transmision con ancho mayor a 1
             #             return None
             #
             #         #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
             #         realPulseIndex = pulseIndex[realIndex]
             #
             #         period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
             #
             #         print "IPP = %d samples" %period
             #
             #         self.__newNSamples = dataOut.nHeights #int(period)
             #         self.__startIndex = int(realPulseIndex[0])
             #
             #         return 1
             #
             #
             #     def setup(self, nSamples, nChannels, buffer_size = 4):
             #
             #         self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
             #                                           maxlen = buffer_size*nSamples)
             #
             #         bufferList = []
             #
             #         for i in range(nChannels):
             #             bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) +  numpy.NAN,
             #                                           maxlen = buffer_size*nSamples)
             #
             #             bufferList.append(bufferByChannel)
             #
             #         self.__nSamples = nSamples
             #         self.__nChannels = nChannels
             #         self.__bufferList = bufferList
             #
             #     def run(self, dataOut, channel = 0):
             #
             #         if not self.isConfig:
             #             nSamples = dataOut.nHeights
             #             nChannels = dataOut.nChannels
             #             self.setup(nSamples, nChannels)
             #             self.isConfig = True
             #
             #         #Append new data to internal buffer
             #         for thisChannel in range(self.__nChannels):
             #             bufferByChannel = self.__bufferList[thisChannel]
             #             bufferByChannel.extend(dataOut.data[thisChannel])
             #
             #         if self.__pulseFound:
             #             self.__startIndex -= self.__nSamples
             #
             #         #Finding Tx Pulse
             #         if not self.__pulseFound:
             #             indexFound = self.__findTxPulse(dataOut, channel)
             #
             #             if indexFound == None:
             #                 dataOut.flagNoData = True
             #                 return
             #
             #             self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
             #             self.__pulseFound = True
             #             self.__startIndex = indexFound
             #
             #         #If pulse was found ...
             #         for thisChannel in range(self.__nChannels):
             #             bufferByChannel = self.__bufferList[thisChannel]
             #             #print self.__startIndex
             #             x = numpy.array(bufferByChannel)
             #             self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
             #
             #         deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
             #         dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
             # #             dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
             #
             #         dataOut.data = self.__arrayBuffer
             #
             #         self.__startIndex += self.__newNSamples
             #
             #         return

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages