schain Commit - r473:e228fb468666 · Jicamarca Repository

             '''
             $Author: murco $
             $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
             '''
             import os, sys
             import copy
             import numpy
             import datetime
             from jroheaderIO import SystemHeader, RadarControllerHeader
             def hildebrand_sekhon(data, navg):
                 data = data.copy()
                 sortdata = numpy.sort(data,axis=None)
                 lenOfData = len(sortdata)
                 nums_min = lenOfData/10
                 if (lenOfData/10) > 2:
                     nums_min = lenOfData/10
                 else:
                     nums_min = 2
                 sump = 0.
                 sumq = 0.
                 j = 0
                 cont = 1
                 while((cont==1)and(j<lenOfData)):
                     sump += sortdata[j]
                     sumq += sortdata[j]**2
                     j += 1
                     if j > nums_min:
                         rtest = float(j)/(j-1) + 1.0/navg
                         if ((sumq*j) > (rtest*sump**2)):
                             j = j - 1
                             sump  = sump - sortdata[j]
                             sumq =  sumq - sortdata[j]**2
                             cont = 0
                 lnoise = sump /j
                 stdv = numpy.sqrt((sumq - lnoise**2)/(j - 1))
                 return lnoise
             class JROData:
             #    m_BasicHeader = BasicHeader()
             #    m_ProcessingHeader = ProcessingHeader()
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
             #    data = None
                 type = None
                 dtype = None
             #    nChannels = None
             #    nHeights = None
                 nProfiles = None
                 heightList = None
                 channelList = None
                 flagNoData = True
                 flagTimeBlock = False
                 useLocalTime = False
                 utctime = None
                 timeZone = None
                 dstFlag = None
                 errorCount = None
                 blocksize = None
                 nCode = None
                 nBaud = None
                 code = None
                 flagDecodeData = False #asumo q la data no esta decodificada
                 flagDeflipData = False #asumo q la data no esta sin flip
                 flagShiftFFT = False
                 ippSeconds = None
                 timeInterval = None
                 nCohInt = None
                 noise = None
                 windowOfFilter = 1
                 #Speed of ligth
                 C = 3e8
                 frequency = 49.92e6
                 realtime = False
                 beacon_heiIndexList = None
                 last_block = None
                 blocknow = None
                 def __init__(self):
                     raise ValueError, "This class has not been implemented"
                 def copy(self, inputObj=None):
                     if inputObj == None:
                         return copy.deepcopy(self)
                     for key in inputObj.__dict__.keys():
                         self.__dict__[key] = inputObj.__dict__[key]
                 def deepcopy(self):
                     return copy.deepcopy(self)
                 def isEmpty(self):
                     return self.flagNoData
                 def getNoise(self):
                     raise ValueError, "Not implemented"
                 def getNChannels(self):
                     return len(self.channelList)
                 def getChannelIndexList(self):
                     return range(self.nChannels)
                 def getNHeights(self):
                     return len(self.heightList)
                 def getHeiRange(self, extrapoints=0):
                     heis = self.heightList
             #        deltah = self.heightList[1] - self.heightList[0]
             #
             #        heis.append(self.heightList[-1])
                     return heis
                 def getltctime(self):
                     if self.useLocalTime:
                          return self.utctime - self.timeZone*60
                     return self.utctime
                 def getDatatime(self):
                     datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
                     return datatime
                 def getTimeRange(self):
                     datatime = []
                     datatime.append(self.ltctime)
                     datatime.append(self.ltctime + self.timeInterval)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getFmax(self):
                     PRF = 1./(self.ippSeconds * self.nCohInt)
                     fmax = PRF/2.
                     return fmax
                 def getVmax(self):
                     _lambda = self.C/self.frequency
                     vmax = self.getFmax() * _lambda
                     return vmax
                 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
                 channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
                 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
                 noise = property(getNoise, "I'm the 'nHeights' property.")
                 datatime = property(getDatatime, "I'm the 'datatime' property")
                 ltctime = property(getltctime, "I'm the 'ltctime' property")
             class Voltage(JROData):
                 #data es un numpy array de 2 dmensiones (canales, alturas)
                 data = None
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Voltage"
                     self.data = None
                     self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.flagNoData = True
                     self.flagTimeBlock = False
                     self.utctime = None
                     self.timeZone = None
                     self.dstFlag = None
                     self.errorCount = None
                     self.nCohInt = None
                     self.blocksize = None
                     self.flagDecodeData = False #asumo q la data no esta decodificada
                     self.flagDeflipData = False #asumo q la data no esta sin flip
                     self.flagShiftFFT = False
                 def getNoisebyHildebrand(self):
                     """
                     Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                     Return:
                         noiselevel
                     """
                     for channel in range(self.nChannels):
                         daux = self.data_spc[channel,:,:]
                         self.noise[channel] = hildebrand_sekhon(daux, self.nCohInt)
                     return self.noise
                 def getNoise(self, type = 1):
                     self.noise = numpy.zeros(self.nChannels)
                     if type == 1:
                         noise = self.getNoisebyHildebrand()
                     return 10*numpy.log10(noise)
             class Spectra(JROData):
                 #data es un numpy array de 2 dmensiones (canales, perfiles, alturas)
                 data_spc = None
                 #data es un numpy array de 2 dmensiones (canales, pares, alturas)
                 data_cspc = None
                 #data es un numpy array de 2 dmensiones (canales, alturas)
                 data_dc = None
                 nFFTPoints = None
                 nPairs = None
                 pairsList = None
                 nIncohInt = None
                 wavelength = None #Necesario para cacular el rango de velocidad desde la frecuencia
                 nCohInt = None #se requiere para determinar el valor de timeInterval
                 ippFactor = None
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Spectra"
             #        self.data = None
                     self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.flagNoData = True
                     self.flagTimeBlock = False
                     self.utctime = None
                     self.nCohInt = None
                     self.nIncohInt = None
                     self.blocksize = None
                     self.nFFTPoints = None
                     self.wavelength = None
                     self.flagDecodeData = False #asumo q la data no esta decodificada
                     self.flagDeflipData = False #asumo q la data no esta sin flip
                     self.flagShiftFFT = False
                     self.ippFactor = 1
                     self.noise = None
                     self.beacon_heiIndexList = []
                 def getNoisebyHildebrand(self):
                     """
                     Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                     Return:
                         noiselevel
                     """
                     noise = numpy.zeros(self.nChannels)
                     for channel in range(self.nChannels):
                         daux = self.data_spc[channel,:,:]
                         noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
                     return noise
                 def getNoise(self):
                     if self.noise != None:
                         return self.noise
                     else:
                         noise = self.getNoisebyHildebrand()
                         return noise
                 def getFreqRange(self, extrapoints=0):
                     deltafreq = self.getFmax() / (self.nFFTPoints*self.ippFactor)
                     freqrange = deltafreq*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltafreq/2
                     return freqrange
                 def getVelRange(self, extrapoints=0):
                     deltav = self.getVmax() / (self.nFFTPoints*self.ippFactor)
                     velrange = deltav*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltav/2
                     return velrange
                 def getNPairs(self):
                     return len(self.pairsList)
                 def getPairsIndexList(self):
                     return range(self.nPairs)
                 def getNormFactor(self):
                     pwcode = 1
                     if self.flagDecodeData:
                         pwcode = numpy.sum(self.code[0]**2)
                     #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
                     normFactor = self.nProfiles*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
                     return normFactor
                 def getFlagCspc(self):
                     if self.data_cspc == None:
                         return True
                     return False
                 def getFlagDc(self):
                     if self.data_dc == None:
                         return True
                     return False
                 nPairs = property(getNPairs, "I'm the 'nPairs' property.")
                 pairsIndexList = property(getPairsIndexList, "I'm the 'pairsIndexList' property.")
                 normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
                 flag_cspc = property(getFlagCspc)
                 flag_dc = property(getFlagDc)
             class SpectraHeis(JROData):
                 data_spc = None
                 data_cspc = None
                 data_dc = None
                 nFFTPoints = None
                 nPairs = None
                 pairsList = None
                 nIncohInt = None
                 def __init__(self):
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "SpectraHeis"
                     self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.flagNoData = True
                     self.flagTimeBlock = False
                     self.nPairs = 0
                     self.utctime = None
                     self.blocksize = None
             class Fits:
                 heightList = None
                 channelList = None
                 flagNoData = True
                 flagTimeBlock = False
                 useLocalTime = False
                 utctime = None
                 timeZone = None
                 ippSeconds = None
                 timeInterval = None
                 nCohInt = None
                 nIncohInt = None
                 noise = None
                 windowOfFilter = 1
                 #Speed of ligth
                 C = 3e8
                 frequency = 49.92e6
                 realtime = False
                 def __init__(self):
                     self.type = "Fits"
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #         self.channelIndexList = None
                     self.flagNoData = True
                     self.utctime = None
                     self.nCohInt = None
                     self.nIncohInt = None
                     self.useLocalTime = True
             #         self.utctime = None
             #         self.timeZone = None
             #         self.ltctime = None
             #         self.timeInterval = None
             #         self.header = None
             #         self.data_header = None
             #         self.data = None
             #         self.datatime = None
             #         self.flagNoData = False
             #         self.expName = ''
             #         self.nChannels = None
             #         self.nSamples = None
             #         self.dataBlocksPerFile = None
             #         self.comments = ''
             #
                 def getltctime(self):
                     if self.useLocalTime:
                          return self.utctime - self.timeZone*60
                     return self.utctime
                 def getDatatime(self):
                     datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
                     return datatime
                 def getTimeRange(self):
                     datatime = []
                     datatime.append(self.ltctime)
                     datatime.append(self.ltctime + self.timeInterval)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getHeiRange(self):
                     heis = self.heightList
                     return heis
                 def isEmpty(self):
                     return self.flagNoData
                 def getNHeights(self):
                     return len(self.heightList)
                 def getNChannels(self):
                     return len(self.channelList)
                 def getChannelIndexList(self):
                     return range(self.nChannels)
                 def getNoise(self, type = 1):
                     self.noise = numpy.zeros(self.nChannels)
                     if type == 1:
                         noise = self.getNoisebyHildebrand()
                     if type == 2:
                         noise = self.getNoisebySort()
                     if type == 3:
                         noise = self.getNoisebyWindow()
                     return noise
                 datatime = property(getDatatime, "I'm the 'datatime' property")
                 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
                 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
                 channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
                 noise = property(getNoise, "I'm the 'nHeights' property.")
                 datatime = property(getDatatime, "I'm the 'datatime' property")
                 ltctime = property(getltctime, "I'm the 'ltctime' property")
-                ltctime = property(getltctime, "I'm the 'ltctime' property")
  No newline at end of file
+                ltctime = property(getltctime, "I'm the 'ltctime' property")
+            class AMISR:
+                def __init__(self):
+                    self.flagNoData = True
+                    self.data = None
+                    self.utctime = None
+                    self.type = "AMISR"
+                def isEmpty(self):
+                    return self.flagNoData
  No newline at end of file

             '''
             $Author: murco $
             $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
             '''
             import os, sys
             import glob
             import time
             import numpy
             import fnmatch
             import time, datetime
+            import h5py
             from xml.etree.ElementTree import Element, SubElement, ElementTree
             try:
                 import pyfits
             except:
                 print "pyfits module has not been imported, it should be installed to save files in fits format"
             from jrodata import *
             from jroheaderIO import *
             from jroprocessing import *
             LOCALTIME = True #-18000
             def isNumber(str):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float( str )
                     return True
                 except:
                     return False
             def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename,'rb')
                 except:
                     raise IOError, "The file %s can't be opened" %(filename)
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isFileinThisTime(filename, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename,'rb')
                 except:
                     raise IOError, "The file %s can't be opened" %(filename)
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = basicHeaderObj.datatime
                 thisTime = basicHeaderObj.datatime.time()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return None
                 if not ((startTime <= thisTime) and (endTime > thisTime)):
                     return None
                 return thisDatetime
             def getFileFromSet(path,ext,set):
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for file in fileList:
                     try:
                         year = int(file[1:5])
                         doy  = int(file[5:8])
                     except:
                         continue
                     if (os.path.splitext(file)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(file)
                 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
                 if len(myfile)!= 0:
                     return myfile[0]
                 else:
                     filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
                     print 'the filename %s does not exist'%filename
                     print '...going to the last file: '
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for file in fileList:
                     try:
                         year = int(file[1:5])
                         doy  = int(file[5:8])
                     except:
                         continue
                     if (os.path.splitext(file)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(file)
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def checkForRealPath(path, foldercounter, year, doy, set, ext):
                 """
                 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
                 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
                 el path exacto de un determinado file.
                 Example    :
                     nombre correcto del file es  .../.../D2009307/P2009307367.ext
                     Entonces la funcion prueba con las siguientes combinaciones
                         .../.../y2009307367.ext
                         .../.../Y2009307367.ext
                         .../.../x2009307/y2009307367.ext
                         .../.../x2009307/Y2009307367.ext
                         .../.../X2009307/y2009307367.ext
                         .../.../X2009307/Y2009307367.ext
                     siendo para este caso, la ultima combinacion de letras, identica al file buscado
                 Return:
                     Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
                     caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
                     para el filename
                 """
                 fullfilename = None
                 find_flag = False
                 filename = None
                 prefixDirList = [None,'d','D']
                 if ext.lower() == ".r": #voltage
                     prefixFileList = ['d','D']
                 elif ext.lower() == ".pdata": #spectra
                     prefixFileList = ['p','P']
                 else:
                     return None, filename
                 #barrido por las combinaciones posibles
                 for prefixDir in prefixDirList:
                     thispath = path
                     if prefixDir != None:
                         #formo el nombre del directorio xYYYYDDD (x=d o x=D)
                         if foldercounter == 0:
                             thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
                         else:
                             thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
                     for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
                         filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
                         fullfilename = os.path.join( thispath, filename ) #formo el path completo
                         if os.path.exists( fullfilename ): #verifico que exista
                             find_flag = True
                             break
                     if find_flag:
                         break
                 if not(find_flag):
                     return None, filename
                 return fullfilename, filename
             def isDoyFolder(folder):
                 try:
                     year = int(folder[1:5])
                 except:
                     return 0
                 try:
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
             class JRODataIO:
                 c = 3E8
                 isConfig = False
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
                 processingHeaderObj = ProcessingHeader()
                 online = 0
                 dtype = None
                 pathList = []
                 filenameList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagTimeBlock = 0
                 flagIsNewBlock = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
                 ippSeconds = None
                 fileSizeByHeader = None
                 fileIndex = None
                 profileIndex = None
                 blockIndex = None
                 nTotalBlocks = None
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
                 blocksize = None
                 def __init__(self):
                     raise ValueError, "Not implemented"
                 def run(self):
                     raise ValueError, "Not implemented"
                 def getOutput(self):
                     return self.dataOut
             class JRODataReader(JRODataIO, ProcessingUnit):
                 nReadBlocks = 0
                 delay  = 10   #number of seconds waiting a new file
                 nTries  = 3  #quantity tries
                 nFiles = 3   #number of files for searching
                 path = None
                 foldercounter = 0
                 flagNoMoreFiles = 0
                 datetimeList = []
                 __isFirstTimeOnline = 1
                 __printInfo = True
                 profileIndex = None
                 def __init__(self):
                     """
                     """
                     raise ValueError, "This method has not been implemented"
                 def createObjByDefault(self):
                     """
                     """
                     raise ValueError, "This method has not been implemented"
                 def getBlockDimension(self):
                     raise ValueError, "No implemented"
                 def __searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.r',
                                         walk=True):
                     pathList = []
                     if not walk:
                         #pathList.append(path)
                         multi_path = path.split(',')
                         for single_path in multi_path:
                             pathList.append(single_path)
                     else:
                         #dirList = []
                         multi_path = path.split(',')
                         for single_path in multi_path:
                             dirList = []
                             for thisPath in os.listdir(single_path):
                                 if not os.path.isdir(os.path.join(single_path,thisPath)):
                                     continue
                                 if not isDoyFolder(thisPath):
                                     continue
                                 dirList.append(thisPath)
                             if not(dirList):
                                 return None, None
                             thisDate = startDate
                             while(thisDate <= endDate):
                                 year = thisDate.timetuple().tm_year
                                 doy = thisDate.timetuple().tm_yday
                                 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
                                 if len(matchlist) == 0:
                                     thisDate += datetime.timedelta(1)
                                     continue
                                 for match in matchlist:
                                     pathList.append(os.path.join(single_path,match,expLabel))
                                 thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
                         return None, None
                     print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
                     filenameList = []
                     datetimeList = []
                     pathDict = {}
                     filenameList_to_sort = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         pathDict.setdefault(fileList[0])
                         pathDict[fileList[0]] = i
                         filenameList_to_sort.append(fileList[0])
                     filenameList_to_sort.sort()
                     for file in filenameList_to_sort:
                         thisPath = pathList[pathDict[file]]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             thisDatetime = isFileinThisTime(filename, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "Any file was found for the time range %s - %s" %(startTime, endTime)
                         return None, None
                     print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
                     """
                     Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
                     devuelve el archivo encontrado ademas de otros datos.
                     Input:
                         path           :    carpeta donde estan contenidos los files que contiene data
                         expLabel        :     Nombre del subexperimento (subfolder)
                         ext              :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
                         directory   :    eL directorio donde esta el file encontrado
                         filename    :    el ultimo file de una determinada carpeta
                         year        :    el anho
                         doy         :    el numero de dia del anho
                         set         :    el set del archivo
                     """
                     dirList = []
                     if not walk:
                         fullpath = path
                         foldercounter = 0
                     else:
                         #Filtra solo los directorios
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isDoyFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None, None, None, None, None
                         dirList = sorted( dirList, key=str.lower )
                         doypath = dirList[-1]
                         foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
                         fullpath = os.path.join(path, doypath, expLabel)
                     print "%s folder was found: " %(fullpath )
                     if set == None:
                         filename = getlastFileFromPath(fullpath, ext)
                     else:
                         filename = getFileFromSet(fullpath, ext, set)
                     if not(filename):
                         return None, None, None, None, None, None
                     print "%s file was found" %(filename)
                     if not(self.__verifyFile(os.path.join(fullpath, filename))):
                         return None, None, None, None, None, None
                     year = int( filename[1:5] )
                     doy  = int( filename[5:8] )
                     set  = int( filename[8:11] )
                     return fullpath, foldercounter, filename, year, doy, set
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print "No more Files"
                             return 0
                         filename = self.filenameList[idFile]
                         if not(self.__verifyFile(filename)):
                             continue
                         fileSize = os.path.getsize(filename)
                         fp = open(filename,'rb')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fp = fp
                     print "Setting the file: %s"%self.filename
                     return 1
                 def __setNextFileOnline(self):
                     """
                     Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
                     no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
                     siguientes.
                     Affected:
                         self.flagIsNewFile
                         self.filename
                         self.fileSize
                         self.fp
                         self.set
                         self.flagNoMoreFiles
                     Return:
 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
 : si el file fue abierto con exito y esta listo a ser leido
                     Excepciones:
                         Si un determinado file no puede ser abierto
                     """
                     nFiles = 0
                     fileOk_flag = False
                     firstTime_flag = True
                     self.set += 1
                     if self.set > 999:
                         self.set = 0
                         self.foldercounter += 1
                     #busca el 1er file disponible
                     fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                     if fullfilename:
                         if self.__verifyFile(fullfilename, False):
                             fileOk_flag = True
                     #si no encuentra un file entonces espera y vuelve a buscar
                     if not(fileOk_flag):
                         for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
                             if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
                                 tries = self.nTries
                             else:
                                 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
                             for nTries in range( tries ):
                                 if firstTime_flag:
                                     print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
                                     time.sleep( self.delay )
                                 else:
                                     print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
                                 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                                 if fullfilename:
                                     if self.__verifyFile(fullfilename):
                                         fileOk_flag = True
                                         break
                             if fileOk_flag:
                                 break
                             firstTime_flag = False
                             print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename
                             self.set += 1
                             if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
                                 self.set = 0
                                 self.doy += 1
                                 self.foldercounter = 0
                     if fileOk_flag:
                         self.fileSize = os.path.getsize( fullfilename )
                         self.filename = fullfilename
                         self.flagIsNewFile = 1
                         if self.fp != None: self.fp.close()
                         self.fp = open(fullfilename, 'rb')
                         self.flagNoMoreFiles = 0
                         print 'Setting the file: %s' % fullfilename
                     else:
                         self.fileSize = 0
                         self.filename = None
                         self.flagIsNewFile = 0
                         self.fp = None
                         self.flagNoMoreFiles = 1
                         print 'No more Files'
                     return fileOk_flag
                 def setNextFile(self):
                     if self.fp != None:
                         self.fp.close()
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         return 0
                     self.__readFirstHeader()
                     self.nReadBlocks = 0
                     return 1
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.__rdBasicHeader()
                             return 1
                         if self.fileSize == self.fileSizeByHeader:
             #                self.flagEoF = True
                             return 0
                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         time.sleep( self.delay )
                     return 0
                 def waitDataBlock(self,pointer_location):
                     currentPointer = pointer_location
                     neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             return 1
                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         time.sleep( self.delay )
                     return 0
                 def __jumpToLastBlock(self):
                     if not(self.__isFirstTimeOnline):
                         return
                     csize = self.fileSize - self.fp.tell()
                     blocksize = self.processingHeaderObj.blockSize
                     #salta el primer bloque de datos
                     if csize > self.processingHeaderObj.blockSize:
                         self.fp.seek(self.fp.tell() + blocksize)
                     else:
                         return
                     csize = self.fileSize - self.fp.tell()
                     neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     while True:
                         if self.fp.tell()<self.fileSize:
                             self.fp.seek(self.fp.tell() + neededsize)
                         else:
                             self.fp.seek(self.fp.tell() - neededsize)
                             break
             #        csize = self.fileSize - self.fp.tell()
             #        neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
             #        factor = int(csize/neededsize)
             #        if factor > 0:
             #            self.fp.seek(self.fp.tell() + factor*neededsize)
                     self.flagIsNewFile = 0
                     self.__isFirstTimeOnline = 0
                 def __setNewBlock(self):
                     if self.fp == None:
                         return 0
                     if self.online:
                         self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         return 1
                     self.lastUTTime = self.basicHeaderObj.utc
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.__rdBasicHeader()
                         return 1
                     if self.__waitNewBlock():
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
                     self.flagTimeBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagTimeBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def __rdProcessingHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.processingHeaderObj.read(fp)
                 def __rdRadarControllerHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.radarControllerHeaderObj.read(fp)
                 def __rdSystemHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.systemHeaderObj.read(fp)
                 def __rdBasicHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.basicHeaderObj.read(fp)
                 def __readFirstHeader(self):
                     self.__rdBasicHeader()
                     self.__rdSystemHeader()
                     self.__rdRadarControllerHeader()
                     self.__rdProcessingHeader()
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
                     else:
                         raise ValueError, 'Data type was not defined'
                     self.dtype = datatype_str
                     self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
             #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
             #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def __verifyFile(self, filename, msgFlag=True):
                     msg = None
                     try:
                         fp = open(filename, 'rb')
                         currentPosition = fp.tell()
                     except:
                         if msgFlag:
                             print "The file %s can't be opened" % (filename)
                         return False
                     neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                     if neededSize == 0:
                         basicHeaderObj = BasicHeader(LOCALTIME)
                         systemHeaderObj = SystemHeader()
                         radarControllerHeaderObj = RadarControllerHeader()
                         processingHeaderObj = ProcessingHeader()
                         try:
                             if not( basicHeaderObj.read(fp) ): raise IOError
                             if not( systemHeaderObj.read(fp) ): raise IOError
                             if not( radarControllerHeaderObj.read(fp) ): raise IOError
                             if not( processingHeaderObj.read(fp) ): raise IOError
                             data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                             neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
                         except:
                             if msgFlag:
                                 print "\tThe file %s is empty or it hasn't enough data" % filename
                             fp.close()
                             return False
                     else:
                         msg = "\tSkipping the file %s due to it hasn't enough data" %filename
                     fp.close()
                     fileSize = os.path.getsize(filename)
                     currentSize = fileSize - currentPosition
                     if currentSize < neededSize:
                         if msgFlag and (msg != None):
                             print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
                         return False
                     return True
                 def setup(self,
                             path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=None,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True):
                     if path == None:
                         raise ValueError, "The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if online:
                         print "Searching files in online mode..."
                         for nTries in range( self.nTries ):
                             fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
                             if fullpath:
                                 break
                             print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
                             time.sleep( self.delay )
                         if not(fullpath):
                             print "There 'isn't valied files in %s" % path
                             return None
                         self.year = year
                         self.doy  = doy
                         self.set  = set - 1
                         self.path = path
                         self.foldercounter = foldercounter
                         last_set = None
                     else:
                         print "Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
                             print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime())
                             sys.exit(-1)
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                         file_name = os.path.basename(filenameList[-1])
                         basename, ext = os.path.splitext(file_name)
                         last_set = int(basename[-3:])
                     self.online = online
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "No files"
                         sys.exit(-1)
             #        self.updateDataHeader()
                     if last_set != None:
                         self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
                     return self.dataOut
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds
                     self.dataOut.flagTimeBlock = self.flagTimeBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                 def getFirstHeader(self):
                     raise ValueError, "This method has not been implemented"
                 def getData():
                     raise ValueError, "This method has not been implemented"
                 def hasNotDataInBuffer():
                     raise ValueError, "This method has not been implemented"
                 def readBlock():
                     raise ValueError, "This method has not been implemented"
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print "Number of read blocks per file %04d" %self.nReadBlocks
                 def printTotalBlocks(self):
                     print "Number of read blocks %04d" %self.nTotalBlocks
                 def printNumberOfBlock(self):
                     if self.flagIsNewBlock:
                         print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
                         self.dataOut.blocknow = self.basicHeaderObj.dataBlock
                 def printInfo(self):
                     if self.__printInfo == False:
                         return
                     self.basicHeaderObj.printInfo()
                     self.systemHeaderObj.printInfo()
                     self.radarControllerHeaderObj.printInfo()
                     self.processingHeaderObj.printInfo()
                     self.__printInfo = False
                 def run(self, **kwargs):
                     if not(self.isConfig):
             #            self.dataOut = dataOut
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             class JRODataWriter(JRODataIO, Operation):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 blockIndex = 0
                 path = None
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 def __init__(self, dataOut=None):
                     raise ValueError, "Not implemented"
                 def hasAllDataInBuffer(self):
                     raise ValueError, "Not implemented"
                 def setBlockDimension(self):
                     raise ValueError, "Not implemented"
                 def writeBlock(self):
                     raise ValueError, "No implemented"
                 def putData(self):
                     raise ValueError, "No implemented"
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize #bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond  = (self.dataOut.utctime - utc)* 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
                     raise ValueError, "No implemented"
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                     self.dtype = self.dataOut.dtype
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def setNextFile(self):
                     """
                     Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, file )
                     fp = open( filename,'wb' )
                     self.blockIndex = 0
                     #guardando atributos
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.setFirstHeader()
                     print 'Writing the file: %s'%self.filename
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path      :    el path destino en el cual se escribiran los files a crear
                         format    :    formato en el cual sera salvado un file
                         set       :    el setebo del file
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     ext = ext.lower()
                     self.ext = ext
                     self.path = path
                     self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     if not(self.setNextFile()):
                         print "There isn't a next file"
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class VoltageReader(JRODataReader):
                 """
                 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
                 perfiles*alturas*canales) son almacenados en la variable "buffer".
                                          perfiles * alturas * canales
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = VoltageReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         #to get one profile
                         profile =  readerObj.getData()
                         #print the profile
                         print profile
                         #If you want to see all datablock
                         print readerObj.datablock
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 ext = ".r"
                 optchar = "D"
                 dataOut = None
                 def __init__(self):
                     """
                     Inicializador de la clase VoltageReader para la lectura de datos de voltage.
                     Input:
                         dataOut    :    Objeto de la clase Voltage. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Variables afectadas:
                         self.dataOut
                     Return:
                         None
                     """
                     self.isConfig = False
                     self.datablock = None
                     self.utc = 0
                     self.ext = ".r"
                     self.optchar = "D"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.online = 0
                     self.fp = None
                     self.idFile = None
                     self.dtype = None
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.filename = None
                     self.fileSize = None
                     self.firstHeaderSize = 0
                     self.basicHeaderSize = 24
                     self.pathList = []
                     self.filenameList = []
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
                     self.flagNoMoreFiles = 0
                     self.set = 0
                     self.path = None
                     self.profileIndex = 2**32-1
                     self.delay  = 3   #seconds
                     self.nTries  = 3  #quantity tries
                     self.nFiles = 3   #number of files for searching
                     self.nReadBlocks = 0
                     self.flagIsNewFile = 1
                     self.__isFirstTimeOnline = 1
                     self.ippSeconds = 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     self.nTotalBlocks = 0
                     self.blocksize = 0
                     self.dataOut = self.createObjByDefault()
                 def createObjByDefault(self):
                     dataObj = Voltage()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
                         return 1
                     return 0
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.blocksize
                     Return:
                         None
                     """
                     pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
                     self.blocksize = pts2read
                 def readBlock(self):
                     """
                     readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Inputs:
                         None
                     Return:
                         None
                     Affected:
                         self.profileIndex
                         self.datablock
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
                     current_pointer_location = self.fp.tell()
                     junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
                     try:
                         junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
                     except:
                         #print "The read block (%3d) has not enough data" %self.nReadBlocks
                         if self.waitDataBlock(pointer_location=current_pointer_location):
                             junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
                             junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
             #             return 0
                     junk = numpy.transpose(junk, (2,0,1))
                     self.datablock = junk['real'] + junk['imag']*1j
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.dataOut.dtype = self.dtype
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
                     self.dataOut.ippSeconds = self.ippSeconds
                     self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.flagShiftFFT = False
                     if self.radarControllerHeaderObj.code != None:
                         self.dataOut.nCode = self.radarControllerHeaderObj.nCode
                         self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
                         self.dataOut.code = self.radarControllerHeaderObj.code
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip
                     self.dataOut.flagShiftFFT = False
                 def getData(self):
                     """
                     getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Ademas incrementa el contador del buffer en 1.
                     Return:
                         data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                      buffer. Si no hay mas archivos a leer retorna None.
                     Variables afectadas:
                         self.dataOut
                         self.profileIndex
                     Affected:
                         self.dataOut
                         self.profileIndex
                         self.flagTimeBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             return 0
                         self.getFirstHeader()
                     if self.datablock == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.dataOut.data = self.datablock[:,self.profileIndex,:]
                     self.dataOut.flagNoData = False
                     self.getBasicHeader()
                     self.profileIndex += 1
                     self.dataOut.realtime = self.online
                     return self.dataOut.data
             class VoltageWriter(JRODataWriter):
                 """
                 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".r"
                 optchar = "D"
                 shapeBuffer = None
                 def __init__(self):
                     """
                     Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                     Return: None
                     """
                     self.nTotalBlocks = 0
                     self.profileIndex = 0
                     self.isConfig = False
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.nTotalBlocks = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
                         return 1
                     return 0
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
                                         self.processingHeaderObj.nHeights,
                                         self.systemHeaderObj.nChannels)
                     self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
                                                  self.processingHeaderObj.profilesPerBlock,
                                                  self.processingHeaderObj.nHeights),
                                                  dtype=numpy.dtype('complex64'))
                 def writeBlock(self):
                     """
                     Escribe el buffer en el file designado
                     Affected:
                         self.profileIndex
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.blockIndex
                     Return: None
                     """
                     data = numpy.zeros( self.shapeBuffer, self.dtype )
                     junk = numpy.transpose(self.datablock, (1,2,0))
                     data['real'] = junk.real
                     data['imag'] = junk.imag
                     data = data.reshape( (-1) )
                     data.tofile( self.fp )
                     self.datablock.fill(0)
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.flagIsNewBlock
                         self.profileIndex
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagTimeBlock:
                         self.datablock.fill(0)
                         self.profileIndex = 0
                         self.setNextFile()
                     if self.profileIndex == 0:
                         self.setBasicHeader()
                     self.datablock[:,self.profileIndex,:] = self.dataOut.data
                     self.profileIndex += 1
                     if self.hasAllDataInBuffer():
                         #if self.flagIsNewFile:
                         self.writeNextBlock()
             #            self.setFirstHeader()
                     return 1
                 def __getProcessFlags(self):
                     processFlags = 0
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList =  [PROCFLAG.DATATYPE_CHAR,
                                        PROCFLAG.DATATYPE_SHORT,
                                        PROCFLAG.DATATYPE_LONG,
                                        PROCFLAG.DATATYPE_INT64,
                                        PROCFLAG.DATATYPE_FLOAT,
                                        PROCFLAG.DATATYPE_DOUBLE]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             dtypeValue = datatypeValueList[index]
                             break
                     processFlags += dtypeValue
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code != None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nCohInt > 1:
                         processFlags += PROCFLAG.COHERENT_INTEGRATION
                     return processFlags
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
                     '''
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList = [1,2,4,8,4,8]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             datatypeValue = datatypeValueList[index]
                             break
                     blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * datatypeValue * 2)
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.setBasicHeader()
                     processingHeaderSize = 40 # bytes
                     self.processingHeaderObj.dtype = 0 # Voltage
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.processFlags = self.__getProcessFlags()
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
                     self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
                     self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
             #         if self.dataOut.code != None:
             #             self.processingHeaderObj.code = self.dataOut.code
             #             self.processingHeaderObj.nCode = self.dataOut.nCode
             #             self.processingHeaderObj.nBaud = self.dataOut.nBaud
             #             codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)
             #             processingHeaderSize += codesize
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                         processingHeaderSize += 12
                     self.processingHeaderObj.size = processingHeaderSize
             class SpectraReader(JRODataReader):
                 """
                 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
                 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
                                     paresCanalesIguales    * alturas * perfiles  (Self Spectra)
                                     paresCanalesDiferentes * alturas * perfiles  (Cross Spectra)
                                     canales * alturas                            (DC Channels)
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = SpectraReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         readerObj.getData()
                         print readerObj.data_spc
                         print readerObj.data_cspc
                         print readerObj.data_dc
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 pts2read_SelfSpectra = 0
                 pts2read_CrossSpectra = 0
                 pts2read_DCchannels = 0
                 ext = ".pdata"
                 optchar = "P"
                 dataOut = None
                 nRdChannels = None
                 nRdPairs = None
                 rdPairList = []
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraReader para la lectura de datos de espectros.
                     Inputs:
                         dataOut    :    Objeto de la clase Spectra. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Affected:
                         self.dataOut
                     Return      : None
                     """
                     self.isConfig = False
                     self.pts2read_SelfSpectra = 0
                     self.pts2read_CrossSpectra = 0
                     self.pts2read_DCchannels = 0
                     self.datablock = None
                     self.utc = None
                     self.ext = ".pdata"
                     self.optchar = "P"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.online = 0
                     self.fp = None
                     self.idFile = None
                     self.dtype = None
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.filename = None
                     self.fileSize = None
                     self.firstHeaderSize = 0
                     self.basicHeaderSize = 24
                     self.pathList = []
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
                     self.flagNoMoreFiles = 0
                     self.set = 0
                     self.path = None
                     self.delay  = 60   #seconds
                     self.nTries = 3  #quantity tries
                     self.nFiles = 3   #number of files for searching
                     self.nReadBlocks = 0
                     self.flagIsNewFile = 1
                     self.__isFirstTimeOnline = 1
                     self.ippSeconds = 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     self.nTotalBlocks = 0
                     self.blocksize = 0
                     self.dataOut = self.createObjByDefault()
                     self.profileIndex = 1 #Always
                 def createObjByDefault(self):
                     dataObj = Spectra()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     return 1
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.nRdChannels
                         self.nRdPairs
                         self.pts2read_SelfSpectra
                         self.pts2read_CrossSpectra
                         self.pts2read_DCchannels
                         self.blocksize
                         self.dataOut.nChannels
                         self.dataOut.nPairs
                     Return:
                         None
                     """
                     self.nRdChannels = 0
                     self.nRdPairs = 0
                     self.rdPairList = []
                     for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
                         if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
                             self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
                         else:
                             self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
                             self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
                     pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
                     self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
                     self.blocksize = self.pts2read_SelfSpectra
                     if self.processingHeaderObj.flag_cspc:
                         self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
                         self.blocksize += self.pts2read_CrossSpectra
                     if self.processingHeaderObj.flag_dc:
                         self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
                         self.blocksize += self.pts2read_DCchannels
             #        self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
                 def readBlock(self):
                     """
                     Lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Return: None
                     Variables afectadas:
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
                     blockOk_flag = False
                     fpointer = self.fp.tell()
                     spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
                     spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
                         cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_dc:
                         dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
                         dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
                     if not(self.processingHeaderObj.shif_fft):
                         #desplaza a la derecha en el eje 2 determinadas posiciones
                         shift = int(self.processingHeaderObj.profilesPerBlock/2)
                         spc = numpy.roll( spc, shift , axis=2 )
                         if self.processingHeaderObj.flag_cspc:
                             #desplaza a la derecha en el eje 2 determinadas posiciones
                             cspc = numpy.roll( cspc, shift, axis=2 )
             #            self.processingHeaderObj.shif_fft = True
                     spc = numpy.transpose( spc, (0,2,1) )
                     self.data_spc = spc
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.transpose( cspc, (0,2,1) )
                         self.data_cspc = cspc['real'] + cspc['imag']*1j
                     else:
                         self.data_cspc = None
                     if self.processingHeaderObj.flag_dc:
                         self.data_dc = dc['real'] + dc['imag']*1j
                     else:
                         self.data_dc = None
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.dataOut.dtype = self.dtype
                     self.dataOut.nPairs = self.nRdPairs
                     self.dataOut.pairsList = self.rdPairList
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
                     self.dataOut.ippSeconds = self.ippSeconds
                     self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
                     self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip
                     if self.radarControllerHeaderObj.code != None:
                         self.dataOut.nCode = self.radarControllerHeaderObj.nCode
                         self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
                         self.dataOut.code = self.radarControllerHeaderObj.code
                         self.dataOut.flagDecodeData = True
                 def getData(self):
                     """
                     Copia el buffer de lectura a la clase "Spectra",
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Return:
 :    Si no hay mas archivos disponibles
 :    Si hizo una buena copia del buffer
                     Affected:
                         self.dataOut
                         self.flagTimeBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             self.dataOut.flagNoData = True
                             return 0
                     #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
                     if self.data_dc == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.getBasicHeader()
                     self.getFirstHeader()
                     self.dataOut.data_spc = self.data_spc
                     self.dataOut.data_cspc = self.data_cspc
                     self.dataOut.data_dc = self.data_dc
                     self.dataOut.flagNoData = False
                     self.dataOut.realtime = self.online
                     return self.dataOut.data_spc
             class SpectraWriter(JRODataWriter):
                 """
                 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".pdata"
                 optchar = "P"
                 shape_spc_Buffer = None
                 shape_cspc_Buffer = None
                 shape_dc_Buffer = None
                 data_spc = None
                 data_cspc = None
                 data_dc = None
             #    dataOut = None
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj
                     Return: None
                     """
                     self.isConfig = False
                     self.nTotalBlocks = 0
                     self.data_spc = None
                     self.data_cspc = None
                     self.data_dc = None
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.nTotalBlocks = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.noMoreFiles = 0
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     return 1
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shape_spc_Buffer = (self.dataOut.nChannels,
                                              self.processingHeaderObj.nHeights,
                                              self.processingHeaderObj.profilesPerBlock)
                     self.shape_cspc_Buffer = (self.dataOut.nPairs,
                                               self.processingHeaderObj.nHeights,
                                               self.processingHeaderObj.profilesPerBlock)
                     self.shape_dc_Buffer = (self.dataOut.nChannels,
                                             self.processingHeaderObj.nHeights)
                 def writeBlock(self):
                     """
                     Escribe el buffer en el file designado
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.nWriteBlocks
                     Return: None
                     """
                     spc = numpy.transpose( self.data_spc, (0,2,1) )
                     if not( self.processingHeaderObj.shif_fft ):
                         spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                     data = spc.reshape((-1))
                     data = data.astype(self.dtype[0])
                     data.tofile(self.fp)
                     if self.data_cspc != None:
                         data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
                         cspc = numpy.transpose( self.data_cspc, (0,2,1) )
                         if not( self.processingHeaderObj.shif_fft ):
                             cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                         data['real'] = cspc.real
                         data['imag'] = cspc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     if self.data_dc != None:
                         data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
                         dc = self.data_dc
                         data['real'] = dc.real
                         data['imag'] = dc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     self.data_spc.fill(0)
                     if self.data_dc != None:
                         self.data_dc.fill(0)
                     if self.data_cspc != None:
                         self.data_cspc.fill(0)
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nWriteBlocks += 1
                     self.blockIndex += 1
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagTimeBlock:
                         self.data_spc.fill(0)
                         self.data_cspc.fill(0)
                         self.data_dc.fill(0)
                         self.setNextFile()
                     if self.flagIsNewFile == 0:
                         self.setBasicHeader()
                     self.data_spc = self.dataOut.data_spc.copy()
                     if self.dataOut.data_cspc != None:
                         self.data_cspc = self.dataOut.data_cspc.copy()
                     self.data_dc = self.dataOut.data_dc.copy()
                     # #self.processingHeaderObj.dataBlocksPerFile)
                     if self.hasAllDataInBuffer():
             #            self.setFirstHeader()
                         self.writeNextBlock()
                     return 1
                 def __getProcessFlags(self):
                     processFlags = 0
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList =  [PROCFLAG.DATATYPE_CHAR,
                                        PROCFLAG.DATATYPE_SHORT,
                                        PROCFLAG.DATATYPE_LONG,
                                        PROCFLAG.DATATYPE_INT64,
                                        PROCFLAG.DATATYPE_FLOAT,
                                        PROCFLAG.DATATYPE_DOUBLE]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             dtypeValue = datatypeValueList[index]
                             break
                     processFlags += dtypeValue
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code != None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nIncohInt > 1:
                         processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                     if self.dataOut.data_dc != None:
                         processFlags += PROCFLAG.SAVE_CHANNELS_DC
                     return processFlags
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
                     '''
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList = [1,2,4,8,4,8]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             datatypeValue = datatypeValueList[index]
                             break
                     pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
                     pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
                     blocksize = (pts2write_SelfSpectra*datatypeValue)
                     if self.dataOut.data_cspc != None:
                         pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
                         blocksize += (pts2write_CrossSpectra*datatypeValue*2)
                     if self.dataOut.data_dc != None:
                         pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
                         blocksize += (pts2write_DCchannels*datatypeValue*2)
                     blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     old_code_size = self.dataOut.radarControllerHeaderObj.code_size
                     new_code_size = int(numpy.ceil(self.dataOut.nBaud/32.))*self.dataOut.nCode*4
                     self.radarControllerHeaderObj.size = self.radarControllerHeaderObj.size - old_code_size + new_code_size
                     self.setBasicHeader()
                     processingHeaderSize = 40 # bytes
                     self.processingHeaderObj.dtype = 1 # Spectra
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.processFlags = self.__getProcessFlags()
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
                     self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
                     self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
                     self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
                     if self.processingHeaderObj.totalSpectra > 0:
                         channelList = []
                         for channel in range(self.dataOut.nChannels):
                             channelList.append(channel)
                             channelList.append(channel)
                         pairsList = []
                         if self.dataOut.nPairs > 0:
                             for pair in self.dataOut.pairsList:
                                 pairsList.append(pair[0])
                                 pairsList.append(pair[1])
                         spectraComb = channelList + pairsList
                         spectraComb = numpy.array(spectraComb,dtype="u1")
                         self.processingHeaderObj.spectraComb = spectraComb
                         sizeOfSpcComb = len(spectraComb)
                         processingHeaderSize += sizeOfSpcComb
             #        The processing header should not have information about code
             #        if self.dataOut.code != None:
             #            self.processingHeaderObj.code = self.dataOut.code
             #            self.processingHeaderObj.nCode = self.dataOut.nCode
             #            self.processingHeaderObj.nBaud = self.dataOut.nBaud
             #            nCodeSize = 4 # bytes
             #            nBaudSize = 4 # bytes
             #            codeSize = 4 # bytes
             #            sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)
             #            processingHeaderSize += sizeOfCode
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                         sizeOfFirstHeight = 4
                         sizeOfdeltaHeight = 4
                         sizeOfnHeights = 4
                         sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows
                         processingHeaderSize += sizeOfWindows
                     self.processingHeaderObj.size = processingHeaderSize
             class SpectraHeisWriter(Operation):
             #    set = None
                 setFile = None
                 idblock = None
                 doypath = None
                 subfolder = None
                 def __init__(self):
                     self.wrObj = FITS()
             #        self.dataOut = dataOut
                     self.nTotalBlocks=0
             #        self.set = None
                     self.setFile = None
                     self.idblock = 0
                     self.wrpath = None
                     self.doypath = None
                     self.subfolder = None
                     self.isConfig = False
                 def isNumber(str):
                     """
                     Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                     Excepciones:
                     Si un determinado string no puede ser convertido a numero
                     Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                     Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                     """
                     try:
                         float( str )
                         return True
                     except:
                         return False
                 def setup(self, dataOut, wrpath):
                     if not(os.path.exists(wrpath)):
                         os.mkdir(wrpath)
                     self.wrpath = wrpath
             #        self.setFile = 0
                     self.dataOut = dataOut
                 def putData(self):
                     name= time.localtime( self.dataOut.utctime)
                     ext=".fits"
                     if self.doypath == None:
                        self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
                        self.doypath = os.path.join( self.wrpath, self.subfolder )
                        os.mkdir(self.doypath)
                     if self.setFile == None:
             #           self.set = self.dataOut.set
                        self.setFile = 0
             #        if self.set != self.dataOut.set:
             ##            self.set = self.dataOut.set
             #            self.setFile = 0
                     #make the filename
                     file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
                     filename = os.path.join(self.wrpath,self.subfolder, file)
                     idblock = numpy.array([self.idblock],dtype="int64")
                     header=self.wrObj.cFImage(idblock=idblock,
                                             year=time.gmtime(self.dataOut.utctime).tm_year,
                                             month=time.gmtime(self.dataOut.utctime).tm_mon,
                                             day=time.gmtime(self.dataOut.utctime).tm_mday,
                                             hour=time.gmtime(self.dataOut.utctime).tm_hour,
                                             minute=time.gmtime(self.dataOut.utctime).tm_min,
                                             second=time.gmtime(self.dataOut.utctime).tm_sec)
                     c=3E8
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
                     colList = []
                     colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
                     colList.append(colFreq)
                     nchannel=self.dataOut.nChannels
                     for i in range(nchannel):
                         col = self.wrObj.writeData(name="PCh"+str(i+1),
                                                      format=str(self.dataOut.nFFTPoints)+'E',
                                                       data=10*numpy.log10(self.dataOut.data_spc[i,:]))
                         colList.append(col)
                     data=self.wrObj.Ctable(colList=colList)
                     self.wrObj.CFile(header,data)
                     self.wrObj.wFile(filename)
                     #update the setFile
                     self.setFile += 1
                     self.idblock += 1
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class ParameterConf:
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.name = ''
                     self.value = ''
                 def readXml(self, parmElement):
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                 def getElementName(self):
                     return self.ELEMENTNAME
             class Metadata:
                 def __init__(self, filename):
                     self.parmConfObjList = []
                     self.readXml(filename)
                 def readXml(self, filename):
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     self.projectElement = ElementTree().parse(filename)
                     self.project = self.projectElement.tag
                     parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         self.parmConfObjList.append(parmConfObj)
             class FitsWriter(Operation):
                 def __init__(self):
                     self.isConfig = False
                     self.dataBlocksPerFile = None
                     self.blockIndex = 0
                     self.flagIsNewFile = 1
                     self.fitsObj = None
                     self.optchar = 'P'
                     self.ext = '.fits'
                     self.setFile = 0
                 def setFitsHeader(self, dataOut, metadatafile):
                     header_data = pyfits.PrimaryHDU()
                     metadata4fits = Metadata(metadatafile)
                     for parameter in metadata4fits.parmConfObjList:
                         parm_name = parameter.name
                         parm_value = parameter.value
             #             if parm_value == 'fromdatadatetime':
             #                 value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
             #             elif parm_value == 'fromdataheights':
             #                 value = dataOut.nHeights
             #             elif parm_value == 'fromdatachannel':
             #                 value = dataOut.nChannels
             #             elif parm_value == 'fromdatasamples':
             #                 value = dataOut.nFFTPoints
             #             else:
             #                 value = parm_value
                         header_data.header[parm_name] = parm_value
                     header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
                     header_data.header['CHANNELLIST'] = str(dataOut.channelList)
                     header_data.header['NCHANNELS'] = dataOut.nChannels
                     #header_data.header['HEIGHTS'] = dataOut.heightList
                     header_data.header['NHEIGHTS'] = dataOut.nHeights
                     header_data.header['IPPSECONDS'] = dataOut.ippSeconds
                     header_data.header['NCOHINT'] = dataOut.nCohInt
                     header_data.header['NINCOHINT'] = dataOut.nIncohInt
                     header_data.header['TIMEZONE'] = dataOut.timeZone
                     header_data.header['NBLOCK'] = self.blockIndex
                     header_data.writeto(self.filename)
                     self.addExtension(dataOut.heightList,'HEIGHTLIST')
                 def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):
                     self.path = path
                     self.dataOut = dataOut
                     self.metadatafile = metadatafile
                     self.dataBlocksPerFile = dataBlocksPerFile
                 def open(self):
                     self.fitsObj = pyfits.open(self.filename, mode='update')
                 def addExtension(self, data, tagname):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=tagname)
                     #extension.header['TAG'] = tagname
                     self.fitsObj.append(extension)
                     self.write()
                 def addData(self, data):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
                     extension.header['UTCTIME'] = self.dataOut.utctime
                     self.fitsObj.append(extension)
                     self.blockIndex += 1
                     self.fitsObj[0].header['NBLOCK'] = self.blockIndex
                     self.write()
                 def write(self):
                     self.fitsObj.flush(verbose=True)
                     self.fitsObj.close()
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, file )
                     self.blockIndex = 0
                     self.filename = filename
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     print 'Writing the file: %s'%self.filename
                     self.setFitsHeader(self.dataOut, self.metadatafile)
                     return 1
                 def writeBlock(self):
                     self.addData(self.dataOut.data_spc)
                     self.flagIsNewFile = 0
                 def __setNewBlock(self):
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.dataBlocksPerFile:
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def putData(self):
                     if self.flagIsNewFile:
                         self.setNextFile()
                     self.writeNextBlock()
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class FitsReader(ProcessingUnit):
             #     __TIMEZONE = time.timezone
                 expName = None
                 datetimestr = None
                 utc = None
                 nChannels = None
                 nSamples = None
                 dataBlocksPerFile = None
                 comments = None
                 lastUTTime = None
                 header_dict = None
                 data = None
                 data_header_dict = None
                 def __init__(self):
                     self.isConfig = False
                     self.ext = '.fits'
                     self.setFile = 0
                     self.flagNoMoreFiles = 0
                     self.flagIsNewFile = 1
                     self.flagTimeBlock = None
                     self.fileIndex = None
                     self.filename = None
                     self.fileSize = None
                     self.fitsObj = None
                     self.timeZone = None
                     self.nReadBlocks = 0
                     self.nTotalBlocks = 0
                     self.dataOut = self.createObjByDefault()
                     self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
                     self.blockIndex = 1
                 def createObjByDefault(self):
                     dataObj = Fits()
                     return dataObj
                 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
                     try:
                         fitsObj = pyfits.open(filename,'readonly')
                     except:
                         raise IOError, "The file %s can't be opened" %(filename)
                     header = fitsObj[0].header
                     struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
                     utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
                     ltc = utc
                     if useLocalTime:
                         ltc -= time.timezone
                     thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
                     thisTime = thisDatetime.time()
                     if not ((startTime <= thisTime) and (endTime > thisTime)):
                         return None
                     return thisDatetime
                 def __setNextFileOnline(self):
                     raise ValueError, "No implemented"
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print "No more Files"
                             return 0
                         filename = self.filenameList[idFile]
             #            if not(self.__verifyFile(filename)):
             #                continue
                         fileSize = os.path.getsize(filename)
                         fitsObj = pyfits.open(filename,'readonly')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fitsObj = fitsObj
                     self.blockIndex = 0
                     print "Setting the file: %s"%self.filename
                     return 1
                 def readHeader(self):
                     headerObj = self.fitsObj[0]
                     self.header_dict = headerObj.header
                     if 'EXPNAME' in headerObj.header.keys():
                         self.expName = headerObj.header['EXPNAME']
                     if 'DATATYPE' in headerObj.header.keys():
                         self.dataType = headerObj.header['DATATYPE']
                     self.datetimestr = headerObj.header['DATETIME']
                     channelList = headerObj.header['CHANNELLIST']
                     channelList = channelList.split('[')
                     channelList = channelList[1].split(']')
                     channelList = channelList[0].split(',')
                     channelList = [int(ch) for ch in channelList]
                     self.channelList = channelList
                     self.nChannels = headerObj.header['NCHANNELS']
                     self.nHeights = headerObj.header['NHEIGHTS']
                     self.ippSeconds = headerObj.header['IPPSECONDS']
                     self.nCohInt = headerObj.header['NCOHINT']
                     self.nIncohInt = headerObj.header['NINCOHINT']
                     self.dataBlocksPerFile = headerObj.header['NBLOCK']
                     self.timeZone = headerObj.header['TIMEZONE']
                     self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     if 'COMMENT' in headerObj.header.keys():
                         self.comments = headerObj.header['COMMENT']
                     self.readHeightList()
                 def readHeightList(self):
                     self.blockIndex = self.blockIndex + 1
                     obj = self.fitsObj[self.blockIndex]
                     self.heightList = obj.data
                     self.blockIndex = self.blockIndex + 1
                 def readExtension(self):
                     obj = self.fitsObj[self.blockIndex]
                     self.heightList = obj.data
                     self.blockIndex = self.blockIndex + 1
                 def setNextFile(self):
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         return 0
                     self.readHeader()
                     self.nReadBlocks = 0
             #         self.blockIndex = 1
                     return 1
                 def __searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.fits',
                                         walk=True):
                     pathList = []
                     if not walk:
                         pathList.append(path)
                     else:
                         dirList = []
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isDoyFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None
                         thisDate = startDate
                         while(thisDate <= endDate):
                             year = thisDate.timetuple().tm_year
                             doy = thisDate.timetuple().tm_yday
                             matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
                             if len(matchlist) == 0:
                                 thisDate += datetime.timedelta(1)
                                 continue
                             for match in matchlist:
                                 pathList.append(os.path.join(path,match,expLabel))
                             thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
                         return None, None
                     print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
                     filenameList = []
                     datetimeList = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "Any file was found for the time range %s - %s" %(startTime, endTime)
                         return None, None
                     print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def setup(self, path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=0,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True):
                     if path == None:
                         raise ValueError, "The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if not(online):
                         print "Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
                             print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime())
                             sys.exit(-1)
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                     self.online = online
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "No files"
                         sys.exit(-1)
                 def readBlock(self):
                     dataObj = self.fitsObj[self.blockIndex]
                     self.data = dataObj.data
                     self.data_header_dict = dataObj.header
                     self.utc = self.data_header_dict['UTCTIME']
                     self.flagIsNewFile = 0
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def __jumpToLastBlock(self):
                     raise ValueError, "No implemented"
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.__rdBasicHeader()
                             return 1
                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         time.sleep( self.delay )
                     return 0
                 def __setNewBlock(self):
                     if self.online:
                         self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         return 1
                     self.lastUTTime = self.utc
                     if self.online:
                         if self.__waitNewBlock():
                             return 1
                     if self.nReadBlocks < self.dataBlocksPerFile:
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.utc - self.lastUTTime
                     self.flagTimeBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagTimeBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     if not(self.readNextBlock()):
                         return 0
                     if self.data == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.dataOut.data = self.data
                     self.dataOut.data_header = self.data_header_dict
                     self.dataOut.utctime = self.utc
                     self.dataOut.header = self.header_dict
                     self.dataOut.expName = self.expName
                     self.dataOut.nChannels = self.nChannels
                     self.dataOut.timeZone = self.timeZone
                     self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
                     self.dataOut.comments = self.comments
                     self.dataOut.timeInterval = self.timeInterval
                     self.dataOut.channelList = self.channelList
                     self.dataOut.heightList = self.heightList
                     self.dataOut.flagNoData = False
                     return self.dataOut.data
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
-                    self.getData()
  No newline at end of file
+                    self.getData()
+            class RadacHeader():
+                def __init__(self, fp):
+                    header = 'Raw11/Data/RadacHeader'
+                    self.beamCode = fp.get(header+'/BeamCode')
+                    self.code = fp.get(header+'/Code')
+                    self.frameCount = fp.get(header+'/FrameCount')
+                    self.modeGroup = fp.get(header+'/ModeGroup')
+                    self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
+                    self.pulseCount = fp.get(header+'/PulseCount')
+                    self.radacTime = fp.get(header+'/RadacTime')
+                    self.timeCount = fp.get(header+'/TimeCount')
+                    self.timeStatus = fp.get(header+'/TimeStatus')
+                    self.nrecords = self.pulseCount.shape[0] #numero de bloques
+                    self.npulses = self.pulseCount.shape[1] #numero de perfiles
+                    self.nsamples = self.nsamplesPulse[0,0] #numero de alturas
+                def getIndexRangeToPulse(self, idrecord=0):
+                    indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
+                    startPulseCountId = indexToZero[0][0]
+                    endPulseCountId = startPulseCountId - 1
+                    range1 = numpy.arange(startPulseCountId,self.npulses,1)
+                    range2 = numpy.arange(0,startPulseCountId,1)
+                    return range1, range2
+            class AMISRReader(ProcessingUnit):
+                path = None
+                startDate = None
+                endDate = None
+                startTime = None
+                endTime = None
+                walk = None
+                isConfig = False
+                def __init__(self):
+                    self.set = None
+                    self.subset = None
+                    self.extension_file = '.h5'
+                    self.dtc_str = 'dtc'
+                    self.dtc_id = 0
+                    self.status = True
+                    self.isConfig = False
+                    self.dirnameList = []
+                    self.filenameList = []
+                    self.fileIndex = None
+                    self.flagNoMoreFiles = False
+                    self.flagIsNewFile = 0
+                    self.filename = ''
+                    self.amisrFilePointer = None
+                    self.radacHeaderObj = None
+                    self.dataOut = self.__createObjByDefault()
+                    self.datablock = None
+                    self.rest_datablock = None
+                    self.range = None
+                    self.idrecord_count = 0
+                    self.profileIndex = 0
+                    self.idpulse_range1 = None
+                    self.idpulse_range2 = None
+                    self.beamCodeByFrame = None
+                    self.radacTimeByFrame = None
+                    #atributos originales tal y como esta en el archivo de datos
+                    self.beamCodesFromFile = None
+                    self.radacTimeFromFile = None
+                    self.rangeFromFile = None
+                    self.dataByFrame = None
+                    self.dataset = None
+                def __createObjByDefault(self):
+                    dataObj = AMISR()
+                    return dataObj
+                def __setParameters(self,path,startDate,endDate,startTime,endTime,walk):
+                    self.path = path
+                    self.startDate = startDate
+                    self.endDate = endDate
+                    self.startTime = startTime
+                    self.endTime = endTime
+                    self.walk = walk
+                def __checkPath(self):
+                    if os.path.exists(self.path):
+                        self.status = 1
+                    else:
+                        self.status = 0
+                        print 'Path:%s does not exists'%self.path
+                    return
+                def __selDates(self, amisr_dirname_format):
+                    year = int(amisr_dirname_format[0:4])
+                    month = int(amisr_dirname_format[4:6])
+                    dom = int(amisr_dirname_format[6:8])
+                    thisDate = datetime.date(year,month,dom)
+                    if (thisDate>=self.startDate and thisDate <= self.endDate):
+                        return amisr_dirname_format
+                def __findDataForDates(self):
+                    import re
+                    if not(self.status):
+                        return None
+                    pat = '\d+.\d+'
+                    dirnameList = [re.search(pat,x).string for x in os.listdir(self.path)]
+                    dirnameList = [self.__selDates(x) for x in dirnameList]
+                    dirnameList = filter(lambda x:x!=None,dirnameList)
+                    if len(dirnameList)>0:
+                        self.status = 1
+                        self.dirnameList = dirnameList
+                        self.dirnameList.sort()
+                    else:
+                        self.status = 0
+                        return None
+                def __getTimeFromData(self):
+                    pass
+                def __filterByGlob1(self, dirName):
+                    filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
+                    filterDict = {}
+                    filterDict.setdefault(dirName)
+                    filterDict[dirName] = filter_files
+                    return filterDict
+                def __getFilenameList(self, fileListInKeys, dirList):
+                    for value in fileListInKeys:
+                        dirName = value.keys()[0]
+                        for file in value[dirName]:
+                            filename = os.path.join(dirName, file)
+                            self.filenameList.append(filename)
+                def __selectDataForTimes(self):
+                    #aun no esta implementado el filtro for tiempo
+                    if not(self.status):
+                        return None
+                    dirList = [os.path.join(self.path,x) for x in self.dirnameList]
+                    fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
+                    self.__getFilenameList(fileListInKeys, dirList)
+                    if len(self.filenameList)>0:
+                        self.status = 1
+                        self.filenameList.sort()
+                    else:
+                        self.status = 0
+                        return None
+                def __searchFilesOffline(self,
+                                        path,
+                                        startDate,
+                                        endDate,
+                                        startTime=datetime.time(0,0,0),
+                                        endTime=datetime.time(23,59,59),
+                                        walk=True):
+                    self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
+                    self.__checkPath()
+                    self.__findDataForDates()
+                    self.__selectDataForTimes()
+                    for i in range(len(self.filenameList)):
+                        print "%s" %(self.filenameList[i])
+                    return
+                def __setNextFileOffline(self):
+                    idFile = self.fileIndex
+                    while (True):
+                        idFile += 1
+                        if not(idFile < len(self.filenameList)):
+                            self.flagNoMoreFiles = 1
+                            print "No more Files"
+                            return 0
+                        filename = self.filenameList[idFile]
+                        amisrFilePointer = h5py.File(filename,'r')
+                        break
+                    self.flagIsNewFile = 1
+                    self.fileIndex = idFile
+                    self.filename = filename
+                    self.amisrFilePointer = amisrFilePointer
+                    print "Setting the file: %s"%self.filename
+                    return 1
+                def __readHeader(self):
+                    self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
+                    self.flagIsNewFile = 1
+                def __setNextFile(self):
+                    newFile = self.__setNextFileOffline()
+                    if not(newFile):
+                        return 0
+                    self.__readHeader()
+                    self.readDataBlock()
+                def setup(self,path=None,
+                                startDate=None,
+                                endDate=None,
+                                startTime=datetime.time(0,0,0),
+                                endTime=datetime.time(23,59,59),
+                                walk=True):
+                    #Busqueda de archivos offline
+                    self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
+                    if not(self.filenameList):
+                        print "There is no files into the folder: %s"%(path)
+                        sys.exit(-1)
+                    self.fileIndex = -1
+                    self.__setNextFile()
+                def readRanges(self):
+                    dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
+                    self.rangeFromFile = dataset.values()
+                    return range
+                def readRadacTime(self,idrecord, range1, range2):
+                    self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
+                    radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
+                    #radacTimeByFrame = dataset[idrecord - 1,range1]
+                    #radacTimeByFrame = dataset[idrecord,range2]
+                    return radacTimeByFrame
+                def readBeamCode(self, idrecord, range1, range2):
+                    dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
+                    beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
+                    self.beamCodesFromFile = dataset.value
+                    #beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
+                    #beamcodeByFrame[range2] = dataset[idrecord, range2]
+                    beamcodeByFrame[range1] = dataset[idrecord, range1]
+                    beamcodeByFrame[range2] = dataset[idrecord, range2]
+                    return beamcodeByFrame
+                def __setDataByFrame(self):
+                    ndata = 2 # porque es complejo
+                    dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
+                    return dataByFrame
+                def __readDataSet(self):
+                    dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
+                    return dataset
+                def __setDataBlock(self,):
+                    real = self.dataByFrame[:,:,0] #asumo que 0 es real
+                    imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
+                    datablock = real + imag*1j #armo el complejo
+                    return datablock
+                def readSamples_version1(self,idrecord):
+                    #estas tres primeras lineas solo se deben ejecutar una vez
+                    if self.flagIsNewFile:
+                        self.idpulse_range1, self.idpulse_range2 = self.radacHeaderObj.getIndexRangeToPulse(0)
+                        self.dataByFrame = self.__setDataByFrame()
+                        self.beamCodeByFrame = self.readBeamCode(idrecord, self.idpulse_range1, self.idpulse_range2)
+                        self.radacTimeByFrame = self.readRadacTime(idrecord, self.idpulse_range1, self.idpulse_range2)
+                        #reading dataset
+                        self.dataset = self.__readDataSet()
+                    if idrecord == 0:
+                        if len(numpy.where(self.dataByFrame!=0.0)[0]) or len(numpy.where(self.dataByFrame!=0.0)[1]) or len(numpy.where(self.dataByFrame!=0.0)[2]):
+                            #falta agregar una condicion para datos discontinuos
+                            #por defecto une los datos del record anterior
+                            self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :]
+                            #timepulse
+                            self.radacTimeByFrame[self.idpulse_range2] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range2]
+                        else:
+                            self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :]
+                            self.radacTimeByFrame[self.idpulse_range1] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range1]
+                        datablock = self.__setDataBlock()
+                        return datablock
+                    self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord - 1,self.idpulse_range1, :, :]
+                    self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :]
+                    datablock = self.__setDataBlock()
+                    self.flagIsNewFile = 0
+                    self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :]
+                    return datablock
+                def readSamples(self,idrecord):
+                    if self.flagIsNewFile:
+                        self.dataByFrame = self.__setDataByFrame()
+                        self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
+                        #reading dataset
+                        self.dataset = self.__readDataSet()
+                    self.flagIsNewFile = 0
+                    self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
+                    self.dataByFrame = self.dataset[idrecord, :, :, :]
+                    datablock = self.__setDataBlock()
+                    return datablock
+                def readDataBlock(self):
+                    #self.datablock = self.readSamples(self.idrecord_count)
+                    self.datablock = self.readSamples(self.idrecord_count)
+                    #print 'record:', self.idrecord_count
+                    self.idrecord_count += 1
+                    self.profileIndex = 0
+                    if self.idrecord_count >= self.radacHeaderObj.nrecords:
+                        self.idrecord_count = 0
+                        self.flagIsNewFile = 1
+                def readNextBlock(self):
+                    self.readDataBlock()
+                    if self.flagIsNewFile:
+                        self.__setNextFile()
+                    pass
+                def __hasNotDataInBuffer(self):
+                    #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
+                    if self.profileIndex >= self.radacHeaderObj.npulses:
+                        return 1
+                    return 0
+                def getData(self):
+                    if self.flagNoMoreFiles:
+                        self.dataOut.flagNoData = True
+                        print 'Process finished'
+                        return 0
+                    if self.__hasNotDataInBuffer():
+                        self.readNextBlock()
+            #             if not( self.readNextBlock() ):
+            #                 return 0
+            #             self.getFirstHeader()
+                    if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
+                        self.dataOut.flagNoData = True
+                        return 0
+                    self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
+                    self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
+                    self.dataOut.flagNoData = False
+                    self.profileIndex += 1
+                    return self.dataOut.data
+                def run(self, **kwargs):
+                    if not(self.isConfig):
+                        self.setup(**kwargs)
+                        self.isConfig = True
+                    self.getData()

1

'''

1

'''

2

3

$Author: murco $

3

$Author: murco $

4

$Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $

4

$Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $

5

'''

5

'''

6

7

import os, sys

7

import os, sys

8

import glob

8

import glob

9

import time

9

import time

10

import numpy

10

import numpy

11

import fnmatch

11

import fnmatch

12

import time, datetime

12

import time, datetime

13

import h5py

13

from xml.etree.ElementTree import Element, SubElement, ElementTree

14

from xml.etree.ElementTree import Element, SubElement, ElementTree

14

try:

15

try:

15

import pyfits

16

import pyfits

16

except:

17

except:

17

print "pyfits module has not been imported, it should be installed to save files in fits format"

18

print "pyfits module has not been imported, it should be installed to save files in fits format"

18

19

from jrodata import *

20

from jrodata import *

20

from jroheaderIO import *

21

from jroheaderIO import *

21

from jroprocessing import *

22

from jroprocessing import *

22

23

LOCALTIME = True #-18000

24

LOCALTIME = True #-18000

24

25

def isNumber(str):

26

def isNumber(str):

26

"""

27

"""

27

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

28

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

28

29

Excepciones:

30

Excepciones:

30

Si un determinado string no puede ser convertido a numero

31

Si un determinado string no puede ser convertido a numero

31

Input:

32

Input:

32

str, string al cual se le analiza para determinar si convertible a un numero o no

33

str, string al cual se le analiza para determinar si convertible a un numero o no

33

34

Return:

35

Return:

35

True : si el string es uno numerico

36

True : si el string es uno numerico

36

False : no es un string numerico

37

False : no es un string numerico

37

"""

38

"""

38

try:

39

try:

39

float( str )

40

float( str )

40

return True

41

return True

41

except:

42

except:

42

return False

43

return False

43

44

def isThisFileinRange(filename, startUTSeconds, endUTSeconds):

45

def isThisFileinRange(filename, startUTSeconds, endUTSeconds):

45

"""

46

"""

46

Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.

47

Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.

47

48

Inputs:

49

Inputs:

49

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

50

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

50

51

startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en

52

startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en

52

segundos contados desde 01/01/1970.

53

segundos contados desde 01/01/1970.

53

endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en

54

endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en

54

segundos contados desde 01/01/1970.

55

segundos contados desde 01/01/1970.

55

56

Return:

57

Return:

57

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

58

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

58

fecha especificado, de lo contrario retorna False.

59

fecha especificado, de lo contrario retorna False.

59

60

Excepciones:

61

Excepciones:

61

Si el archivo no existe o no puede ser abierto

62

Si el archivo no existe o no puede ser abierto

62

Si la cabecera no puede ser leida.

63

Si la cabecera no puede ser leida.

63

64

"""

65

"""

65

basicHeaderObj = BasicHeader(LOCALTIME)

66

basicHeaderObj = BasicHeader(LOCALTIME)

66

67

try:

68

try:

68

fp = open(filename,'rb')

69

fp = open(filename,'rb')

69

except:

70

except:

70

raise IOError, "The file %s can't be opened" %(filename)

71

raise IOError, "The file %s can't be opened" %(filename)

71

72

sts = basicHeaderObj.read(fp)

73

sts = basicHeaderObj.read(fp)

73

fp.close()

74

fp.close()

74

75

if not(sts):

76

if not(sts):

76

print "Skipping the file %s because it has not a valid header" %(filename)

77

print "Skipping the file %s because it has not a valid header" %(filename)

77

return 0

78

return 0

78

79

if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):

80

if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):

80

return 0

81

return 0

81

82

return 1

83

return 1

83

84

def isFileinThisTime(filename, startTime, endTime):

85

def isFileinThisTime(filename, startTime, endTime):

85

"""

86

"""

86

Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.

87

Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.

87

88

Inputs:

89

Inputs:

89

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

90

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

90

91

startTime : tiempo inicial del rango seleccionado en formato datetime.time

92

startTime : tiempo inicial del rango seleccionado en formato datetime.time

92

93

endTime : tiempo final del rango seleccionado en formato datetime.time

94

endTime : tiempo final del rango seleccionado en formato datetime.time

94

95

Return:

96

Return:

96

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

97

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

97

fecha especificado, de lo contrario retorna False.

98

fecha especificado, de lo contrario retorna False.

98

99

Excepciones:

100

Excepciones:

100

Si el archivo no existe o no puede ser abierto

101

Si el archivo no existe o no puede ser abierto

101

Si la cabecera no puede ser leida.

102

Si la cabecera no puede ser leida.

102

103

"""

104

"""

104

105

106

try:

107

try:

107

fp = open(filename,'rb')

108

fp = open(filename,'rb')

108

except:

109

except:

109

raise IOError, "The file %s can't be opened" %(filename)

110

raise IOError, "The file %s can't be opened" %(filename)

110

111

basicHeaderObj = BasicHeader(LOCALTIME)

112

basicHeaderObj = BasicHeader(LOCALTIME)

112

sts = basicHeaderObj.read(fp)

113

sts = basicHeaderObj.read(fp)

113

fp.close()

114

fp.close()

114

115

thisDatetime = basicHeaderObj.datatime

116

thisDatetime = basicHeaderObj.datatime

116

thisTime = basicHeaderObj.datatime.time()

117

thisTime = basicHeaderObj.datatime.time()

117

118

if not(sts):

119

if not(sts):

119

print "Skipping the file %s because it has not a valid header" %(filename)

120

print "Skipping the file %s because it has not a valid header" %(filename)

120

return None

121

return None

121

122

if not ((startTime <= thisTime) and (endTime > thisTime)):

123

if not ((startTime <= thisTime) and (endTime > thisTime)):

123

return None

124

return None

124

125

return thisDatetime

126

return thisDatetime

126

127

def getFileFromSet(path,ext,set):

128

def getFileFromSet(path,ext,set):

128

validFilelist = []

129

validFilelist = []

129

fileList = os.listdir(path)

130

fileList = os.listdir(path)

130

131

# 0 1234 567 89A BCDE

132

# 0 1234 567 89A BCDE

132

# H YYYY DDD SSS .ext

133

# H YYYY DDD SSS .ext

133

134

for file in fileList:

135

for file in fileList:

135

try:

136

try:

136

year = int(file[1:5])

137

year = int(file[1:5])

137

doy = int(file[5:8])

138

doy = int(file[5:8])

138

139

140

except:

141

except:

141

continue

142

continue

142

143

if (os.path.splitext(file)[-1].lower() != ext.lower()):

144

if (os.path.splitext(file)[-1].lower() != ext.lower()):

144

continue

145

continue

145

146

validFilelist.append(file)

147

validFilelist.append(file)

147

148

myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))

149

myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))

149

150

if len(myfile)!= 0:

151

if len(myfile)!= 0:

151

return myfile[0]

152

return myfile[0]

152

else:

153

else:

153

filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())

154

filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())

154

print 'the filename %s does not exist'%filename

155

print 'the filename %s does not exist'%filename

155

print '...going to the last file: '

156

print '...going to the last file: '

156

157

if validFilelist:

158

if validFilelist:

158

validFilelist = sorted( validFilelist, key=str.lower )

159

validFilelist = sorted( validFilelist, key=str.lower )

159

return validFilelist[-1]

160

return validFilelist[-1]

160

161

return None

162

return None

162

163

164

def getlastFileFromPath(path, ext):

165

def getlastFileFromPath(path, ext):

165

"""

166

"""

166

Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"

167

Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"

167

al final de la depuracion devuelve el ultimo file de la lista que quedo.

168

al final de la depuracion devuelve el ultimo file de la lista que quedo.

168

169

Input:

170

Input:

170

fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta

171

fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta

171

ext : extension de los files contenidos en una carpeta

172

ext : extension de los files contenidos en una carpeta

172

173

Return:

174

Return:

174

El ultimo file de una determinada carpeta, no se considera el path.

175

El ultimo file de una determinada carpeta, no se considera el path.

175

"""

176

"""

176

validFilelist = []

177

validFilelist = []

177

fileList = os.listdir(path)

178

fileList = os.listdir(path)

178

179

# 0 1234 567 89A BCDE

180

# 0 1234 567 89A BCDE

180

# H YYYY DDD SSS .ext

181

# H YYYY DDD SSS .ext

181

182

for file in fileList:

183

for file in fileList:

183

try:

184

try:

184

year = int(file[1:5])

185

year = int(file[1:5])

185

doy = int(file[5:8])

186

doy = int(file[5:8])

186

187

188

except:

189

except:

189

continue

190

continue

190

191

if (os.path.splitext(file)[-1].lower() != ext.lower()):

192

if (os.path.splitext(file)[-1].lower() != ext.lower()):

192

continue

193

continue

193

194

validFilelist.append(file)

195

validFilelist.append(file)

195

196

if validFilelist:

197

if validFilelist:

197

validFilelist = sorted( validFilelist, key=str.lower )

198

validFilelist = sorted( validFilelist, key=str.lower )

198

return validFilelist[-1]

199

return validFilelist[-1]

199

200

return None

201

return None

201

202

def checkForRealPath(path, foldercounter, year, doy, set, ext):

203

def checkForRealPath(path, foldercounter, year, doy, set, ext):

203

"""

204

"""

204

Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,

205

Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,

205

Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar

206

Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar

206

el path exacto de un determinado file.

207

el path exacto de un determinado file.

207

208

Example :

209

Example :

209

nombre correcto del file es .../.../D2009307/P2009307367.ext

210

nombre correcto del file es .../.../D2009307/P2009307367.ext

210

211

Entonces la funcion prueba con las siguientes combinaciones

212

Entonces la funcion prueba con las siguientes combinaciones

212

.../.../y2009307367.ext

213

.../.../y2009307367.ext

213

.../.../Y2009307367.ext

214

.../.../Y2009307367.ext

214

.../.../x2009307/y2009307367.ext

215

.../.../x2009307/y2009307367.ext

215

.../.../x2009307/Y2009307367.ext

216

.../.../x2009307/Y2009307367.ext

216

.../.../X2009307/y2009307367.ext

217

.../.../X2009307/y2009307367.ext

217

.../.../X2009307/Y2009307367.ext

218

.../.../X2009307/Y2009307367.ext

218

siendo para este caso, la ultima combinacion de letras, identica al file buscado

219

siendo para este caso, la ultima combinacion de letras, identica al file buscado

219

220

Return:

221

Return:

221

Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file

222

Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file

222

caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas

223

caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas

223

para el filename

224

para el filename

224

"""

225

"""

225

fullfilename = None

226

fullfilename = None

226

find_flag = False

227

find_flag = False

227

filename = None

228

filename = None

228

229

prefixDirList = [None,'d','D']

230

prefixDirList = [None,'d','D']

230

if ext.lower() == ".r": #voltage

231

if ext.lower() == ".r": #voltage

231

prefixFileList = ['d','D']

232

prefixFileList = ['d','D']

232

elif ext.lower() == ".pdata": #spectra

233

elif ext.lower() == ".pdata": #spectra

233

prefixFileList = ['p','P']

234

prefixFileList = ['p','P']

234

else:

235

else:

235

return None, filename

236

return None, filename

236

237

#barrido por las combinaciones posibles

238

#barrido por las combinaciones posibles

238

for prefixDir in prefixDirList:

239

for prefixDir in prefixDirList:

239

thispath = path

240

thispath = path

240

if prefixDir != None:

241

if prefixDir != None:

241

#formo el nombre del directorio xYYYYDDD (x=d o x=D)

242

#formo el nombre del directorio xYYYYDDD (x=d o x=D)

242

if foldercounter == 0:

243

if foldercounter == 0:

243

thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))

244

thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))

244

else:

245

else:

245

thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))

246

thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))

246

for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"

247

for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"

247

filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext

248

filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext

248

fullfilename = os.path.join( thispath, filename ) #formo el path completo

249

fullfilename = os.path.join( thispath, filename ) #formo el path completo

249

250

if os.path.exists( fullfilename ): #verifico que exista

251

if os.path.exists( fullfilename ): #verifico que exista

251

find_flag = True

252

find_flag = True

252

break

253

break

253

if find_flag:

254

if find_flag:

254

break

255

break

255

256

if not(find_flag):

257

if not(find_flag):

257

return None, filename

258

return None, filename

258

259

return fullfilename, filename

260

return fullfilename, filename

260

261

def isDoyFolder(folder):

262

def isDoyFolder(folder):

262

try:

263

try:

263

year = int(folder[1:5])

264

year = int(folder[1:5])

264

except:

265

except:

265

return 0

266

return 0

266

267

try:

268

try:

268

doy = int(folder[5:8])

269

doy = int(folder[5:8])

269

except:

270

except:

270

return 0

271

return 0

271

272

return 1

273

return 1

273

274

class JRODataIO:

275

class JRODataIO:

275

276

c = 3E8

277

c = 3E8

277

278

isConfig = False

279

isConfig = False

279

280

basicHeaderObj = BasicHeader(LOCALTIME)

281

basicHeaderObj = BasicHeader(LOCALTIME)

281

282

systemHeaderObj = SystemHeader()

283

systemHeaderObj = SystemHeader()

283

284

radarControllerHeaderObj = RadarControllerHeader()

285

radarControllerHeaderObj = RadarControllerHeader()

285

286

processingHeaderObj = ProcessingHeader()

287

processingHeaderObj = ProcessingHeader()

287

288

online = 0

289

online = 0

289

290

dtype = None

291

dtype = None

291

292

pathList = []

293

pathList = []

293

294

filenameList = []

295

filenameList = []

295

296

filename = None

297

filename = None

297

298

ext = None

299

ext = None

299

300

flagIsNewFile = 1

301

flagIsNewFile = 1

301

302

flagTimeBlock = 0

303

flagTimeBlock = 0

303

304

flagIsNewBlock = 0

305

flagIsNewBlock = 0

305

306

fp = None

307

fp = None

307

308

firstHeaderSize = 0

309

firstHeaderSize = 0

309

310

basicHeaderSize = 24

311

basicHeaderSize = 24

311

312

versionFile = 1103

313

versionFile = 1103

313

314

fileSize = None

315

fileSize = None

315

316

ippSeconds = None

317

ippSeconds = None

317

318

fileSizeByHeader = None

319

fileSizeByHeader = None

319

320

fileIndex = None

321

fileIndex = None

321

322

profileIndex = None

323

profileIndex = None

323

324

blockIndex = None

325

blockIndex = None

325

326

nTotalBlocks = None

327

nTotalBlocks = None

327

328

maxTimeStep = 30

329

maxTimeStep = 30

329

330

lastUTTime = None

331

lastUTTime = None

331

332

datablock = None

333

datablock = None

333

334

dataOut = None

335

dataOut = None

335

336

blocksize = None

337

blocksize = None

337

338

def __init__(self):

339

def __init__(self):

339

340

raise ValueError, "Not implemented"

341

raise ValueError, "Not implemented"

341

342

def run(self):

343

def run(self):

343

344

raise ValueError, "Not implemented"

345

raise ValueError, "Not implemented"

345

346

def getOutput(self):

347

def getOutput(self):

347

348

return self.dataOut

349

return self.dataOut

349

350

class JRODataReader(JRODataIO, ProcessingUnit):

351

class JRODataReader(JRODataIO, ProcessingUnit):

351

352

nReadBlocks = 0

353

nReadBlocks = 0

353

354

delay = 10 #number of seconds waiting a new file

355

delay = 10 #number of seconds waiting a new file

355

356

nTries = 3 #quantity tries

357

nTries = 3 #quantity tries

357

358

nFiles = 3 #number of files for searching

359

nFiles = 3 #number of files for searching

359

360

path = None

361

path = None

361

362

foldercounter = 0

363

foldercounter = 0

363

364

flagNoMoreFiles = 0

365

flagNoMoreFiles = 0

365

366

datetimeList = []

367

datetimeList = []

367

368

__isFirstTimeOnline = 1

369

__isFirstTimeOnline = 1

369

370

__printInfo = True

371

__printInfo = True

371

372

profileIndex = None

373

profileIndex = None

373

374

def __init__(self):

375

def __init__(self):

375

376

"""

377

"""

377

378

"""

379

"""

379

380

raise ValueError, "This method has not been implemented"

381

raise ValueError, "This method has not been implemented"

381

382

383

def createObjByDefault(self):

384

def createObjByDefault(self):

384

"""

385

"""

385

386

"""

387

"""

387

raise ValueError, "This method has not been implemented"

388

raise ValueError, "This method has not been implemented"

388

389

def getBlockDimension(self):

390

def getBlockDimension(self):

390

391

raise ValueError, "No implemented"

392

raise ValueError, "No implemented"

392

393

def __searchFilesOffLine(self,

394

def __searchFilesOffLine(self,

394

path,

395

path,

395

startDate,

396

startDate,

396

endDate,

397

endDate,

397

startTime=datetime.time(0,0,0),

398

startTime=datetime.time(0,0,0),

398

endTime=datetime.time(23,59,59),

399

endTime=datetime.time(23,59,59),

399

set=None,

400

set=None,

400

expLabel='',

401

expLabel='',

401

ext='.r',

402

ext='.r',

402

walk=True):

403

walk=True):

403

404

pathList = []

405

pathList = []

405

406

if not walk:

407

if not walk:

407

#pathList.append(path)

408

#pathList.append(path)

408

multi_path = path.split(',')

409

multi_path = path.split(',')

409

for single_path in multi_path:

410

for single_path in multi_path:

410

pathList.append(single_path)

411

pathList.append(single_path)

411

412

else:

413

else:

413

#dirList = []

414

#dirList = []

414

multi_path = path.split(',')

415

multi_path = path.split(',')

415

for single_path in multi_path:

416

for single_path in multi_path:

416

dirList = []

417

dirList = []

417

for thisPath in os.listdir(single_path):

418

for thisPath in os.listdir(single_path):

418

if not os.path.isdir(os.path.join(single_path,thisPath)):

419

if not os.path.isdir(os.path.join(single_path,thisPath)):

419

continue

420

continue

420

if not isDoyFolder(thisPath):

421

if not isDoyFolder(thisPath):

421

continue

422

continue

422

423

dirList.append(thisPath)

424

dirList.append(thisPath)

424

425

if not(dirList):

426

if not(dirList):

426

return None, None

427

return None, None

427

428

thisDate = startDate

429

thisDate = startDate

429

430

while(thisDate <= endDate):

431

while(thisDate <= endDate):

431

year = thisDate.timetuple().tm_year

432

year = thisDate.timetuple().tm_year

432

doy = thisDate.timetuple().tm_yday

433

doy = thisDate.timetuple().tm_yday

433

434

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

435

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

435

if len(matchlist) == 0:

436

if len(matchlist) == 0:

436

thisDate += datetime.timedelta(1)

437

thisDate += datetime.timedelta(1)

437

continue

438

continue

438

for match in matchlist:

439

for match in matchlist:

439

pathList.append(os.path.join(single_path,match,expLabel))

440

pathList.append(os.path.join(single_path,match,expLabel))

440

441

thisDate += datetime.timedelta(1)

442

thisDate += datetime.timedelta(1)

442

443

if pathList == []:

444

if pathList == []:

444

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

445

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

445

return None, None

446

return None, None

446

447

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

448

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

448

449

filenameList = []

450

filenameList = []

450

datetimeList = []

451

datetimeList = []

451

pathDict = {}

452

pathDict = {}

452

filenameList_to_sort = []

453

filenameList_to_sort = []

453

454

for i in range(len(pathList)):

455

for i in range(len(pathList)):

455

456

thisPath = pathList[i]

457

thisPath = pathList[i]

457

458

fileList = glob.glob1(thisPath, "*%s" %ext)

459

fileList = glob.glob1(thisPath, "*%s" %ext)

459

fileList.sort()

460

fileList.sort()

460

pathDict.setdefault(fileList[0])

461

pathDict.setdefault(fileList[0])

461

pathDict[fileList[0]] = i

462

pathDict[fileList[0]] = i

462

filenameList_to_sort.append(fileList[0])

463

filenameList_to_sort.append(fileList[0])

463

464

filenameList_to_sort.sort()

465

filenameList_to_sort.sort()

465

466

for file in filenameList_to_sort:

467

for file in filenameList_to_sort:

467

thisPath = pathList[pathDict[file]]

468

thisPath = pathList[pathDict[file]]

468

469

fileList = glob.glob1(thisPath, "*%s" %ext)

470

fileList = glob.glob1(thisPath, "*%s" %ext)

470

fileList.sort()

471

fileList.sort()

471

472

for file in fileList:

473

for file in fileList:

473

474

filename = os.path.join(thisPath,file)

475

filename = os.path.join(thisPath,file)

475

thisDatetime = isFileinThisTime(filename, startTime, endTime)

476

thisDatetime = isFileinThisTime(filename, startTime, endTime)

476

477

if not(thisDatetime):

478

if not(thisDatetime):

478

continue

479

continue

479

480

filenameList.append(filename)

481

filenameList.append(filename)

481

datetimeList.append(thisDatetime)

482

datetimeList.append(thisDatetime)

482

483

if not(filenameList):

484

if not(filenameList):

484

print "Any file was found for the time range %s - %s" %(startTime, endTime)

485

print "Any file was found for the time range %s - %s" %(startTime, endTime)

485

return None, None

486

return None, None

486

487

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

488

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

488

print

489

print

489

490

for i in range(len(filenameList)):

491

for i in range(len(filenameList)):

491

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

492

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

492

493

self.filenameList = filenameList

494

self.filenameList = filenameList

494

self.datetimeList = datetimeList

495

self.datetimeList = datetimeList

495

496

return pathList, filenameList

497

return pathList, filenameList

497

498

def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):

499

def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):

499

500

"""

501

"""

501

Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y

502

Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y

502

devuelve el archivo encontrado ademas de otros datos.

503

devuelve el archivo encontrado ademas de otros datos.

503

504

Input:

505

Input:

505

path : carpeta donde estan contenidos los files que contiene data

506

path : carpeta donde estan contenidos los files que contiene data

506

507

expLabel : Nombre del subexperimento (subfolder)

508

expLabel : Nombre del subexperimento (subfolder)

508

509

ext : extension de los files

510

ext : extension de los files

510

511

walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)

512

walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)

512

513

Return:

514

Return:

514

directory : eL directorio donde esta el file encontrado

515

directory : eL directorio donde esta el file encontrado

515

filename : el ultimo file de una determinada carpeta

516

filename : el ultimo file de una determinada carpeta

516

year : el anho

517

year : el anho

517

doy : el numero de dia del anho

518

doy : el numero de dia del anho

518

set : el set del archivo

519

set : el set del archivo

519

520

521

"""

522

"""

522

dirList = []

523

dirList = []

523

524

if not walk:

525

if not walk:

525

fullpath = path

526

fullpath = path

526

foldercounter = 0

527

foldercounter = 0

527

else:

528

else:

528

#Filtra solo los directorios

529

#Filtra solo los directorios

529

for thisPath in os.listdir(path):

530

for thisPath in os.listdir(path):

530

if not os.path.isdir(os.path.join(path,thisPath)):

531

if not os.path.isdir(os.path.join(path,thisPath)):

531

continue

532

continue

532

if not isDoyFolder(thisPath):

533

if not isDoyFolder(thisPath):

533

continue

534

continue

534

535

dirList.append(thisPath)

536

dirList.append(thisPath)

536

537

if not(dirList):

538

if not(dirList):

538

return None, None, None, None, None, None

539

return None, None, None, None, None, None

539

540

dirList = sorted( dirList, key=str.lower )

541

dirList = sorted( dirList, key=str.lower )

541

542

doypath = dirList[-1]

543

doypath = dirList[-1]

543

foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0

544

foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0

544

fullpath = os.path.join(path, doypath, expLabel)

545

fullpath = os.path.join(path, doypath, expLabel)

545

546

547

print "%s folder was found: " %(fullpath )

548

print "%s folder was found: " %(fullpath )

548

549

if set == None:

550

if set == None:

550

filename = getlastFileFromPath(fullpath, ext)

551

filename = getlastFileFromPath(fullpath, ext)

551

else:

552

else:

552

filename = getFileFromSet(fullpath, ext, set)

553

filename = getFileFromSet(fullpath, ext, set)

553

554

if not(filename):

555

if not(filename):

555

return None, None, None, None, None, None

556

return None, None, None, None, None, None

556

557

print "%s file was found" %(filename)

558

print "%s file was found" %(filename)

558

559

if not(self.__verifyFile(os.path.join(fullpath, filename))):

560

if not(self.__verifyFile(os.path.join(fullpath, filename))):

560

return None, None, None, None, None, None

561

return None, None, None, None, None, None

561

562

year = int( filename[1:5] )

563

year = int( filename[1:5] )

563

doy = int( filename[5:8] )

564

doy = int( filename[5:8] )

564

set = int( filename[8:11] )

565

set = int( filename[8:11] )

565

566

return fullpath, foldercounter, filename, year, doy, set

567

return fullpath, foldercounter, filename, year, doy, set

567

568

def __setNextFileOffline(self):

569

def __setNextFileOffline(self):

569

570

idFile = self.fileIndex

571

idFile = self.fileIndex

571

572

while (True):

573

while (True):

573

idFile += 1

574

idFile += 1

574

if not(idFile < len(self.filenameList)):

575

if not(idFile < len(self.filenameList)):

575

self.flagNoMoreFiles = 1

576

self.flagNoMoreFiles = 1

576

print "No more Files"

577

print "No more Files"

577

return 0

578

return 0

578

579

filename = self.filenameList[idFile]

580

filename = self.filenameList[idFile]

580

581

if not(self.__verifyFile(filename)):

582

if not(self.__verifyFile(filename)):

582

continue

583

continue

583

584

fileSize = os.path.getsize(filename)

585

fileSize = os.path.getsize(filename)

585

fp = open(filename,'rb')

586

fp = open(filename,'rb')

586

break

587

break

587

588

self.flagIsNewFile = 1

589

self.flagIsNewFile = 1

589

self.fileIndex = idFile

590

self.fileIndex = idFile

590

self.filename = filename

591

self.filename = filename

591

self.fileSize = fileSize

592

self.fileSize = fileSize

592

self.fp = fp

593

self.fp = fp

593

594

print "Setting the file: %s"%self.filename

595

print "Setting the file: %s"%self.filename

595

596

return 1

597

return 1

597

598

def __setNextFileOnline(self):

599

def __setNextFileOnline(self):

599

"""

600

"""

600

Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si

601

Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si

601

no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files

602

no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files

602

siguientes.

603

siguientes.

603

604

Affected:

605

Affected:

605

self.flagIsNewFile

606

self.flagIsNewFile

606

self.filename

607

self.filename

607

self.fileSize

608

self.fileSize

608

self.fp

609

self.fp

609

self.set

610

self.set

610

self.flagNoMoreFiles

611

self.flagNoMoreFiles

611

612

Return:

613

Return:

613

0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado

614

0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado

614

1 : si el file fue abierto con exito y esta listo a ser leido

615

1 : si el file fue abierto con exito y esta listo a ser leido

615

616

Excepciones:

617

Excepciones:

617

Si un determinado file no puede ser abierto

618

Si un determinado file no puede ser abierto

618

"""

619

"""

619

nFiles = 0

620

nFiles = 0

620

fileOk_flag = False

621

fileOk_flag = False

621

firstTime_flag = True

622

firstTime_flag = True

622

623

self.set += 1

624

self.set += 1

624

625

if self.set > 999:

626

if self.set > 999:

626

self.set = 0

627

self.set = 0

627

self.foldercounter += 1

628

self.foldercounter += 1

628

629

#busca el 1er file disponible

630

#busca el 1er file disponible

630

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

631

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

631

if fullfilename:

632

if fullfilename:

632

if self.__verifyFile(fullfilename, False):

633

if self.__verifyFile(fullfilename, False):

633

fileOk_flag = True

634

fileOk_flag = True

634

635

#si no encuentra un file entonces espera y vuelve a buscar

636

#si no encuentra un file entonces espera y vuelve a buscar

636

if not(fileOk_flag):

637

if not(fileOk_flag):

637

for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles

638

for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles

638

639

if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces

640

if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces

640

tries = self.nTries

641

tries = self.nTries

641

else:

642

else:

642

tries = 1 #si no es la 1era vez entonces solo lo hace una vez

643

tries = 1 #si no es la 1era vez entonces solo lo hace una vez

643

644

for nTries in range( tries ):

645

for nTries in range( tries ):

645

if firstTime_flag:

646

if firstTime_flag:

646

print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )

647

print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )

647

time.sleep( self.delay )

648

time.sleep( self.delay )

648

else:

649

else:

649

print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)

650

print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)

650

651

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

652

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

652

if fullfilename:

653

if fullfilename:

653

if self.__verifyFile(fullfilename):

654

if self.__verifyFile(fullfilename):

654

fileOk_flag = True

655

fileOk_flag = True

655

break

656

break

656

657

if fileOk_flag:

658

if fileOk_flag:

658

break

659

break

659

660

firstTime_flag = False

661

firstTime_flag = False

661

662

print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename

663

print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename

663

self.set += 1

664

self.set += 1

664

665

if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta

666

if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta

666

self.set = 0

667

self.set = 0

667

self.doy += 1

668

self.doy += 1

668

self.foldercounter = 0

669

self.foldercounter = 0

669

670

if fileOk_flag:

671

if fileOk_flag:

671

self.fileSize = os.path.getsize( fullfilename )

672

self.fileSize = os.path.getsize( fullfilename )

672

self.filename = fullfilename

673

self.filename = fullfilename

673

self.flagIsNewFile = 1

674

self.flagIsNewFile = 1

674

if self.fp != None: self.fp.close()

675

if self.fp != None: self.fp.close()

675

self.fp = open(fullfilename, 'rb')

676

self.fp = open(fullfilename, 'rb')

676

self.flagNoMoreFiles = 0

677

self.flagNoMoreFiles = 0

677

print 'Setting the file: %s' % fullfilename

678

print 'Setting the file: %s' % fullfilename

678

else:

679

else:

679

self.fileSize = 0

680

self.fileSize = 0

680

self.filename = None

681

self.filename = None

681

self.flagIsNewFile = 0

682

self.flagIsNewFile = 0

682

self.fp = None

683

self.fp = None

683

self.flagNoMoreFiles = 1

684

self.flagNoMoreFiles = 1

684

print 'No more Files'

685

print 'No more Files'

685

686

return fileOk_flag

687

return fileOk_flag

687

688

689

def setNextFile(self):

690

def setNextFile(self):

690

if self.fp != None:

691

if self.fp != None:

691

self.fp.close()

692

self.fp.close()

692

693

if self.online:

694

if self.online:

694

newFile = self.__setNextFileOnline()

695

newFile = self.__setNextFileOnline()

695

else:

696

else:

696

newFile = self.__setNextFileOffline()

697

newFile = self.__setNextFileOffline()

697

698

if not(newFile):

699

if not(newFile):

699

return 0

700

return 0

700

701

self.__readFirstHeader()

702

self.__readFirstHeader()

702

self.nReadBlocks = 0

703

self.nReadBlocks = 0

703

return 1

704

return 1

704

705

def __waitNewBlock(self):

706

def __waitNewBlock(self):

706

"""

707

"""

707

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

708

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

708

709

Si el modo de lectura es OffLine siempre retorn 0

710

Si el modo de lectura es OffLine siempre retorn 0

710

"""

711

"""

711

if not self.online:

712

if not self.online:

712

return 0

713

return 0

713

714

if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):

715

if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):

715

return 0

716

return 0

716

717

currentPointer = self.fp.tell()

718

currentPointer = self.fp.tell()

718

719

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

720

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

720

721

for nTries in range( self.nTries ):

722

for nTries in range( self.nTries ):

722

723

self.fp.close()

724

self.fp.close()

724

self.fp = open( self.filename, 'rb' )

725

self.fp = open( self.filename, 'rb' )

725

self.fp.seek( currentPointer )

726

self.fp.seek( currentPointer )

726

727

self.fileSize = os.path.getsize( self.filename )

728

self.fileSize = os.path.getsize( self.filename )

728

currentSize = self.fileSize - currentPointer

729

currentSize = self.fileSize - currentPointer

729

730

if ( currentSize >= neededSize ):

731

if ( currentSize >= neededSize ):

731

self.__rdBasicHeader()

732

self.__rdBasicHeader()

732

return 1

733

return 1

733

734

if self.fileSize == self.fileSizeByHeader:

735

if self.fileSize == self.fileSizeByHeader:

735

# self.flagEoF = True

736

# self.flagEoF = True

736

return 0

737

return 0

737

738

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

739

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

739

time.sleep( self.delay )

740

time.sleep( self.delay )

740

741

742

return 0

743

return 0

743

744

def waitDataBlock(self,pointer_location):

745

def waitDataBlock(self,pointer_location):

745

746

currentPointer = pointer_location

747

currentPointer = pointer_location

747

748

neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize

749

neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize

749

750

for nTries in range( self.nTries ):

751

for nTries in range( self.nTries ):

751

self.fp.close()

752

self.fp.close()

752

self.fp = open( self.filename, 'rb' )

753

self.fp = open( self.filename, 'rb' )

753

self.fp.seek( currentPointer )

754

self.fp.seek( currentPointer )

754

755

self.fileSize = os.path.getsize( self.filename )

756

self.fileSize = os.path.getsize( self.filename )

756

currentSize = self.fileSize - currentPointer

757

currentSize = self.fileSize - currentPointer

757

758

if ( currentSize >= neededSize ):

759

if ( currentSize >= neededSize ):

759

return 1

760

return 1

760

761

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

762

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

762

time.sleep( self.delay )

763

time.sleep( self.delay )

763

764

return 0

765

return 0

765

766

767

def __jumpToLastBlock(self):

768

def __jumpToLastBlock(self):

768

769

if not(self.__isFirstTimeOnline):

770

if not(self.__isFirstTimeOnline):

770

return

771

return

771

772

csize = self.fileSize - self.fp.tell()

773

csize = self.fileSize - self.fp.tell()

773

blocksize = self.processingHeaderObj.blockSize

774

blocksize = self.processingHeaderObj.blockSize

774

775

#salta el primer bloque de datos

776

#salta el primer bloque de datos

776

if csize > self.processingHeaderObj.blockSize:

777

if csize > self.processingHeaderObj.blockSize:

777

self.fp.seek(self.fp.tell() + blocksize)

778

self.fp.seek(self.fp.tell() + blocksize)

778

else:

779

else:

779

return

780

return

780

781

csize = self.fileSize - self.fp.tell()

782

csize = self.fileSize - self.fp.tell()

782

neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

783

neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

783

while True:

784

while True:

784

785

if self.fp.tell()<self.fileSize:

786

if self.fp.tell()<self.fileSize:

786

self.fp.seek(self.fp.tell() + neededsize)

787

self.fp.seek(self.fp.tell() + neededsize)

787

else:

788

else:

788

self.fp.seek(self.fp.tell() - neededsize)

789

self.fp.seek(self.fp.tell() - neededsize)

789

break

790

break

790

791

# csize = self.fileSize - self.fp.tell()

792

# csize = self.fileSize - self.fp.tell()

792

# neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

793

# neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

793

# factor = int(csize/neededsize)

794

# factor = int(csize/neededsize)

794

# if factor > 0:

795

# if factor > 0:

795

# self.fp.seek(self.fp.tell() + factor*neededsize)

796

# self.fp.seek(self.fp.tell() + factor*neededsize)

796

797

self.flagIsNewFile = 0

798

self.flagIsNewFile = 0

798

self.__isFirstTimeOnline = 0

799

self.__isFirstTimeOnline = 0

799

800

801

def __setNewBlock(self):

802

def __setNewBlock(self):

802

803

if self.fp == None:

804

if self.fp == None:

804

return 0

805

return 0

805

806

if self.online:

807

if self.online:

807

self.__jumpToLastBlock()

808

self.__jumpToLastBlock()

808

809

if self.flagIsNewFile:

810

if self.flagIsNewFile:

810

return 1

811

return 1

811

812

self.lastUTTime = self.basicHeaderObj.utc

813

self.lastUTTime = self.basicHeaderObj.utc

813

currentSize = self.fileSize - self.fp.tell()

814

currentSize = self.fileSize - self.fp.tell()

814

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

815

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

815

816

if (currentSize >= neededSize):

817

if (currentSize >= neededSize):

817

self.__rdBasicHeader()

818

self.__rdBasicHeader()

818

return 1

819

return 1

819

820

if self.__waitNewBlock():

821

if self.__waitNewBlock():

821

return 1

822

return 1

822

823

if not(self.setNextFile()):

824

if not(self.setNextFile()):

824

return 0

825

return 0

825

826

deltaTime = self.basicHeaderObj.utc - self.lastUTTime #

827

deltaTime = self.basicHeaderObj.utc - self.lastUTTime #

827

828

self.flagTimeBlock = 0

829

self.flagTimeBlock = 0

829

830

if deltaTime > self.maxTimeStep:

831

if deltaTime > self.maxTimeStep:

831

self.flagTimeBlock = 1

832

self.flagTimeBlock = 1

832

833

return 1

834

return 1

834

835

836

def readNextBlock(self):

837

def readNextBlock(self):

837

if not(self.__setNewBlock()):

838

if not(self.__setNewBlock()):

838

return 0

839

return 0

839

840

if not(self.readBlock()):

841

if not(self.readBlock()):

841

return 0

842

return 0

842

843

return 1

844

return 1

844

845

def __rdProcessingHeader(self, fp=None):

846

def __rdProcessingHeader(self, fp=None):

846

if fp == None:

847

if fp == None:

847

fp = self.fp

848

fp = self.fp

848

849

self.processingHeaderObj.read(fp)

850

self.processingHeaderObj.read(fp)

850

851

def __rdRadarControllerHeader(self, fp=None):

852

def __rdRadarControllerHeader(self, fp=None):

852

if fp == None:

853

if fp == None:

853

fp = self.fp

854

fp = self.fp

854

855

self.radarControllerHeaderObj.read(fp)

856

self.radarControllerHeaderObj.read(fp)

856

857

def __rdSystemHeader(self, fp=None):

858

def __rdSystemHeader(self, fp=None):

858

if fp == None:

859

if fp == None:

859

fp = self.fp

860

fp = self.fp

860

861

self.systemHeaderObj.read(fp)

862

self.systemHeaderObj.read(fp)

862

863

def __rdBasicHeader(self, fp=None):

864

def __rdBasicHeader(self, fp=None):

864

if fp == None:

865

if fp == None:

865

fp = self.fp

866

fp = self.fp

866

867

self.basicHeaderObj.read(fp)

868

self.basicHeaderObj.read(fp)

868

869

870

def __readFirstHeader(self):

871

def __readFirstHeader(self):

871

self.__rdBasicHeader()

872

self.__rdBasicHeader()

872

self.__rdSystemHeader()

873

self.__rdSystemHeader()

873

self.__rdRadarControllerHeader()

874

self.__rdRadarControllerHeader()

874

self.__rdProcessingHeader()

875

self.__rdProcessingHeader()

875

876

self.firstHeaderSize = self.basicHeaderObj.size

877

self.firstHeaderSize = self.basicHeaderObj.size

877

878

datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

879

datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

879

if datatype == 0:

880

if datatype == 0:

880

datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])

881

datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])

881

elif datatype == 1:

882

elif datatype == 1:

882

datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])

883

datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])

883

elif datatype == 2:

884

elif datatype == 2:

884

datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])

885

datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])

885

elif datatype == 3:

886

elif datatype == 3:

886

datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])

887

datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])

887

elif datatype == 4:

888

elif datatype == 4:

888

datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])

889

datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])

889

elif datatype == 5:

890

elif datatype == 5:

890

datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])

891

datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])

891

else:

892

else:

892

raise ValueError, 'Data type was not defined'

893

raise ValueError, 'Data type was not defined'

893

894

self.dtype = datatype_str

895

self.dtype = datatype_str

895

self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c

896

self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c

896

self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)

897

self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)

897

# self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)

898

# self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)

898

# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)

899

# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)

899

self.getBlockDimension()

900

self.getBlockDimension()

900

901

902

def __verifyFile(self, filename, msgFlag=True):

903

def __verifyFile(self, filename, msgFlag=True):

903

msg = None

904

msg = None

904

try:

905

try:

905

fp = open(filename, 'rb')

906

fp = open(filename, 'rb')

906

currentPosition = fp.tell()

907

currentPosition = fp.tell()

907

except:

908

except:

908

if msgFlag:

909

if msgFlag:

909

print "The file %s can't be opened" % (filename)

910

print "The file %s can't be opened" % (filename)

910

return False

911

return False

911

912

neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize

913

neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize

913

914

if neededSize == 0:

915

if neededSize == 0:

915

basicHeaderObj = BasicHeader(LOCALTIME)

916

basicHeaderObj = BasicHeader(LOCALTIME)

916

systemHeaderObj = SystemHeader()

917

systemHeaderObj = SystemHeader()

917

radarControllerHeaderObj = RadarControllerHeader()

918

radarControllerHeaderObj = RadarControllerHeader()

918

processingHeaderObj = ProcessingHeader()

919

processingHeaderObj = ProcessingHeader()

919

920

try:

921

try:

921

if not( basicHeaderObj.read(fp) ): raise IOError

922

if not( basicHeaderObj.read(fp) ): raise IOError

922

if not( systemHeaderObj.read(fp) ): raise IOError

923

if not( systemHeaderObj.read(fp) ): raise IOError

923

if not( radarControllerHeaderObj.read(fp) ): raise IOError

924

if not( radarControllerHeaderObj.read(fp) ): raise IOError

924

if not( processingHeaderObj.read(fp) ): raise IOError

925

if not( processingHeaderObj.read(fp) ): raise IOError

925

data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

926

data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

926

927

neededSize = processingHeaderObj.blockSize + basicHeaderObj.size

928

neededSize = processingHeaderObj.blockSize + basicHeaderObj.size

928

929

except:

930

except:

930

if msgFlag:

931

if msgFlag:

931

print "\tThe file %s is empty or it hasn't enough data" % filename

932

print "\tThe file %s is empty or it hasn't enough data" % filename

932

933

fp.close()

934

fp.close()

934

return False

935

return False

935

else:

936

else:

936

msg = "\tSkipping the file %s due to it hasn't enough data" %filename

937

msg = "\tSkipping the file %s due to it hasn't enough data" %filename

937

938

fp.close()

939

fp.close()

939

fileSize = os.path.getsize(filename)

940

fileSize = os.path.getsize(filename)

940

currentSize = fileSize - currentPosition

941

currentSize = fileSize - currentPosition

941

if currentSize < neededSize:

942

if currentSize < neededSize:

942

if msgFlag and (msg != None):

943

if msgFlag and (msg != None):

943

print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename

944

print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename

944

return False

945

return False

945

946

return True

947

return True

947

948

def setup(self,

949

def setup(self,

949

path=None,

950

path=None,

950

startDate=None,

951

startDate=None,

951

endDate=None,

952

endDate=None,

952

startTime=datetime.time(0,0,0),

953

startTime=datetime.time(0,0,0),

953

endTime=datetime.time(23,59,59),

954

endTime=datetime.time(23,59,59),

954

set=None,

955

set=None,

955

expLabel = "",

956

expLabel = "",

956

ext = None,

957

ext = None,

957

online = False,

958

online = False,

958

delay = 60,

959

delay = 60,

959

walk = True):

960

walk = True):

960

961

if path == None:

962

if path == None:

962

raise ValueError, "The path is not valid"

963

raise ValueError, "The path is not valid"

963

964

if ext == None:

965

if ext == None:

965

ext = self.ext

966

ext = self.ext

966

967

if online:

968

if online:

968

print "Searching files in online mode..."

969

print "Searching files in online mode..."

969

970

for nTries in range( self.nTries ):

971

for nTries in range( self.nTries ):

971

fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)

972

fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)

972

973

if fullpath:

974

if fullpath:

974

break

975

break

975

976

print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)

977

print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)

977

time.sleep( self.delay )

978

time.sleep( self.delay )

978

979

if not(fullpath):

980

if not(fullpath):

980

print "There 'isn't valied files in %s" % path

981

print "There 'isn't valied files in %s" % path

981

return None

982

return None

982

983

self.year = year

984

self.year = year

984

self.doy = doy

985

self.doy = doy

985

self.set = set - 1

986

self.set = set - 1

986

self.path = path

987

self.path = path

987

self.foldercounter = foldercounter

988

self.foldercounter = foldercounter

988

last_set = None

989

last_set = None

989

990

else:

991

else:

991

print "Searching files in offline mode ..."

992

print "Searching files in offline mode ..."

992

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

993

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

993

startTime=startTime, endTime=endTime,

994

startTime=startTime, endTime=endTime,

994

set=set, expLabel=expLabel, ext=ext,

995

set=set, expLabel=expLabel, ext=ext,

995

walk=walk)

996

walk=walk)

996

997

if not(pathList):

998

if not(pathList):

998

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

999

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

999

datetime.datetime.combine(startDate,startTime).ctime(),

1000

datetime.datetime.combine(startDate,startTime).ctime(),

1000

datetime.datetime.combine(endDate,endTime).ctime())

1001

datetime.datetime.combine(endDate,endTime).ctime())

1001

1002

sys.exit(-1)

1003

sys.exit(-1)

1003

1004

1005

self.fileIndex = -1

1006

self.fileIndex = -1

1006

self.pathList = pathList

1007

self.pathList = pathList

1007

self.filenameList = filenameList

1008

self.filenameList = filenameList

1008

file_name = os.path.basename(filenameList[-1])

1009

file_name = os.path.basename(filenameList[-1])

1009

basename, ext = os.path.splitext(file_name)

1010

basename, ext = os.path.splitext(file_name)

1010

last_set = int(basename[-3:])

1011

last_set = int(basename[-3:])

1011

1012

self.online = online

1013

self.online = online

1013

self.delay = delay

1014

self.delay = delay

1014

ext = ext.lower()

1015

ext = ext.lower()

1015

self.ext = ext

1016

self.ext = ext

1016

1017

if not(self.setNextFile()):

1018

if not(self.setNextFile()):

1018

if (startDate!=None) and (endDate!=None):

1019

if (startDate!=None) and (endDate!=None):

1019

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

1020

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

1020

elif startDate != None:

1021

elif startDate != None:

1021

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

1022

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

1022

else:

1023

else:

1023

print "No files"

1024

print "No files"

1024

1025

sys.exit(-1)

1026

sys.exit(-1)

1026

1027

# self.updateDataHeader()

1028

# self.updateDataHeader()

1028

if last_set != None:

1029

if last_set != None:

1029

self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock

1030

self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock

1030

return self.dataOut

1031

return self.dataOut

1031

1032

def getBasicHeader(self):

1033

def getBasicHeader(self):

1033

1034

self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds

1035

self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds

1035

1036

self.dataOut.flagTimeBlock = self.flagTimeBlock

1037

self.dataOut.flagTimeBlock = self.flagTimeBlock

1037

1038

self.dataOut.timeZone = self.basicHeaderObj.timeZone

1039

self.dataOut.timeZone = self.basicHeaderObj.timeZone

1039

1040

self.dataOut.dstFlag = self.basicHeaderObj.dstFlag

1041

self.dataOut.dstFlag = self.basicHeaderObj.dstFlag

1041

1042

self.dataOut.errorCount = self.basicHeaderObj.errorCount

1043

self.dataOut.errorCount = self.basicHeaderObj.errorCount

1043

1044

self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime

1045

self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime

1045

1046

def getFirstHeader(self):

1047

def getFirstHeader(self):

1047

1048

raise ValueError, "This method has not been implemented"

1049

raise ValueError, "This method has not been implemented"

1049

1050

def getData():

1051

def getData():

1051

1052

raise ValueError, "This method has not been implemented"

1053

raise ValueError, "This method has not been implemented"

1053

1054

def hasNotDataInBuffer():

1055

def hasNotDataInBuffer():

1055

1056

raise ValueError, "This method has not been implemented"

1057

raise ValueError, "This method has not been implemented"

1057

1058

def readBlock():

1059

def readBlock():

1059

1060

raise ValueError, "This method has not been implemented"

1061

raise ValueError, "This method has not been implemented"

1061

1062

def isEndProcess(self):

1063

def isEndProcess(self):

1063

1064

return self.flagNoMoreFiles

1065

return self.flagNoMoreFiles

1065

1066

def printReadBlocks(self):

1067

def printReadBlocks(self):

1067

1068

print "Number of read blocks per file %04d" %self.nReadBlocks

1069

print "Number of read blocks per file %04d" %self.nReadBlocks

1069

1070

def printTotalBlocks(self):

1071

def printTotalBlocks(self):

1071

1072

print "Number of read blocks %04d" %self.nTotalBlocks

1073

print "Number of read blocks %04d" %self.nTotalBlocks

1073

1074

def printNumberOfBlock(self):

1075

def printNumberOfBlock(self):

1075

1076

if self.flagIsNewBlock:

1077

if self.flagIsNewBlock:

1077

print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())

1078

print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())

1078

self.dataOut.blocknow = self.basicHeaderObj.dataBlock

1079

self.dataOut.blocknow = self.basicHeaderObj.dataBlock

1079

def printInfo(self):

1080

def printInfo(self):

1080

1081

if self.__printInfo == False:

1082

if self.__printInfo == False:

1082

return

1083

return

1083

1084

self.basicHeaderObj.printInfo()

1085

self.basicHeaderObj.printInfo()

1085

self.systemHeaderObj.printInfo()

1086

self.systemHeaderObj.printInfo()

1086

self.radarControllerHeaderObj.printInfo()

1087

self.radarControllerHeaderObj.printInfo()

1087

self.processingHeaderObj.printInfo()

1088

self.processingHeaderObj.printInfo()

1088

1089

self.__printInfo = False

1090

self.__printInfo = False

1090

1091

1092

def run(self, **kwargs):

1093

def run(self, **kwargs):

1093

1094

if not(self.isConfig):

1095

if not(self.isConfig):

1095

1096

# self.dataOut = dataOut

1097

# self.dataOut = dataOut

1097

self.setup(**kwargs)

1098

self.setup(**kwargs)

1098

self.isConfig = True

1099

self.isConfig = True

1099

1100

self.getData()

1101

self.getData()

1101

1102

class JRODataWriter(JRODataIO, Operation):

1103

class JRODataWriter(JRODataIO, Operation):

1103

1104

"""

1105

"""

1105

Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura

1106

Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura

1106

de los datos siempre se realiza por bloques.

1107

de los datos siempre se realiza por bloques.

1107

"""

1108

"""

1108

1109

blockIndex = 0

1110

blockIndex = 0

1110

1111

path = None

1112

path = None

1112

1113

setFile = None

1114

setFile = None

1114

1115

profilesPerBlock = None

1116

profilesPerBlock = None

1116

1117

blocksPerFile = None

1118

blocksPerFile = None

1118

1119

nWriteBlocks = 0

1120

nWriteBlocks = 0

1120

1121

def __init__(self, dataOut=None):

1122

def __init__(self, dataOut=None):

1122

raise ValueError, "Not implemented"

1123

raise ValueError, "Not implemented"

1123

1124

1125

def hasAllDataInBuffer(self):

1126

def hasAllDataInBuffer(self):

1126

raise ValueError, "Not implemented"

1127

raise ValueError, "Not implemented"

1127

1128

1129

def setBlockDimension(self):

1130

def setBlockDimension(self):

1130

raise ValueError, "Not implemented"

1131

raise ValueError, "Not implemented"

1131

1132

1133

def writeBlock(self):

1134

def writeBlock(self):

1134

raise ValueError, "No implemented"

1135

raise ValueError, "No implemented"

1135

1136

1137

def putData(self):

1138

def putData(self):

1138

raise ValueError, "No implemented"

1139

raise ValueError, "No implemented"

1139

1140

1141

def setBasicHeader(self):

1142

def setBasicHeader(self):

1142

1143

self.basicHeaderObj.size = self.basicHeaderSize #bytes

1144

self.basicHeaderObj.size = self.basicHeaderSize #bytes

1144

self.basicHeaderObj.version = self.versionFile

1145

self.basicHeaderObj.version = self.versionFile

1145

self.basicHeaderObj.dataBlock = self.nTotalBlocks

1146

self.basicHeaderObj.dataBlock = self.nTotalBlocks

1146

1147

utc = numpy.floor(self.dataOut.utctime)

1148

utc = numpy.floor(self.dataOut.utctime)

1148

milisecond = (self.dataOut.utctime - utc)* 1000.0

1149

milisecond = (self.dataOut.utctime - utc)* 1000.0

1149

1150

self.basicHeaderObj.utc = utc

1151

self.basicHeaderObj.utc = utc

1151

self.basicHeaderObj.miliSecond = milisecond

1152

self.basicHeaderObj.miliSecond = milisecond

1152

self.basicHeaderObj.timeZone = self.dataOut.timeZone

1153

self.basicHeaderObj.timeZone = self.dataOut.timeZone

1153

self.basicHeaderObj.dstFlag = self.dataOut.dstFlag

1154

self.basicHeaderObj.dstFlag = self.dataOut.dstFlag

1154

self.basicHeaderObj.errorCount = self.dataOut.errorCount

1155

self.basicHeaderObj.errorCount = self.dataOut.errorCount

1155

1156

def setFirstHeader(self):

1157

def setFirstHeader(self):

1157

"""

1158

"""

1158

Obtiene una copia del First Header

1159

Obtiene una copia del First Header

1159

1160

Affected:

1161

Affected:

1161

1162

self.basicHeaderObj

1163

self.basicHeaderObj

1163

self.systemHeaderObj

1164

self.systemHeaderObj

1164

self.radarControllerHeaderObj

1165

self.radarControllerHeaderObj

1165

self.processingHeaderObj self.

1166

self.processingHeaderObj self.

1166

1167

Return:

1168

Return:

1168

None

1169

None

1169

"""

1170

"""

1170

1171

raise ValueError, "No implemented"

1172

raise ValueError, "No implemented"

1172

1173

def __writeFirstHeader(self):

1174

def __writeFirstHeader(self):

1174

"""

1175

"""

1175

Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)

1176

Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)

1176

1177

Affected:

1178

Affected:

1178

__dataType

1179

__dataType

1179

1180

Return:

1181

Return:

1181

None

1182

None

1182

"""

1183

"""

1183

1184

# CALCULAR PARAMETROS

1185

# CALCULAR PARAMETROS

1185

1186

sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size

1187

sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size

1187

self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader

1188

self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader

1188

1189

self.basicHeaderObj.write(self.fp)

1190

self.basicHeaderObj.write(self.fp)

1190

self.systemHeaderObj.write(self.fp)

1191

self.systemHeaderObj.write(self.fp)

1191

self.radarControllerHeaderObj.write(self.fp)

1192

self.radarControllerHeaderObj.write(self.fp)

1192

self.processingHeaderObj.write(self.fp)

1193

self.processingHeaderObj.write(self.fp)

1193

1194

self.dtype = self.dataOut.dtype

1195

self.dtype = self.dataOut.dtype

1195

1196

def __setNewBlock(self):

1197

def __setNewBlock(self):

1197

"""

1198

"""

1198

Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header

1199

Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header

1199

1200

Return:

1201

Return:

1201

0 : si no pudo escribir nada

1202

0 : si no pudo escribir nada

1202

1 : Si escribio el Basic el First Header

1203

1 : Si escribio el Basic el First Header

1203

"""

1204

"""

1204

if self.fp == None:

1205

if self.fp == None:

1205

self.setNextFile()

1206

self.setNextFile()

1206

1207

if self.flagIsNewFile:

1208

if self.flagIsNewFile:

1208

return 1

1209

return 1

1209

1210

if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:

1211

if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:

1211

self.basicHeaderObj.write(self.fp)

1212

self.basicHeaderObj.write(self.fp)

1212

return 1

1213

return 1

1213

1214

if not( self.setNextFile() ):

1215

if not( self.setNextFile() ):

1215

return 0

1216

return 0

1216

1217

return 1

1218

return 1

1218

1219

1220

def writeNextBlock(self):

1221

def writeNextBlock(self):

1221

"""

1222

"""

1222

Selecciona el bloque siguiente de datos y los escribe en un file

1223

Selecciona el bloque siguiente de datos y los escribe en un file

1223

1224

Return:

1225

Return:

1225

0 : Si no hizo pudo escribir el bloque de datos

1226

0 : Si no hizo pudo escribir el bloque de datos

1226

1 : Si no pudo escribir el bloque de datos

1227

1 : Si no pudo escribir el bloque de datos

1227

"""

1228

"""

1228

if not( self.__setNewBlock() ):

1229

if not( self.__setNewBlock() ):

1229

return 0

1230

return 0

1230

1231

self.writeBlock()

1232

self.writeBlock()

1232

1233

return 1

1234

return 1

1234

1235

def setNextFile(self):

1236

def setNextFile(self):

1236

"""

1237

"""

1237

Determina el siguiente file que sera escrito

1238

Determina el siguiente file que sera escrito

1238

1239

Affected:

1240

Affected:

1240

self.filename

1241

self.filename

1241

self.subfolder

1242

self.subfolder

1242

self.fp

1243

self.fp

1243

self.setFile

1244

self.setFile

1244

self.flagIsNewFile

1245

self.flagIsNewFile

1245

1246

Return:

1247

Return:

1247

0 : Si el archivo no puede ser escrito

1248

0 : Si el archivo no puede ser escrito

1248

1 : Si el archivo esta listo para ser escrito

1249

1 : Si el archivo esta listo para ser escrito

1249

"""

1250

"""

1250

ext = self.ext

1251

ext = self.ext

1251

path = self.path

1252

path = self.path

1252

1253

if self.fp != None:

1254

if self.fp != None:

1254

self.fp.close()

1255

self.fp.close()

1255

1256

timeTuple = time.localtime( self.dataOut.utctime)

1257

timeTuple = time.localtime( self.dataOut.utctime)

1257

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

1258

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

1258

1259

fullpath = os.path.join( path, subfolder )

1260

fullpath = os.path.join( path, subfolder )

1260

if not( os.path.exists(fullpath) ):

1261

if not( os.path.exists(fullpath) ):

1261

os.mkdir(fullpath)

1262

os.mkdir(fullpath)

1262

self.setFile = -1 #inicializo mi contador de seteo

1263

self.setFile = -1 #inicializo mi contador de seteo

1263

else:

1264

else:

1264

filesList = os.listdir( fullpath )

1265

filesList = os.listdir( fullpath )

1265

if len( filesList ) > 0:

1266

if len( filesList ) > 0:

1266

filesList = sorted( filesList, key=str.lower )

1267

filesList = sorted( filesList, key=str.lower )

1267

filen = filesList[-1]

1268

filen = filesList[-1]

1268

# el filename debera tener el siguiente formato

1269

# el filename debera tener el siguiente formato

1269

# 0 1234 567 89A BCDE (hex)

1270

# 0 1234 567 89A BCDE (hex)

1270

# x YYYY DDD SSS .ext

1271

# x YYYY DDD SSS .ext

1271

if isNumber( filen[8:11] ):

1272

if isNumber( filen[8:11] ):

1272

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

1273

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

1273

else:

1274

else:

1274

self.setFile = -1

1275

self.setFile = -1

1275

else:

1276

else:

1276

self.setFile = -1 #inicializo mi contador de seteo

1277

self.setFile = -1 #inicializo mi contador de seteo

1277

1278

setFile = self.setFile

1279

setFile = self.setFile

1279

setFile += 1

1280

setFile += 1

1280

1281

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

1282

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

1282

timeTuple.tm_year,

1283

timeTuple.tm_year,

1283

timeTuple.tm_yday,

1284

timeTuple.tm_yday,

1284

setFile,

1285

setFile,

1285

ext )

1286

ext )

1286

1287

filename = os.path.join( path, subfolder, file )

1288

filename = os.path.join( path, subfolder, file )

1288

1289

fp = open( filename,'wb' )

1290

fp = open( filename,'wb' )

1290

1291

self.blockIndex = 0

1292

self.blockIndex = 0

1292

1293

#guardando atributos

1294

#guardando atributos

1294

self.filename = filename

1295

self.filename = filename

1295

self.subfolder = subfolder

1296

self.subfolder = subfolder

1296

self.fp = fp

1297

self.fp = fp

1297

self.setFile = setFile

1298

self.setFile = setFile

1298

self.flagIsNewFile = 1

1299

self.flagIsNewFile = 1

1299

1300

self.setFirstHeader()

1301

self.setFirstHeader()

1301

1302

print 'Writing the file: %s'%self.filename

1303

print 'Writing the file: %s'%self.filename

1303

1304

self.__writeFirstHeader()

1305

self.__writeFirstHeader()

1305

1306

return 1

1307

return 1

1307

1308

def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):

1309

def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):

1309

"""

1310

"""

1310

Setea el tipo de formato en la cual sera guardada la data y escribe el First Header

1311

Setea el tipo de formato en la cual sera guardada la data y escribe el First Header

1311

1312

Inputs:

1313

Inputs:

1313

path : el path destino en el cual se escribiran los files a crear

1314

path : el path destino en el cual se escribiran los files a crear

1314

format : formato en el cual sera salvado un file

1315

format : formato en el cual sera salvado un file

1315

set : el setebo del file

1316

set : el setebo del file

1316

1317

Return:

1318

Return:

1318

0 : Si no realizo un buen seteo

1319

0 : Si no realizo un buen seteo

1319

1 : Si realizo un buen seteo

1320

1 : Si realizo un buen seteo

1320

"""

1321

"""

1321

1322

if ext == None:

1323

if ext == None:

1323

ext = self.ext

1324

ext = self.ext

1324

1325

ext = ext.lower()

1326

ext = ext.lower()

1326

1327

self.ext = ext

1328

self.ext = ext

1328

1329

self.path = path

1330

self.path = path

1330

1331

self.setFile = set - 1

1332

self.setFile = set - 1

1332

1333

self.blocksPerFile = blocksPerFile

1334

self.blocksPerFile = blocksPerFile

1334

1335

self.profilesPerBlock = profilesPerBlock

1336

self.profilesPerBlock = profilesPerBlock

1336

1337

self.dataOut = dataOut

1338

self.dataOut = dataOut

1338

1339

if not(self.setNextFile()):

1340

if not(self.setNextFile()):

1340

print "There isn't a next file"

1341

print "There isn't a next file"

1341

return 0

1342

return 0

1342

1343

self.setBlockDimension()

1344

self.setBlockDimension()

1344

1345

return 1

1346

return 1

1346

1347

def run(self, dataOut, **kwargs):

1348

def run(self, dataOut, **kwargs):

1348

1349

if not(self.isConfig):

1350

if not(self.isConfig):

1350

1351

self.setup(dataOut, **kwargs)

1352

self.setup(dataOut, **kwargs)

1352

self.isConfig = True

1353

self.isConfig = True

1353

1354

self.putData()

1355

self.putData()

1355

1356

class VoltageReader(JRODataReader):

1357

class VoltageReader(JRODataReader):

1357

"""

1358

"""

1358

Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura

1359

Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura

1359

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:

1360

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:

1360

perfiles*alturas*canales) son almacenados en la variable "buffer".

1361

perfiles*alturas*canales) son almacenados en la variable "buffer".

1361

1362

perfiles * alturas * canales

1363

perfiles * alturas * canales

1363

1364

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1365

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1365

RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la

1366

RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la

1366

cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de

1367

cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de

1367

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1368

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1368

1369

Example:

1370

Example:

1370

1371

dpath = "/home/myuser/data"

1372

dpath = "/home/myuser/data"

1372

1373

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1374

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1374

1375

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1376

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1376

1377

readerObj = VoltageReader()

1378

readerObj = VoltageReader()

1378

1379

readerObj.setup(dpath, startTime, endTime)

1380

readerObj.setup(dpath, startTime, endTime)

1380

1381

while(True):

1382

while(True):

1382

1383

#to get one profile

1384

#to get one profile

1384

profile = readerObj.getData()

1385

profile = readerObj.getData()

1385

1386

#print the profile

1387

#print the profile

1387

print profile

1388

print profile

1388

1389

#If you want to see all datablock

1390

#If you want to see all datablock

1390

print readerObj.datablock

1391

print readerObj.datablock

1391

1392

if readerObj.flagNoMoreFiles:

1393

if readerObj.flagNoMoreFiles:

1393

break

1394

break

1394

1395

"""

1396

"""

1396

1397

ext = ".r"

1398

ext = ".r"

1398

1399

optchar = "D"

1400

optchar = "D"

1400

dataOut = None

1401

dataOut = None

1401

1402

1403

def __init__(self):

1404

def __init__(self):

1404

"""

1405

"""

1405

Inicializador de la clase VoltageReader para la lectura de datos de voltage.

1406

Inicializador de la clase VoltageReader para la lectura de datos de voltage.

1406

1407

Input:

1408

Input:

1408

dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para

1409

dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para

1409

almacenar un perfil de datos cada vez que se haga un requerimiento

1410

almacenar un perfil de datos cada vez que se haga un requerimiento

1410

(getData). El perfil sera obtenido a partir del buffer de datos,

1411

(getData). El perfil sera obtenido a partir del buffer de datos,

1411

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1412

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1412

bloque de datos.

1413

bloque de datos.

1413

Si este parametro no es pasado se creara uno internamente.

1414

Si este parametro no es pasado se creara uno internamente.

1414

1415

Variables afectadas:

1416

Variables afectadas:

1416

self.dataOut

1417

self.dataOut

1417

1418

Return:

1419

Return:

1419

None

1420

None

1420

"""

1421

"""

1421

1422

self.isConfig = False

1423

self.isConfig = False

1423

1424

self.datablock = None

1425

self.datablock = None

1425

1426

self.utc = 0

1427

self.utc = 0

1427

1428

self.ext = ".r"

1429

self.ext = ".r"

1429

1430

self.optchar = "D"

1431

self.optchar = "D"

1431

1432

self.basicHeaderObj = BasicHeader(LOCALTIME)

1433

self.basicHeaderObj = BasicHeader(LOCALTIME)

1433

1434

self.systemHeaderObj = SystemHeader()

1435

self.systemHeaderObj = SystemHeader()

1435

1436

self.radarControllerHeaderObj = RadarControllerHeader()

1437

self.radarControllerHeaderObj = RadarControllerHeader()

1437

1438

self.processingHeaderObj = ProcessingHeader()

1439

self.processingHeaderObj = ProcessingHeader()

1439

1440

self.online = 0

1441

self.online = 0

1441

1442

self.fp = None

1443

self.fp = None

1443

1444

self.idFile = None

1445

self.idFile = None

1445

1446

self.dtype = None

1447

self.dtype = None

1447

1448

self.fileSizeByHeader = None

1449

self.fileSizeByHeader = None

1449

1450

self.filenameList = []

1451

self.filenameList = []

1451

1452

self.filename = None

1453

self.filename = None

1453

1454

self.fileSize = None

1455

self.fileSize = None

1455

1456

self.firstHeaderSize = 0

1457

self.firstHeaderSize = 0

1457

1458

self.basicHeaderSize = 24

1459

self.basicHeaderSize = 24

1459

1460

self.pathList = []

1461

self.pathList = []

1461

1462

self.filenameList = []

1463

self.filenameList = []

1463

1464

self.lastUTTime = 0

1465

self.lastUTTime = 0

1465

1466

self.maxTimeStep = 30

1467

self.maxTimeStep = 30

1467

1468

self.flagNoMoreFiles = 0

1469

self.flagNoMoreFiles = 0

1469

1470

self.set = 0

1471

self.set = 0

1471

1472

self.path = None

1473

self.path = None

1473

1474

self.profileIndex = 2**32-1

1475

self.profileIndex = 2**32-1

1475

1476

self.delay = 3 #seconds

1477

self.delay = 3 #seconds

1477

1478

self.nTries = 3 #quantity tries

1479

self.nTries = 3 #quantity tries

1479

1480

self.nFiles = 3 #number of files for searching

1481

self.nFiles = 3 #number of files for searching

1481

1482

self.nReadBlocks = 0

1483

self.nReadBlocks = 0

1483

1484

self.flagIsNewFile = 1

1485

self.flagIsNewFile = 1

1485

1486

self.__isFirstTimeOnline = 1

1487

self.__isFirstTimeOnline = 1

1487

1488

self.ippSeconds = 0

1489

self.ippSeconds = 0

1489

1490

self.flagTimeBlock = 0

1491

self.flagTimeBlock = 0

1491

1492

self.flagIsNewBlock = 0

1493

self.flagIsNewBlock = 0

1493

1494

self.nTotalBlocks = 0

1495

self.nTotalBlocks = 0

1495

1496

self.blocksize = 0

1497

self.blocksize = 0

1497

1498

self.dataOut = self.createObjByDefault()

1499

self.dataOut = self.createObjByDefault()

1499

1500

def createObjByDefault(self):

1501

def createObjByDefault(self):

1501

1502

dataObj = Voltage()

1503

dataObj = Voltage()

1503

1504

return dataObj

1505

return dataObj

1505

1506

def __hasNotDataInBuffer(self):

1507

def __hasNotDataInBuffer(self):

1507

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1508

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1508

return 1

1509

return 1

1509

return 0

1510

return 0

1510

1511

1512

def getBlockDimension(self):

1513

def getBlockDimension(self):

1513

"""

1514

"""

1514

Obtiene la cantidad de puntos a leer por cada bloque de datos

1515

Obtiene la cantidad de puntos a leer por cada bloque de datos

1515

1516

Affected:

1517

Affected:

1517

self.blocksize

1518

self.blocksize

1518

1519

Return:

1520

Return:

1520

None

1521

None

1521

"""

1522

"""

1522

pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels

1523

pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels

1523

self.blocksize = pts2read

1524

self.blocksize = pts2read

1524

1525

1526

def readBlock(self):

1527

def readBlock(self):

1527

"""

1528

"""

1528

readBlock lee el bloque de datos desde la posicion actual del puntero del archivo

1529

readBlock lee el bloque de datos desde la posicion actual del puntero del archivo

1529

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

1530

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

1530

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

1531

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

1531

es seteado a 0

1532

es seteado a 0

1532

1533

Inputs:

1534

Inputs:

1534

None

1535

None

1535

1536

Return:

1537

Return:

1537

None

1538

None

1538

1539

Affected:

1540

Affected:

1540

self.profileIndex

1541

self.profileIndex

1541

self.datablock

1542

self.datablock

1542

self.flagIsNewFile

1543

self.flagIsNewFile

1543

self.flagIsNewBlock

1544

self.flagIsNewBlock

1544

self.nTotalBlocks

1545

self.nTotalBlocks

1545

1546

Exceptions:

1547

Exceptions:

1547

Si un bloque leido no es un bloque valido

1548

Si un bloque leido no es un bloque valido

1548

"""

1549

"""

1549

current_pointer_location = self.fp.tell()

1550

current_pointer_location = self.fp.tell()

1550

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1551

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1551

1552

try:

1553

try:

1553

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1554

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1554

except:

1555

except:

1555

#print "The read block (%3d) has not enough data" %self.nReadBlocks

1556

#print "The read block (%3d) has not enough data" %self.nReadBlocks

1556

1557

if self.waitDataBlock(pointer_location=current_pointer_location):

1558

if self.waitDataBlock(pointer_location=current_pointer_location):

1558

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1559

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1559

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1560

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1560

# return 0

1561

# return 0

1561

1562

junk = numpy.transpose(junk, (2,0,1))

1563

junk = numpy.transpose(junk, (2,0,1))

1563

self.datablock = junk['real'] + junk['imag']*1j

1564

self.datablock = junk['real'] + junk['imag']*1j

1564

1565

self.profileIndex = 0

1566

self.profileIndex = 0

1566

1567

self.flagIsNewFile = 0

1568

self.flagIsNewFile = 0

1568

self.flagIsNewBlock = 1

1569

self.flagIsNewBlock = 1

1569

1570

self.nTotalBlocks += 1

1571

self.nTotalBlocks += 1

1571

self.nReadBlocks += 1

1572

self.nReadBlocks += 1

1572

1573

return 1

1574

return 1

1574

1575

def getFirstHeader(self):

1576

def getFirstHeader(self):

1576

1577

self.dataOut.dtype = self.dtype

1578

self.dataOut.dtype = self.dtype

1578

1579

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

1580

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

1580

1581

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

1582

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

1582

1583

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

1584

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

1584

1585

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

1586

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

1586

1587

self.dataOut.ippSeconds = self.ippSeconds

1588

self.dataOut.ippSeconds = self.ippSeconds

1588

1589

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt

1590

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt

1590

1591

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

1592

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

1592

1593

self.dataOut.flagShiftFFT = False

1594

self.dataOut.flagShiftFFT = False

1594

1595

if self.radarControllerHeaderObj.code != None:

1596

if self.radarControllerHeaderObj.code != None:

1596

1597

self.dataOut.nCode = self.radarControllerHeaderObj.nCode

1598

self.dataOut.nCode = self.radarControllerHeaderObj.nCode

1598

1599

self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud

1600

self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud

1600

1601

self.dataOut.code = self.radarControllerHeaderObj.code

1602

self.dataOut.code = self.radarControllerHeaderObj.code

1602

1603

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

1604

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

1604

1605

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

1606

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

1606

1607

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

1608

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

1608

1609

self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip

1610

self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip

1610

1611

self.dataOut.flagShiftFFT = False

1612

self.dataOut.flagShiftFFT = False

1612

1613

def getData(self):

1614

def getData(self):

1614

"""

1615

"""

1615

getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"

1616

getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"

1616

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

1617

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

1617

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

1618

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

1618

1619

Ademas incrementa el contador del buffer en 1.

1620

Ademas incrementa el contador del buffer en 1.

1620

1621

Return:

1622

Return:

1622

data : retorna un perfil de voltages (alturas * canales) copiados desde el

1623

data : retorna un perfil de voltages (alturas * canales) copiados desde el

1623

buffer. Si no hay mas archivos a leer retorna None.

1624

buffer. Si no hay mas archivos a leer retorna None.

1624

1625

Variables afectadas:

1626

Variables afectadas:

1626

self.dataOut

1627

self.dataOut

1627

self.profileIndex

1628

self.profileIndex

1628

1629

Affected:

1630

Affected:

1630

self.dataOut

1631

self.dataOut

1631

self.profileIndex

1632

self.profileIndex

1632

self.flagTimeBlock

1633

self.flagTimeBlock

1633

self.flagIsNewBlock

1634

self.flagIsNewBlock

1634

"""

1635

"""

1635

1636

if self.flagNoMoreFiles:

1637

if self.flagNoMoreFiles:

1637

self.dataOut.flagNoData = True

1638

self.dataOut.flagNoData = True

1638

print 'Process finished'

1639

print 'Process finished'

1639

return 0

1640

return 0

1640

1641

self.flagTimeBlock = 0

1642

self.flagTimeBlock = 0

1642

self.flagIsNewBlock = 0

1643

self.flagIsNewBlock = 0

1643

1644

if self.__hasNotDataInBuffer():

1645

if self.__hasNotDataInBuffer():

1645

1646

if not( self.readNextBlock() ):

1647

if not( self.readNextBlock() ):

1647

return 0

1648

return 0

1648

1649

self.getFirstHeader()

1650

self.getFirstHeader()

1650

1651

if self.datablock == None:

1652

if self.datablock == None:

1652

self.dataOut.flagNoData = True

1653

self.dataOut.flagNoData = True

1653

return 0

1654

return 0

1654

1655

self.dataOut.data = self.datablock[:,self.profileIndex,:]

1656

self.dataOut.data = self.datablock[:,self.profileIndex,:]

1656

1657

self.dataOut.flagNoData = False

1658

self.dataOut.flagNoData = False

1658

1659

self.getBasicHeader()

1660

self.getBasicHeader()

1660

1661

self.profileIndex += 1

1662

self.profileIndex += 1

1662

1663

self.dataOut.realtime = self.online

1664

self.dataOut.realtime = self.online

1664

1665

return self.dataOut.data

1666

return self.dataOut.data

1666

1667

1668

class VoltageWriter(JRODataWriter):

1669

class VoltageWriter(JRODataWriter):

1669

"""

1670

"""

1670

Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura

1671

Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura

1671

de los datos siempre se realiza por bloques.

1672

de los datos siempre se realiza por bloques.

1672

"""

1673

"""

1673

1674

ext = ".r"

1675

ext = ".r"

1675

1676

optchar = "D"

1677

optchar = "D"

1677

1678

shapeBuffer = None

1679

shapeBuffer = None

1679

1680

1681

def __init__(self):

1682

def __init__(self):

1682

"""

1683

"""

1683

Inicializador de la clase VoltageWriter para la escritura de datos de espectros.

1684

Inicializador de la clase VoltageWriter para la escritura de datos de espectros.

1684

1685

Affected:

1686

Affected:

1686

self.dataOut

1687

self.dataOut

1687

1688

Return: None

1689

Return: None

1689

"""

1690

"""

1690

1691

self.nTotalBlocks = 0

1692

self.nTotalBlocks = 0

1692

1693

self.profileIndex = 0

1694

self.profileIndex = 0

1694

1695

self.isConfig = False

1696

self.isConfig = False

1696

1697

self.fp = None

1698

self.fp = None

1698

1699

self.flagIsNewFile = 1

1700

self.flagIsNewFile = 1

1700

1701

self.nTotalBlocks = 0

1702

self.nTotalBlocks = 0

1702

1703

self.flagIsNewBlock = 0

1704

self.flagIsNewBlock = 0

1704

1705

self.setFile = None

1706

self.setFile = None

1706

1707

self.dtype = None

1708

self.dtype = None

1708

1709

self.path = None

1710

self.path = None

1710

1711

self.filename = None

1712

self.filename = None

1712

1713

self.basicHeaderObj = BasicHeader(LOCALTIME)

1714

self.basicHeaderObj = BasicHeader(LOCALTIME)

1714

1715

self.systemHeaderObj = SystemHeader()

1716

self.systemHeaderObj = SystemHeader()

1716

1717

self.radarControllerHeaderObj = RadarControllerHeader()

1718

self.radarControllerHeaderObj = RadarControllerHeader()

1718

1719

self.processingHeaderObj = ProcessingHeader()

1720

self.processingHeaderObj = ProcessingHeader()

1720

1721

def hasAllDataInBuffer(self):

1722

def hasAllDataInBuffer(self):

1722

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1723

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1723

return 1

1724

return 1

1724

return 0

1725

return 0

1725

1726

1727

def setBlockDimension(self):

1728

def setBlockDimension(self):

1728

"""

1729

"""

1729

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

1730

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

1730

1731

Affected:

1732

Affected:

1732

self.shape_spc_Buffer

1733

self.shape_spc_Buffer

1733

self.shape_cspc_Buffer

1734

self.shape_cspc_Buffer

1734

self.shape_dc_Buffer

1735

self.shape_dc_Buffer

1735

1736

Return: None

1737

Return: None

1737

"""

1738

"""

1738

self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,

1739

self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,

1739

self.processingHeaderObj.nHeights,

1740

self.processingHeaderObj.nHeights,

1740

self.systemHeaderObj.nChannels)

1741

self.systemHeaderObj.nChannels)

1741

1742

self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,

1743

self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,

1743

self.processingHeaderObj.profilesPerBlock,

1744

self.processingHeaderObj.profilesPerBlock,

1744

self.processingHeaderObj.nHeights),

1745

self.processingHeaderObj.nHeights),

1745

dtype=numpy.dtype('complex64'))

1746

dtype=numpy.dtype('complex64'))

1746

1747

1748

def writeBlock(self):

1749

def writeBlock(self):

1749

"""

1750

"""

1750

Escribe el buffer en el file designado

1751

Escribe el buffer en el file designado

1751

1752

Affected:

1753

Affected:

1753

self.profileIndex

1754

self.profileIndex

1754

self.flagIsNewFile

1755

self.flagIsNewFile

1755

self.flagIsNewBlock

1756

self.flagIsNewBlock

1756

self.nTotalBlocks

1757

self.nTotalBlocks

1757

self.blockIndex

1758

self.blockIndex

1758

1759

Return: None

1760

Return: None

1760

"""

1761

"""

1761

data = numpy.zeros( self.shapeBuffer, self.dtype )

1762

data = numpy.zeros( self.shapeBuffer, self.dtype )

1762

1763

junk = numpy.transpose(self.datablock, (1,2,0))

1764

junk = numpy.transpose(self.datablock, (1,2,0))

1764

1765

data['real'] = junk.real

1766

data['real'] = junk.real

1766

data['imag'] = junk.imag

1767

data['imag'] = junk.imag

1767

1768

data = data.reshape( (-1) )

1769

data = data.reshape( (-1) )

1769

1770

data.tofile( self.fp )

1771

data.tofile( self.fp )

1771

1772

self.datablock.fill(0)

1773

self.datablock.fill(0)

1773

1774

self.profileIndex = 0

1775

self.profileIndex = 0

1775

self.flagIsNewFile = 0

1776

self.flagIsNewFile = 0

1776

self.flagIsNewBlock = 1

1777

self.flagIsNewBlock = 1

1777

1778

self.blockIndex += 1

1779

self.blockIndex += 1

1779

self.nTotalBlocks += 1

1780

self.nTotalBlocks += 1

1780

1781

def putData(self):

1782

def putData(self):

1782

"""

1783

"""

1783

Setea un bloque de datos y luego los escribe en un file

1784

Setea un bloque de datos y luego los escribe en un file

1784

1785

Affected:

1786

Affected:

1786

self.flagIsNewBlock

1787

self.flagIsNewBlock

1787

self.profileIndex

1788

self.profileIndex

1788

1789

Return:

1790

Return:

1790

0 : Si no hay data o no hay mas files que puedan escribirse

1791

0 : Si no hay data o no hay mas files que puedan escribirse

1791

1 : Si se escribio la data de un bloque en un file

1792

1 : Si se escribio la data de un bloque en un file

1792

"""

1793

"""

1793

if self.dataOut.flagNoData:

1794

if self.dataOut.flagNoData:

1794

return 0

1795

return 0

1795

1796

self.flagIsNewBlock = 0

1797

self.flagIsNewBlock = 0

1797

1798

if self.dataOut.flagTimeBlock:

1799

if self.dataOut.flagTimeBlock:

1799

1800

self.datablock.fill(0)

1801

self.datablock.fill(0)

1801

self.profileIndex = 0

1802

self.profileIndex = 0

1802

self.setNextFile()

1803

self.setNextFile()

1803

1804

if self.profileIndex == 0:

1805

if self.profileIndex == 0:

1805

self.setBasicHeader()

1806

self.setBasicHeader()

1806

1807

self.datablock[:,self.profileIndex,:] = self.dataOut.data

1808

self.datablock[:,self.profileIndex,:] = self.dataOut.data

1808

1809

self.profileIndex += 1

1810

self.profileIndex += 1

1810

1811

if self.hasAllDataInBuffer():

1812

if self.hasAllDataInBuffer():

1812

#if self.flagIsNewFile:

1813

#if self.flagIsNewFile:

1813

self.writeNextBlock()

1814

self.writeNextBlock()

1814

# self.setFirstHeader()

1815

# self.setFirstHeader()

1815

1816

return 1

1817

return 1

1817

1818

def __getProcessFlags(self):

1819

def __getProcessFlags(self):

1819

1820

processFlags = 0

1821

processFlags = 0

1821

1822

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1823

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1823

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1824

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1824

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1825

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1825

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1826

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1826

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1827

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1827

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1828

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1828

1829

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1830

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1830

1831

1832

1833

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

1834

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

1834

PROCFLAG.DATATYPE_SHORT,

1835

PROCFLAG.DATATYPE_SHORT,

1835

PROCFLAG.DATATYPE_LONG,

1836

PROCFLAG.DATATYPE_LONG,

1836

PROCFLAG.DATATYPE_INT64,

1837

PROCFLAG.DATATYPE_INT64,

1837

PROCFLAG.DATATYPE_FLOAT,

1838

PROCFLAG.DATATYPE_FLOAT,

1838

PROCFLAG.DATATYPE_DOUBLE]

1839

PROCFLAG.DATATYPE_DOUBLE]

1839

1840

1841

for index in range(len(dtypeList)):

1842

for index in range(len(dtypeList)):

1842

if self.dataOut.dtype == dtypeList[index]:

1843

if self.dataOut.dtype == dtypeList[index]:

1843

dtypeValue = datatypeValueList[index]

1844

dtypeValue = datatypeValueList[index]

1844

break

1845

break

1845

1846

processFlags += dtypeValue

1847

processFlags += dtypeValue

1847

1848

if self.dataOut.flagDecodeData:

1849

if self.dataOut.flagDecodeData:

1849

processFlags += PROCFLAG.DECODE_DATA

1850

processFlags += PROCFLAG.DECODE_DATA

1850

1851

if self.dataOut.flagDeflipData:

1852

if self.dataOut.flagDeflipData:

1852

processFlags += PROCFLAG.DEFLIP_DATA

1853

processFlags += PROCFLAG.DEFLIP_DATA

1853

1854

if self.dataOut.code != None:

1855

if self.dataOut.code != None:

1855

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

1856

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

1856

1857

if self.dataOut.nCohInt > 1:

1858

if self.dataOut.nCohInt > 1:

1858

processFlags += PROCFLAG.COHERENT_INTEGRATION

1859

processFlags += PROCFLAG.COHERENT_INTEGRATION

1859

1860

return processFlags

1861

return processFlags

1861

1862

1863

def __getBlockSize(self):

1864

def __getBlockSize(self):

1864

'''

1865

'''

1865

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage

1866

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage

1866

'''

1867

'''

1867

1868

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1869

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1869

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1870

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1870

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1871

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1871

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1872

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1872

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1873

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1873

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1874

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1874

1875

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1876

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1876

datatypeValueList = [1,2,4,8,4,8]

1877

datatypeValueList = [1,2,4,8,4,8]

1877

for index in range(len(dtypeList)):

1878

for index in range(len(dtypeList)):

1878

if self.dataOut.dtype == dtypeList[index]:

1879

if self.dataOut.dtype == dtypeList[index]:

1879

datatypeValue = datatypeValueList[index]

1880

datatypeValue = datatypeValueList[index]

1880

break

1881

break

1881

1882

blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * datatypeValue * 2)

1883

blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * datatypeValue * 2)

1883

1884

return blocksize

1885

return blocksize

1885

1886

def setFirstHeader(self):

1887

def setFirstHeader(self):

1887

1888

"""

1889

"""

1889

Obtiene una copia del First Header

1890

Obtiene una copia del First Header

1890

1891

Affected:

1892

Affected:

1892

self.systemHeaderObj

1893

self.systemHeaderObj

1893

self.radarControllerHeaderObj

1894

self.radarControllerHeaderObj

1894

self.dtype

1895

self.dtype

1895

1896

Return:

1897

Return:

1897

None

1898

None

1898

"""

1899

"""

1899

1900

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

1901

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

1901

self.systemHeaderObj.nChannels = self.dataOut.nChannels

1902

self.systemHeaderObj.nChannels = self.dataOut.nChannels

1902

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

1903

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

1903

1904

self.setBasicHeader()

1905

self.setBasicHeader()

1905

1906

processingHeaderSize = 40 # bytes

1907

processingHeaderSize = 40 # bytes

1907

self.processingHeaderObj.dtype = 0 # Voltage

1908

self.processingHeaderObj.dtype = 0 # Voltage

1908

self.processingHeaderObj.blockSize = self.__getBlockSize()

1909

self.processingHeaderObj.blockSize = self.__getBlockSize()

1909

self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock

1910

self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock

1910

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

1911

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

1911

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

1912

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

1912

self.processingHeaderObj.processFlags = self.__getProcessFlags()

1913

self.processingHeaderObj.processFlags = self.__getProcessFlags()

1913

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt

1914

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt

1914

self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage

1915

self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage

1915

self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage

1916

self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage

1916

1917

# if self.dataOut.code != None:

1918

# if self.dataOut.code != None:

1918

# self.processingHeaderObj.code = self.dataOut.code

1919

# self.processingHeaderObj.code = self.dataOut.code

1919

# self.processingHeaderObj.nCode = self.dataOut.nCode

1920

# self.processingHeaderObj.nCode = self.dataOut.nCode

1920

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

1921

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

1921

# codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)

1922

# codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)

1922

# processingHeaderSize += codesize

1923

# processingHeaderSize += codesize

1923

1924

if self.processingHeaderObj.nWindows != 0:

1925

if self.processingHeaderObj.nWindows != 0:

1925

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

1926

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

1926

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

1927

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

1927

self.processingHeaderObj.nHeights = self.dataOut.nHeights

1928

self.processingHeaderObj.nHeights = self.dataOut.nHeights

1928

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

1929

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

1929

processingHeaderSize += 12

1930

processingHeaderSize += 12

1930

1931

self.processingHeaderObj.size = processingHeaderSize

1932

self.processingHeaderObj.size = processingHeaderSize

1932

1933

class SpectraReader(JRODataReader):

1934

class SpectraReader(JRODataReader):

1934

"""

1935

"""

1935

Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura

1936

Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura

1936

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)

1937

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)

1937

son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.

1938

son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.

1938

1939

paresCanalesIguales * alturas * perfiles (Self Spectra)

1940

paresCanalesIguales * alturas * perfiles (Self Spectra)

1940

paresCanalesDiferentes * alturas * perfiles (Cross Spectra)

1941

paresCanalesDiferentes * alturas * perfiles (Cross Spectra)

1941

canales * alturas (DC Channels)

1942

canales * alturas (DC Channels)

1942

1943

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1944

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1944

RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la

1945

RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la

1945

cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de

1946

cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de

1946

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1947

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1947

1948

Example:

1949

Example:

1949

dpath = "/home/myuser/data"

1950

dpath = "/home/myuser/data"

1950

1951

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1952

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1952

1953

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1954

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1954

1955

readerObj = SpectraReader()

1956

readerObj = SpectraReader()

1956

1957

readerObj.setup(dpath, startTime, endTime)

1958

readerObj.setup(dpath, startTime, endTime)

1958

1959

while(True):

1960

while(True):

1960

1961

readerObj.getData()

1962

readerObj.getData()

1962

1963

print readerObj.data_spc

1964

print readerObj.data_spc

1964

1965

print readerObj.data_cspc

1966

print readerObj.data_cspc

1966

1967

print readerObj.data_dc

1968

print readerObj.data_dc

1968

1969

if readerObj.flagNoMoreFiles:

1970

if readerObj.flagNoMoreFiles:

1970

break

1971

break

1971

1972

"""

1973

"""

1973

1974

pts2read_SelfSpectra = 0

1975

pts2read_SelfSpectra = 0

1975

1976

pts2read_CrossSpectra = 0

1977

pts2read_CrossSpectra = 0

1977

1978

pts2read_DCchannels = 0

1979

pts2read_DCchannels = 0

1979

1980

ext = ".pdata"

1981

ext = ".pdata"

1981

1982

optchar = "P"

1983

optchar = "P"

1983

1984

dataOut = None

1985

dataOut = None

1985

1986

nRdChannels = None

1987

nRdChannels = None

1987

1988

nRdPairs = None

1989

nRdPairs = None

1989

1990

rdPairList = []

1991

rdPairList = []

1991

1992

def __init__(self):

1993

def __init__(self):

1993

"""

1994

"""

1994

Inicializador de la clase SpectraReader para la lectura de datos de espectros.

1995

Inicializador de la clase SpectraReader para la lectura de datos de espectros.

1995

1996

Inputs:

1997

Inputs:

1997

dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para

1998

dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para

1998

almacenar un perfil de datos cada vez que se haga un requerimiento

1999

almacenar un perfil de datos cada vez que se haga un requerimiento

1999

(getData). El perfil sera obtenido a partir del buffer de datos,

2000

(getData). El perfil sera obtenido a partir del buffer de datos,

2000

si el buffer esta vacio se hara un nuevo proceso de lectura de un

2001

si el buffer esta vacio se hara un nuevo proceso de lectura de un

2001

bloque de datos.

2002

bloque de datos.

2002

Si este parametro no es pasado se creara uno internamente.

2003

Si este parametro no es pasado se creara uno internamente.

2003

2004

Affected:

2005

Affected:

2005

self.dataOut

2006

self.dataOut

2006

2007

Return : None

2008

Return : None

2008

"""

2009

"""

2009

2010

self.isConfig = False

2011

self.isConfig = False

2011

2012

self.pts2read_SelfSpectra = 0

2013

self.pts2read_SelfSpectra = 0

2013

2014

self.pts2read_CrossSpectra = 0

2015

self.pts2read_CrossSpectra = 0

2015

2016

self.pts2read_DCchannels = 0

2017

self.pts2read_DCchannels = 0

2017

2018

self.datablock = None

2019

self.datablock = None

2019

2020

self.utc = None

2021

self.utc = None

2021

2022

self.ext = ".pdata"

2023

self.ext = ".pdata"

2023

2024

self.optchar = "P"

2025

self.optchar = "P"

2025

2026

self.basicHeaderObj = BasicHeader(LOCALTIME)

2027

self.basicHeaderObj = BasicHeader(LOCALTIME)

2027

2028

self.systemHeaderObj = SystemHeader()

2029

self.systemHeaderObj = SystemHeader()

2029

2030

self.radarControllerHeaderObj = RadarControllerHeader()

2031

self.radarControllerHeaderObj = RadarControllerHeader()

2031

2032

self.processingHeaderObj = ProcessingHeader()

2033

self.processingHeaderObj = ProcessingHeader()

2033

2034

self.online = 0

2035

self.online = 0

2035

2036

self.fp = None

2037

self.fp = None

2037

2038

self.idFile = None

2039

self.idFile = None

2039

2040

self.dtype = None

2041

self.dtype = None

2041

2042

self.fileSizeByHeader = None

2043

self.fileSizeByHeader = None

2043

2044

self.filenameList = []

2045

self.filenameList = []

2045

2046

self.filename = None

2047

self.filename = None

2047

2048

self.fileSize = None

2049

self.fileSize = None

2049

2050

self.firstHeaderSize = 0

2051

self.firstHeaderSize = 0

2051

2052

self.basicHeaderSize = 24

2053

self.basicHeaderSize = 24

2053

2054

self.pathList = []

2055

self.pathList = []

2055

2056

self.lastUTTime = 0

2057

self.lastUTTime = 0

2057

2058

self.maxTimeStep = 30

2059

self.maxTimeStep = 30

2059

2060

self.flagNoMoreFiles = 0

2061

self.flagNoMoreFiles = 0

2061

2062

self.set = 0

2063

self.set = 0

2063

2064

self.path = None

2065

self.path = None

2065

2066

self.delay = 60 #seconds

2067

self.delay = 60 #seconds

2067

2068

self.nTries = 3 #quantity tries

2069

self.nTries = 3 #quantity tries

2069

2070

self.nFiles = 3 #number of files for searching

2071

self.nFiles = 3 #number of files for searching

2071

2072

self.nReadBlocks = 0

2073

self.nReadBlocks = 0

2073

2074

self.flagIsNewFile = 1

2075

self.flagIsNewFile = 1

2075

2076

self.__isFirstTimeOnline = 1

2077

self.__isFirstTimeOnline = 1

2077

2078

self.ippSeconds = 0

2079

self.ippSeconds = 0

2079

2080

self.flagTimeBlock = 0

2081

self.flagTimeBlock = 0

2081

2082

self.flagIsNewBlock = 0

2083

self.flagIsNewBlock = 0

2083

2084

self.nTotalBlocks = 0

2085

self.nTotalBlocks = 0

2085

2086

self.blocksize = 0

2087

self.blocksize = 0

2087

2088

self.dataOut = self.createObjByDefault()

2089

self.dataOut = self.createObjByDefault()

2089

2090

self.profileIndex = 1 #Always

2091

self.profileIndex = 1 #Always

2091

2092

2093

def createObjByDefault(self):

2094

def createObjByDefault(self):

2094

2095

dataObj = Spectra()

2096

dataObj = Spectra()

2096

2097

return dataObj

2098

return dataObj

2098

2099

def __hasNotDataInBuffer(self):

2100

def __hasNotDataInBuffer(self):

2100

return 1

2101

return 1

2101

2102

2103

def getBlockDimension(self):

2104

def getBlockDimension(self):

2104

"""

2105

"""

2105

Obtiene la cantidad de puntos a leer por cada bloque de datos

2106

Obtiene la cantidad de puntos a leer por cada bloque de datos

2106

2107

Affected:

2108

Affected:

2108

self.nRdChannels

2109

self.nRdChannels

2109

self.nRdPairs

2110

self.nRdPairs

2110

self.pts2read_SelfSpectra

2111

self.pts2read_SelfSpectra

2111

self.pts2read_CrossSpectra

2112

self.pts2read_CrossSpectra

2112

self.pts2read_DCchannels

2113

self.pts2read_DCchannels

2113

self.blocksize

2114

self.blocksize

2114

self.dataOut.nChannels

2115

self.dataOut.nChannels

2115

self.dataOut.nPairs

2116

self.dataOut.nPairs

2116

2117

Return:

2118

Return:

2118

None

2119

None

2119

"""

2120

"""

2120

self.nRdChannels = 0

2121

self.nRdChannels = 0

2121

self.nRdPairs = 0

2122

self.nRdPairs = 0

2122

self.rdPairList = []

2123

self.rdPairList = []

2123

2124

for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):

2125

for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):

2125

if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:

2126

if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:

2126

self.nRdChannels = self.nRdChannels + 1 #par de canales iguales

2127

self.nRdChannels = self.nRdChannels + 1 #par de canales iguales

2127

else:

2128

else:

2128

self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes

2129

self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes

2129

self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))

2130

self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))

2130

2131

pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock

2132

pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock

2132

2133

self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)

2134

self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)

2134

self.blocksize = self.pts2read_SelfSpectra

2135

self.blocksize = self.pts2read_SelfSpectra

2135

2136

if self.processingHeaderObj.flag_cspc:

2137

if self.processingHeaderObj.flag_cspc:

2137

self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)

2138

self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)

2138

self.blocksize += self.pts2read_CrossSpectra

2139

self.blocksize += self.pts2read_CrossSpectra

2139

2140

if self.processingHeaderObj.flag_dc:

2141

if self.processingHeaderObj.flag_dc:

2141

self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)

2142

self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)

2142

self.blocksize += self.pts2read_DCchannels

2143

self.blocksize += self.pts2read_DCchannels

2143

2144

# self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels

2145

# self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels

2145

2146

2147

def readBlock(self):

2148

def readBlock(self):

2148

"""

2149

"""

2149

Lee el bloque de datos desde la posicion actual del puntero del archivo

2150

Lee el bloque de datos desde la posicion actual del puntero del archivo

2150

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

2151

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

2151

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

2152

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

2152

es seteado a 0

2153

es seteado a 0

2153

2154

Return: None

2155

Return: None

2155

2156

Variables afectadas:

2157

Variables afectadas:

2157

2158

self.flagIsNewFile

2159

self.flagIsNewFile

2159

self.flagIsNewBlock

2160

self.flagIsNewBlock

2160

self.nTotalBlocks

2161

self.nTotalBlocks

2161

self.data_spc

2162

self.data_spc

2162

self.data_cspc

2163

self.data_cspc

2163

self.data_dc

2164

self.data_dc

2164

2165

Exceptions:

2166

Exceptions:

2166

Si un bloque leido no es un bloque valido

2167

Si un bloque leido no es un bloque valido

2167

"""

2168

"""

2168

blockOk_flag = False

2169

blockOk_flag = False

2169

fpointer = self.fp.tell()

2170

fpointer = self.fp.tell()

2170

2171

spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )

2172

spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )

2172

spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2173

spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2173

2174

if self.processingHeaderObj.flag_cspc:

2175

if self.processingHeaderObj.flag_cspc:

2175

cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )

2176

cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )

2176

cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2177

cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2177

2178

if self.processingHeaderObj.flag_dc:

2179

if self.processingHeaderObj.flag_dc:

2179

dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )

2180

dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )

2180

dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D

2181

dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D

2181

2182

2183

if not(self.processingHeaderObj.shif_fft):

2184

if not(self.processingHeaderObj.shif_fft):

2184

#desplaza a la derecha en el eje 2 determinadas posiciones

2185

#desplaza a la derecha en el eje 2 determinadas posiciones

2185

shift = int(self.processingHeaderObj.profilesPerBlock/2)

2186

shift = int(self.processingHeaderObj.profilesPerBlock/2)

2186

spc = numpy.roll( spc, shift , axis=2 )

2187

spc = numpy.roll( spc, shift , axis=2 )

2187

2188

if self.processingHeaderObj.flag_cspc:

2189

if self.processingHeaderObj.flag_cspc:

2189

#desplaza a la derecha en el eje 2 determinadas posiciones

2190

#desplaza a la derecha en el eje 2 determinadas posiciones

2190

cspc = numpy.roll( cspc, shift, axis=2 )

2191

cspc = numpy.roll( cspc, shift, axis=2 )

2191

2192

# self.processingHeaderObj.shif_fft = True

2193

# self.processingHeaderObj.shif_fft = True

2193

2194

spc = numpy.transpose( spc, (0,2,1) )

2195

spc = numpy.transpose( spc, (0,2,1) )

2195

self.data_spc = spc

2196

self.data_spc = spc

2196

2197

if self.processingHeaderObj.flag_cspc:

2198

if self.processingHeaderObj.flag_cspc:

2198

cspc = numpy.transpose( cspc, (0,2,1) )

2199

cspc = numpy.transpose( cspc, (0,2,1) )

2199

self.data_cspc = cspc['real'] + cspc['imag']*1j

2200

self.data_cspc = cspc['real'] + cspc['imag']*1j

2200

else:

2201

else:

2201

self.data_cspc = None

2202

self.data_cspc = None

2202

2203

if self.processingHeaderObj.flag_dc:

2204

if self.processingHeaderObj.flag_dc:

2204

self.data_dc = dc['real'] + dc['imag']*1j

2205

self.data_dc = dc['real'] + dc['imag']*1j

2205

else:

2206

else:

2206

self.data_dc = None

2207

self.data_dc = None

2207

2208

self.flagIsNewFile = 0

2209

self.flagIsNewFile = 0

2209

self.flagIsNewBlock = 1

2210

self.flagIsNewBlock = 1

2210

2211

self.nTotalBlocks += 1

2212

self.nTotalBlocks += 1

2212

self.nReadBlocks += 1

2213

self.nReadBlocks += 1

2213

2214

return 1

2215

return 1

2215

2216

def getFirstHeader(self):

2217

def getFirstHeader(self):

2217

2218

self.dataOut.dtype = self.dtype

2219

self.dataOut.dtype = self.dtype

2219

2220

self.dataOut.nPairs = self.nRdPairs

2221

self.dataOut.nPairs = self.nRdPairs

2221

2222

self.dataOut.pairsList = self.rdPairList

2223

self.dataOut.pairsList = self.rdPairList

2223

2224

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

2225

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

2225

2226

self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock

2227

self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock

2227

2228

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

2229

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

2229

2230

self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt

2231

self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt

2231

2232

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

2233

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

2233

2234

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

2235

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

2235

2236

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

2237

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

2237

2238

self.dataOut.ippSeconds = self.ippSeconds

2239

self.dataOut.ippSeconds = self.ippSeconds

2239

2240

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints

2241

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints

2241

2242

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

2243

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

2243

2244

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

2245

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

2245

2246

self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft

2247

self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft

2247

2248

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

2249

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

2249

2250

self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip

2251

self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip

2251

2252

if self.radarControllerHeaderObj.code != None:

2253

if self.radarControllerHeaderObj.code != None:

2253

2254

self.dataOut.nCode = self.radarControllerHeaderObj.nCode

2255

self.dataOut.nCode = self.radarControllerHeaderObj.nCode

2255

2256

self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud

2257

self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud

2257

2258

self.dataOut.code = self.radarControllerHeaderObj.code

2259

self.dataOut.code = self.radarControllerHeaderObj.code

2259

2260

self.dataOut.flagDecodeData = True

2261

self.dataOut.flagDecodeData = True

2261

2262

def getData(self):

2263

def getData(self):

2263

"""

2264

"""

2264

Copia el buffer de lectura a la clase "Spectra",

2265

Copia el buffer de lectura a la clase "Spectra",

2265

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

2266

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

2266

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

2267

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

2267

2268

Return:

2269

Return:

2269

0 : Si no hay mas archivos disponibles

2270

0 : Si no hay mas archivos disponibles

2270

1 : Si hizo una buena copia del buffer

2271

1 : Si hizo una buena copia del buffer

2271

2272

Affected:

2273

Affected:

2273

self.dataOut

2274

self.dataOut

2274

2275

self.flagTimeBlock

2276

self.flagTimeBlock

2276

self.flagIsNewBlock

2277

self.flagIsNewBlock

2277

"""

2278

"""

2278

2279

if self.flagNoMoreFiles:

2280

if self.flagNoMoreFiles:

2280

self.dataOut.flagNoData = True

2281

self.dataOut.flagNoData = True

2281

print 'Process finished'

2282

print 'Process finished'

2282

return 0

2283

return 0

2283

2284

self.flagTimeBlock = 0

2285

self.flagTimeBlock = 0

2285

self.flagIsNewBlock = 0

2286

self.flagIsNewBlock = 0

2286

2287

if self.__hasNotDataInBuffer():

2288

if self.__hasNotDataInBuffer():

2288

2289

if not( self.readNextBlock() ):

2290

if not( self.readNextBlock() ):

2290

self.dataOut.flagNoData = True

2291

self.dataOut.flagNoData = True

2291

return 0

2292

return 0

2292

2293

#data es un numpy array de 3 dmensiones (perfiles, alturas y canales)

2294

#data es un numpy array de 3 dmensiones (perfiles, alturas y canales)

2294

2295

if self.data_dc == None:

2296

if self.data_dc == None:

2296

self.dataOut.flagNoData = True

2297

self.dataOut.flagNoData = True

2297

return 0

2298

return 0

2298

2299

self.getBasicHeader()

2300

self.getBasicHeader()

2300

2301

self.getFirstHeader()

2302

self.getFirstHeader()

2302

2303

self.dataOut.data_spc = self.data_spc

2304

self.dataOut.data_spc = self.data_spc

2304

2305

self.dataOut.data_cspc = self.data_cspc

2306

self.dataOut.data_cspc = self.data_cspc

2306

2307

self.dataOut.data_dc = self.data_dc

2308

self.dataOut.data_dc = self.data_dc

2308

2309

self.dataOut.flagNoData = False

2310

self.dataOut.flagNoData = False

2310

2311

self.dataOut.realtime = self.online

2312

self.dataOut.realtime = self.online

2312

2313

return self.dataOut.data_spc

2314

return self.dataOut.data_spc

2314

2315

2316

class SpectraWriter(JRODataWriter):

2317

class SpectraWriter(JRODataWriter):

2317

2318

"""

2319

"""

2319

Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura

2320

Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura

2320

de los datos siempre se realiza por bloques.

2321

de los datos siempre se realiza por bloques.

2321

"""

2322

"""

2322

2323

ext = ".pdata"

2324

ext = ".pdata"

2324

2325

optchar = "P"

2326

optchar = "P"

2326

2327

shape_spc_Buffer = None

2328

shape_spc_Buffer = None

2328

2329

shape_cspc_Buffer = None

2330

shape_cspc_Buffer = None

2330

2331

shape_dc_Buffer = None

2332

shape_dc_Buffer = None

2332

2333

data_spc = None

2334

data_spc = None

2334

2335

data_cspc = None

2336

data_cspc = None

2336

2337

data_dc = None

2338

data_dc = None

2338

2339

# dataOut = None

2340

# dataOut = None

2340

2341

def __init__(self):

2342

def __init__(self):

2342

"""

2343

"""

2343

Inicializador de la clase SpectraWriter para la escritura de datos de espectros.

2344

Inicializador de la clase SpectraWriter para la escritura de datos de espectros.

2344

2345

Affected:

2346

Affected:

2346

self.dataOut

2347

self.dataOut

2347

self.basicHeaderObj

2348

self.basicHeaderObj

2348

self.systemHeaderObj

2349

self.systemHeaderObj

2349

self.radarControllerHeaderObj

2350

self.radarControllerHeaderObj

2350

self.processingHeaderObj

2351

self.processingHeaderObj

2351

2352

Return: None

2353

Return: None

2353

"""

2354

"""

2354

2355

self.isConfig = False

2356

self.isConfig = False

2356

2357

self.nTotalBlocks = 0

2358

self.nTotalBlocks = 0

2358

2359

self.data_spc = None

2360

self.data_spc = None

2360

2361

self.data_cspc = None

2362

self.data_cspc = None

2362

2363

self.data_dc = None

2364

self.data_dc = None

2364

2365

self.fp = None

2366

self.fp = None

2366

2367

self.flagIsNewFile = 1

2368

self.flagIsNewFile = 1

2368

2369

self.nTotalBlocks = 0

2370

self.nTotalBlocks = 0

2370

2371

self.flagIsNewBlock = 0

2372

self.flagIsNewBlock = 0

2372

2373

self.setFile = None

2374

self.setFile = None

2374

2375

self.dtype = None

2376

self.dtype = None

2376

2377

self.path = None

2378

self.path = None

2378

2379

self.noMoreFiles = 0

2380

self.noMoreFiles = 0

2380

2381

self.filename = None

2382

self.filename = None

2382

2383

self.basicHeaderObj = BasicHeader(LOCALTIME)

2384

self.basicHeaderObj = BasicHeader(LOCALTIME)

2384

2385

self.systemHeaderObj = SystemHeader()

2386

self.systemHeaderObj = SystemHeader()

2386

2387

self.radarControllerHeaderObj = RadarControllerHeader()

2388

self.radarControllerHeaderObj = RadarControllerHeader()

2388

2389

self.processingHeaderObj = ProcessingHeader()

2390

self.processingHeaderObj = ProcessingHeader()

2390

2391

2392

def hasAllDataInBuffer(self):

2393

def hasAllDataInBuffer(self):

2393

return 1

2394

return 1

2394

2395

2396

def setBlockDimension(self):

2397

def setBlockDimension(self):

2397

"""

2398

"""

2398

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

2399

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

2399

2400

Affected:

2401

Affected:

2401

self.shape_spc_Buffer

2402

self.shape_spc_Buffer

2402

self.shape_cspc_Buffer

2403

self.shape_cspc_Buffer

2403

self.shape_dc_Buffer

2404

self.shape_dc_Buffer

2404

2405

Return: None

2406

Return: None

2406

"""

2407

"""

2407

self.shape_spc_Buffer = (self.dataOut.nChannels,

2408

self.shape_spc_Buffer = (self.dataOut.nChannels,

2408

self.processingHeaderObj.nHeights,

2409

self.processingHeaderObj.nHeights,

2409

self.processingHeaderObj.profilesPerBlock)

2410

self.processingHeaderObj.profilesPerBlock)

2410

2411

self.shape_cspc_Buffer = (self.dataOut.nPairs,

2412

self.shape_cspc_Buffer = (self.dataOut.nPairs,

2412

self.processingHeaderObj.nHeights,

2413

self.processingHeaderObj.nHeights,

2413

self.processingHeaderObj.profilesPerBlock)

2414

self.processingHeaderObj.profilesPerBlock)

2414

2415

self.shape_dc_Buffer = (self.dataOut.nChannels,

2416

self.shape_dc_Buffer = (self.dataOut.nChannels,

2416

self.processingHeaderObj.nHeights)

2417

self.processingHeaderObj.nHeights)

2417

2418

2419

def writeBlock(self):

2420

def writeBlock(self):

2420

"""

2421

"""

2421

Escribe el buffer en el file designado

2422

Escribe el buffer en el file designado

2422

2423

Affected:

2424

Affected:

2424

self.data_spc

2425

self.data_spc

2425

self.data_cspc

2426

self.data_cspc

2426

self.data_dc

2427

self.data_dc

2427

self.flagIsNewFile

2428

self.flagIsNewFile

2428

self.flagIsNewBlock

2429

self.flagIsNewBlock

2429

self.nTotalBlocks

2430

self.nTotalBlocks

2430

self.nWriteBlocks

2431

self.nWriteBlocks

2431

2432

Return: None

2433

Return: None

2433

"""

2434

"""

2434

2435

spc = numpy.transpose( self.data_spc, (0,2,1) )

2436

spc = numpy.transpose( self.data_spc, (0,2,1) )

2436

if not( self.processingHeaderObj.shif_fft ):

2437

if not( self.processingHeaderObj.shif_fft ):

2437

spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2438

spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2438

data = spc.reshape((-1))

2439

data = spc.reshape((-1))

2439

data = data.astype(self.dtype[0])

2440

data = data.astype(self.dtype[0])

2440

data.tofile(self.fp)

2441

data.tofile(self.fp)

2441

2442

if self.data_cspc != None:

2443

if self.data_cspc != None:

2443

data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )

2444

data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )

2444

cspc = numpy.transpose( self.data_cspc, (0,2,1) )

2445

cspc = numpy.transpose( self.data_cspc, (0,2,1) )

2445

if not( self.processingHeaderObj.shif_fft ):

2446

if not( self.processingHeaderObj.shif_fft ):

2446

cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2447

cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2447

data['real'] = cspc.real

2448

data['real'] = cspc.real

2448

data['imag'] = cspc.imag

2449

data['imag'] = cspc.imag

2449

data = data.reshape((-1))

2450

data = data.reshape((-1))

2450

data.tofile(self.fp)

2451

data.tofile(self.fp)

2451

2452

if self.data_dc != None:

2453

if self.data_dc != None:

2453

data = numpy.zeros( self.shape_dc_Buffer, self.dtype )

2454

data = numpy.zeros( self.shape_dc_Buffer, self.dtype )

2454

dc = self.data_dc

2455

dc = self.data_dc

2455

data['real'] = dc.real

2456

data['real'] = dc.real

2456

data['imag'] = dc.imag

2457

data['imag'] = dc.imag

2457

data = data.reshape((-1))

2458

data = data.reshape((-1))

2458

data.tofile(self.fp)

2459

data.tofile(self.fp)

2459

2460

self.data_spc.fill(0)

2461

self.data_spc.fill(0)

2461

2462

if self.data_dc != None:

2463

if self.data_dc != None:

2463

self.data_dc.fill(0)

2464

self.data_dc.fill(0)

2464

2465

if self.data_cspc != None:

2466

if self.data_cspc != None:

2466

self.data_cspc.fill(0)

2467

self.data_cspc.fill(0)

2467

2468

self.flagIsNewFile = 0

2469

self.flagIsNewFile = 0

2469

self.flagIsNewBlock = 1

2470

self.flagIsNewBlock = 1

2470

self.nTotalBlocks += 1

2471

self.nTotalBlocks += 1

2471

self.nWriteBlocks += 1

2472

self.nWriteBlocks += 1

2472

self.blockIndex += 1

2473

self.blockIndex += 1

2473

2474

2475

def putData(self):

2476

def putData(self):

2476

"""

2477

"""

2477

Setea un bloque de datos y luego los escribe en un file

2478

Setea un bloque de datos y luego los escribe en un file

2478

2479

Affected:

2480

Affected:

2480

self.data_spc

2481

self.data_spc

2481

self.data_cspc

2482

self.data_cspc

2482

self.data_dc

2483

self.data_dc

2483

2484

Return:

2485

Return:

2485

0 : Si no hay data o no hay mas files que puedan escribirse

2486

0 : Si no hay data o no hay mas files que puedan escribirse

2486

1 : Si se escribio la data de un bloque en un file

2487

1 : Si se escribio la data de un bloque en un file

2487

"""

2488

"""

2488

2489

if self.dataOut.flagNoData:

2490

if self.dataOut.flagNoData:

2490

return 0

2491

return 0

2491

2492

self.flagIsNewBlock = 0

2493

self.flagIsNewBlock = 0

2493

2494

if self.dataOut.flagTimeBlock:

2495

if self.dataOut.flagTimeBlock:

2495

self.data_spc.fill(0)

2496

self.data_spc.fill(0)

2496

self.data_cspc.fill(0)

2497

self.data_cspc.fill(0)

2497

self.data_dc.fill(0)

2498

self.data_dc.fill(0)

2498

self.setNextFile()

2499

self.setNextFile()

2499

2500

if self.flagIsNewFile == 0:

2501

if self.flagIsNewFile == 0:

2501

self.setBasicHeader()

2502

self.setBasicHeader()

2502

2503

self.data_spc = self.dataOut.data_spc.copy()

2504

self.data_spc = self.dataOut.data_spc.copy()

2504

if self.dataOut.data_cspc != None:

2505

if self.dataOut.data_cspc != None:

2505

self.data_cspc = self.dataOut.data_cspc.copy()

2506

self.data_cspc = self.dataOut.data_cspc.copy()

2506

self.data_dc = self.dataOut.data_dc.copy()

2507

self.data_dc = self.dataOut.data_dc.copy()

2507

2508

# #self.processingHeaderObj.dataBlocksPerFile)

2509

# #self.processingHeaderObj.dataBlocksPerFile)

2509

if self.hasAllDataInBuffer():

2510

if self.hasAllDataInBuffer():

2510

# self.setFirstHeader()

2511

# self.setFirstHeader()

2511

self.writeNextBlock()

2512

self.writeNextBlock()

2512

2513

return 1

2514

return 1

2514

2515

2516

def __getProcessFlags(self):

2517

def __getProcessFlags(self):

2517

2518

processFlags = 0

2519

processFlags = 0

2519

2520

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2521

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2521

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2522

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2522

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2523

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2523

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2524

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2524

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2525

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2525

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2526

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2526

2527

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2528

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2528

2529

2530

2531

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

2532

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

2532

PROCFLAG.DATATYPE_SHORT,

2533

PROCFLAG.DATATYPE_SHORT,

2533

PROCFLAG.DATATYPE_LONG,

2534

PROCFLAG.DATATYPE_LONG,

2534

PROCFLAG.DATATYPE_INT64,

2535

PROCFLAG.DATATYPE_INT64,

2535

PROCFLAG.DATATYPE_FLOAT,

2536

PROCFLAG.DATATYPE_FLOAT,

2536

PROCFLAG.DATATYPE_DOUBLE]

2537

PROCFLAG.DATATYPE_DOUBLE]

2537

2538

2539

for index in range(len(dtypeList)):

2540

for index in range(len(dtypeList)):

2540

if self.dataOut.dtype == dtypeList[index]:

2541

if self.dataOut.dtype == dtypeList[index]:

2541

dtypeValue = datatypeValueList[index]

2542

dtypeValue = datatypeValueList[index]

2542

break

2543

break

2543

2544

processFlags += dtypeValue

2545

processFlags += dtypeValue

2545

2546

if self.dataOut.flagDecodeData:

2547

if self.dataOut.flagDecodeData:

2547

processFlags += PROCFLAG.DECODE_DATA

2548

processFlags += PROCFLAG.DECODE_DATA

2548

2549

if self.dataOut.flagDeflipData:

2550

if self.dataOut.flagDeflipData:

2550

processFlags += PROCFLAG.DEFLIP_DATA

2551

processFlags += PROCFLAG.DEFLIP_DATA

2551

2552

if self.dataOut.code != None:

2553

if self.dataOut.code != None:

2553

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

2554

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

2554

2555

if self.dataOut.nIncohInt > 1:

2556

if self.dataOut.nIncohInt > 1:

2556

processFlags += PROCFLAG.INCOHERENT_INTEGRATION

2557

processFlags += PROCFLAG.INCOHERENT_INTEGRATION

2557

2558

if self.dataOut.data_dc != None:

2559

if self.dataOut.data_dc != None:

2559

processFlags += PROCFLAG.SAVE_CHANNELS_DC

2560

processFlags += PROCFLAG.SAVE_CHANNELS_DC

2560

2561

return processFlags

2562

return processFlags

2562

2563

2564

def __getBlockSize(self):

2565

def __getBlockSize(self):

2565

'''

2566

'''

2566

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra

2567

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra

2567

'''

2568

'''

2568

2569

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2570

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2570

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2571

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2571

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2572

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2572

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2573

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2573

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2574

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2574

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2575

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2575

2576

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2577

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2577

datatypeValueList = [1,2,4,8,4,8]

2578

datatypeValueList = [1,2,4,8,4,8]

2578

for index in range(len(dtypeList)):

2579

for index in range(len(dtypeList)):

2579

if self.dataOut.dtype == dtypeList[index]:

2580

if self.dataOut.dtype == dtypeList[index]:

2580

datatypeValue = datatypeValueList[index]

2581

datatypeValue = datatypeValueList[index]

2581

break

2582

break

2582

2583

2584

pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints

2585

pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints

2585

2586

pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)

2587

pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)

2587

blocksize = (pts2write_SelfSpectra*datatypeValue)

2588

blocksize = (pts2write_SelfSpectra*datatypeValue)

2588

2589

if self.dataOut.data_cspc != None:

2590

if self.dataOut.data_cspc != None:

2590

pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)

2591

pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)

2591

blocksize += (pts2write_CrossSpectra*datatypeValue*2)

2592

blocksize += (pts2write_CrossSpectra*datatypeValue*2)

2592

2593

if self.dataOut.data_dc != None:

2594

if self.dataOut.data_dc != None:

2594

pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)

2595

pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)

2595

blocksize += (pts2write_DCchannels*datatypeValue*2)

2596

blocksize += (pts2write_DCchannels*datatypeValue*2)

2596

2597

blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO

2598

blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO

2598

2599

return blocksize

2600

return blocksize

2600

2601

def setFirstHeader(self):

2602

def setFirstHeader(self):

2602

2603

"""

2604

"""

2604

Obtiene una copia del First Header

2605

Obtiene una copia del First Header

2605

2606

Affected:

2607

Affected:

2607

self.systemHeaderObj

2608

self.systemHeaderObj

2608

self.radarControllerHeaderObj

2609

self.radarControllerHeaderObj

2609

self.dtype

2610

self.dtype

2610

2611

Return:

2612

Return:

2612

None

2613

None

2613

"""

2614

"""

2614

2615

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

2616

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

2616

self.systemHeaderObj.nChannels = self.dataOut.nChannels

2617

self.systemHeaderObj.nChannels = self.dataOut.nChannels

2617

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

2618

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

2618

old_code_size = self.dataOut.radarControllerHeaderObj.code_size

2619

old_code_size = self.dataOut.radarControllerHeaderObj.code_size

2619

new_code_size = int(numpy.ceil(self.dataOut.nBaud/32.))*self.dataOut.nCode*4

2620

new_code_size = int(numpy.ceil(self.dataOut.nBaud/32.))*self.dataOut.nCode*4

2620

self.radarControllerHeaderObj.size = self.radarControllerHeaderObj.size - old_code_size + new_code_size

2621

self.radarControllerHeaderObj.size = self.radarControllerHeaderObj.size - old_code_size + new_code_size

2621

2622

self.setBasicHeader()

2623

self.setBasicHeader()

2623

2624

processingHeaderSize = 40 # bytes

2625

processingHeaderSize = 40 # bytes

2625

self.processingHeaderObj.dtype = 1 # Spectra

2626

self.processingHeaderObj.dtype = 1 # Spectra

2626

self.processingHeaderObj.blockSize = self.__getBlockSize()

2627

self.processingHeaderObj.blockSize = self.__getBlockSize()

2627

self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints

2628

self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints

2628

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

2629

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

2629

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

2630

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

2630

self.processingHeaderObj.processFlags = self.__getProcessFlags()

2631

self.processingHeaderObj.processFlags = self.__getProcessFlags()

2631

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval

2632

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval

2632

self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt

2633

self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt

2633

self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels

2634

self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels

2634

self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT

2635

self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT

2635

2636

if self.processingHeaderObj.totalSpectra > 0:

2637

if self.processingHeaderObj.totalSpectra > 0:

2637

channelList = []

2638

channelList = []

2638

for channel in range(self.dataOut.nChannels):

2639

for channel in range(self.dataOut.nChannels):

2639

channelList.append(channel)

2640

channelList.append(channel)

2640

channelList.append(channel)

2641

channelList.append(channel)

2641

2642

pairsList = []

2643

pairsList = []

2643

if self.dataOut.nPairs > 0:

2644

if self.dataOut.nPairs > 0:

2644

for pair in self.dataOut.pairsList:

2645

for pair in self.dataOut.pairsList:

2645

pairsList.append(pair[0])

2646

pairsList.append(pair[0])

2646

pairsList.append(pair[1])

2647

pairsList.append(pair[1])

2647

2648

spectraComb = channelList + pairsList

2649

spectraComb = channelList + pairsList

2649

spectraComb = numpy.array(spectraComb,dtype="u1")

2650

spectraComb = numpy.array(spectraComb,dtype="u1")

2650

self.processingHeaderObj.spectraComb = spectraComb

2651

self.processingHeaderObj.spectraComb = spectraComb

2651

sizeOfSpcComb = len(spectraComb)

2652

sizeOfSpcComb = len(spectraComb)

2652

processingHeaderSize += sizeOfSpcComb

2653

processingHeaderSize += sizeOfSpcComb

2653

2654

# The processing header should not have information about code

2655

# The processing header should not have information about code

2655

# if self.dataOut.code != None:

2656

# if self.dataOut.code != None:

2656

# self.processingHeaderObj.code = self.dataOut.code

2657

# self.processingHeaderObj.code = self.dataOut.code

2657

# self.processingHeaderObj.nCode = self.dataOut.nCode

2658

# self.processingHeaderObj.nCode = self.dataOut.nCode

2658

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

2659

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

2659

# nCodeSize = 4 # bytes

2660

# nCodeSize = 4 # bytes

2660

# nBaudSize = 4 # bytes

2661

# nBaudSize = 4 # bytes

2661

# codeSize = 4 # bytes

2662

# codeSize = 4 # bytes

2662

# sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)

2663

# sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)

2663

# processingHeaderSize += sizeOfCode

2664

# processingHeaderSize += sizeOfCode

2664

2665

if self.processingHeaderObj.nWindows != 0:

2666

if self.processingHeaderObj.nWindows != 0:

2666

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

2667

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

2667

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2668

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2668

self.processingHeaderObj.nHeights = self.dataOut.nHeights

2669

self.processingHeaderObj.nHeights = self.dataOut.nHeights

2669

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

2670

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

2670

sizeOfFirstHeight = 4

2671

sizeOfFirstHeight = 4

2671

sizeOfdeltaHeight = 4

2672

sizeOfdeltaHeight = 4

2672

sizeOfnHeights = 4

2673

sizeOfnHeights = 4

2673

sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows

2674

sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows

2674

processingHeaderSize += sizeOfWindows

2675

processingHeaderSize += sizeOfWindows

2675

2676

self.processingHeaderObj.size = processingHeaderSize

2677

self.processingHeaderObj.size = processingHeaderSize

2677

2678

class SpectraHeisWriter(Operation):

2679

class SpectraHeisWriter(Operation):

2679

# set = None

2680

# set = None

2680

setFile = None

2681

setFile = None

2681

idblock = None

2682

idblock = None

2682

doypath = None

2683

doypath = None

2683

subfolder = None

2684

subfolder = None

2684

2685

def __init__(self):

2686

def __init__(self):

2686

self.wrObj = FITS()

2687

self.wrObj = FITS()

2687

# self.dataOut = dataOut

2688

# self.dataOut = dataOut

2688

self.nTotalBlocks=0

2689

self.nTotalBlocks=0

2689

# self.set = None

2690

# self.set = None

2690

self.setFile = None

2691

self.setFile = None

2691

self.idblock = 0

2692

self.idblock = 0

2692

self.wrpath = None

2693

self.wrpath = None

2693

self.doypath = None

2694

self.doypath = None

2694

self.subfolder = None

2695

self.subfolder = None

2695

self.isConfig = False

2696

self.isConfig = False

2696

2697

def isNumber(str):

2698

def isNumber(str):

2698

"""

2699

"""

2699

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

2700

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

2700

2701

Excepciones:

2702

Excepciones:

2702

Si un determinado string no puede ser convertido a numero

2703

Si un determinado string no puede ser convertido a numero

2703

Input:

2704

Input:

2704

str, string al cual se le analiza para determinar si convertible a un numero o no

2705

str, string al cual se le analiza para determinar si convertible a un numero o no

2705

2706

Return:

2707

Return:

2707

True : si el string es uno numerico

2708

True : si el string es uno numerico

2708

False : no es un string numerico

2709

False : no es un string numerico

2709

"""

2710

"""

2710

try:

2711

try:

2711

float( str )

2712

float( str )

2712

return True

2713

return True

2713

except:

2714

except:

2714

return False

2715

return False

2715

2716

def setup(self, dataOut, wrpath):

2717

def setup(self, dataOut, wrpath):

2717

2718

if not(os.path.exists(wrpath)):

2719

if not(os.path.exists(wrpath)):

2719

os.mkdir(wrpath)

2720

os.mkdir(wrpath)

2720

2721

self.wrpath = wrpath

2722

self.wrpath = wrpath

2722

# self.setFile = 0

2723

# self.setFile = 0

2723

self.dataOut = dataOut

2724

self.dataOut = dataOut

2724

2725

def putData(self):

2726

def putData(self):

2726

name= time.localtime( self.dataOut.utctime)

2727

name= time.localtime( self.dataOut.utctime)

2727

ext=".fits"

2728

ext=".fits"

2728

2729

if self.doypath == None:

2730

if self.doypath == None:

2730

self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))

2731

self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))

2731

self.doypath = os.path.join( self.wrpath, self.subfolder )

2732

self.doypath = os.path.join( self.wrpath, self.subfolder )

2732

os.mkdir(self.doypath)

2733

os.mkdir(self.doypath)

2733

2734

if self.setFile == None:

2735

if self.setFile == None:

2735

# self.set = self.dataOut.set

2736

# self.set = self.dataOut.set

2736

self.setFile = 0

2737

self.setFile = 0

2737

# if self.set != self.dataOut.set:

2738

# if self.set != self.dataOut.set:

2738

## self.set = self.dataOut.set

2739

## self.set = self.dataOut.set

2739

# self.setFile = 0

2740

# self.setFile = 0

2740

2741

#make the filename

2742

#make the filename

2742

file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)

2743

file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)

2743

2744

filename = os.path.join(self.wrpath,self.subfolder, file)

2745

filename = os.path.join(self.wrpath,self.subfolder, file)

2745

2746

idblock = numpy.array([self.idblock],dtype="int64")

2747

idblock = numpy.array([self.idblock],dtype="int64")

2747

header=self.wrObj.cFImage(idblock=idblock,

2748

header=self.wrObj.cFImage(idblock=idblock,

2748

year=time.gmtime(self.dataOut.utctime).tm_year,

2749

year=time.gmtime(self.dataOut.utctime).tm_year,

2749

month=time.gmtime(self.dataOut.utctime).tm_mon,

2750

month=time.gmtime(self.dataOut.utctime).tm_mon,

2750

day=time.gmtime(self.dataOut.utctime).tm_mday,

2751

day=time.gmtime(self.dataOut.utctime).tm_mday,

2751

hour=time.gmtime(self.dataOut.utctime).tm_hour,

2752

hour=time.gmtime(self.dataOut.utctime).tm_hour,

2752

minute=time.gmtime(self.dataOut.utctime).tm_min,

2753

minute=time.gmtime(self.dataOut.utctime).tm_min,

2753

second=time.gmtime(self.dataOut.utctime).tm_sec)

2754

second=time.gmtime(self.dataOut.utctime).tm_sec)

2754

2755

c=3E8

2756

c=3E8

2756

deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2757

deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2757

freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))

2758

freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))

2758

2759

colList = []

2760

colList = []

2760

2761

colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)

2762

colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)

2762

2763

colList.append(colFreq)

2764

colList.append(colFreq)

2764

2765

nchannel=self.dataOut.nChannels

2766

nchannel=self.dataOut.nChannels

2766

2767

for i in range(nchannel):

2768

for i in range(nchannel):

2768

col = self.wrObj.writeData(name="PCh"+str(i+1),

2769

col = self.wrObj.writeData(name="PCh"+str(i+1),

2769

format=str(self.dataOut.nFFTPoints)+'E',

2770

format=str(self.dataOut.nFFTPoints)+'E',

2770

data=10*numpy.log10(self.dataOut.data_spc[i,:]))

2771

data=10*numpy.log10(self.dataOut.data_spc[i,:]))

2771

2772

colList.append(col)

2773

colList.append(col)

2773

2774

data=self.wrObj.Ctable(colList=colList)

2775

data=self.wrObj.Ctable(colList=colList)

2775

2776

self.wrObj.CFile(header,data)

2777

self.wrObj.CFile(header,data)

2777

2778

self.wrObj.wFile(filename)

2779

self.wrObj.wFile(filename)

2779

2780

#update the setFile

2781

#update the setFile

2781

self.setFile += 1

2782

self.setFile += 1

2782

self.idblock += 1

2783

self.idblock += 1

2783

2784

return 1

2785

return 1

2785

2786

def run(self, dataOut, **kwargs):

2787

def run(self, dataOut, **kwargs):

2787

2788

if not(self.isConfig):

2789

if not(self.isConfig):

2789

2790

self.setup(dataOut, **kwargs)

2791

self.setup(dataOut, **kwargs)

2791

self.isConfig = True

2792

self.isConfig = True

2792

2793

self.putData()

2794

self.putData()

2794

2795

2796

2797

class ParameterConf:

2798

class ParameterConf:

2798

ELEMENTNAME = 'Parameter'

2799

ELEMENTNAME = 'Parameter'

2799

def __init__(self):

2800

def __init__(self):

2800

self.name = ''

2801

self.name = ''

2801

self.value = ''

2802

self.value = ''

2802

2803

def readXml(self, parmElement):

2804

def readXml(self, parmElement):

2804

self.name = parmElement.get('name')

2805

self.name = parmElement.get('name')

2805

self.value = parmElement.get('value')

2806

self.value = parmElement.get('value')

2806

2807

def getElementName(self):

2808

def getElementName(self):

2808

return self.ELEMENTNAME

2809

return self.ELEMENTNAME

2809

2810

class Metadata:

2811

class Metadata:

2811

2812

def __init__(self, filename):

2813

def __init__(self, filename):

2813

self.parmConfObjList = []

2814

self.parmConfObjList = []

2814

self.readXml(filename)

2815

self.readXml(filename)

2815

2816

def readXml(self, filename):

2817

def readXml(self, filename):

2817

self.projectElement = None

2818

self.projectElement = None

2818

self.procUnitConfObjDict = {}

2819

self.procUnitConfObjDict = {}

2819

self.projectElement = ElementTree().parse(filename)

2820

self.projectElement = ElementTree().parse(filename)

2820

self.project = self.projectElement.tag

2821

self.project = self.projectElement.tag

2821

2822

parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())

2823

parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())

2823

2824

for parmElement in parmElementList:

2825

for parmElement in parmElementList:

2825

parmConfObj = ParameterConf()

2826

parmConfObj = ParameterConf()

2826

parmConfObj.readXml(parmElement)

2827

parmConfObj.readXml(parmElement)

2827

self.parmConfObjList.append(parmConfObj)

2828

self.parmConfObjList.append(parmConfObj)

2828

2829

class FitsWriter(Operation):

2830

class FitsWriter(Operation):

2830

2831

def __init__(self):

2832

def __init__(self):

2832

self.isConfig = False

2833

self.isConfig = False

2833

self.dataBlocksPerFile = None

2834

self.dataBlocksPerFile = None

2834

self.blockIndex = 0

2835

self.blockIndex = 0

2835

self.flagIsNewFile = 1

2836

self.flagIsNewFile = 1

2836

self.fitsObj = None

2837

self.fitsObj = None

2837

self.optchar = 'P'

2838

self.optchar = 'P'

2838

self.ext = '.fits'

2839

self.ext = '.fits'

2839

self.setFile = 0

2840

self.setFile = 0

2840

2841

def setFitsHeader(self, dataOut, metadatafile):

2842

def setFitsHeader(self, dataOut, metadatafile):

2842

2843

header_data = pyfits.PrimaryHDU()

2844

header_data = pyfits.PrimaryHDU()

2844

2845

metadata4fits = Metadata(metadatafile)

2846

metadata4fits = Metadata(metadatafile)

2846

for parameter in metadata4fits.parmConfObjList:

2847

for parameter in metadata4fits.parmConfObjList:

2847

parm_name = parameter.name

2848

parm_name = parameter.name

2848

parm_value = parameter.value

2849

parm_value = parameter.value

2849

2850

# if parm_value == 'fromdatadatetime':

2851

# if parm_value == 'fromdatadatetime':

2851

# value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())

2852

# value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())

2852

# elif parm_value == 'fromdataheights':

2853

# elif parm_value == 'fromdataheights':

2853

# value = dataOut.nHeights

2854

# value = dataOut.nHeights

2854

# elif parm_value == 'fromdatachannel':

2855

# elif parm_value == 'fromdatachannel':

2855

# value = dataOut.nChannels

2856

# value = dataOut.nChannels

2856

# elif parm_value == 'fromdatasamples':

2857

# elif parm_value == 'fromdatasamples':

2857

# value = dataOut.nFFTPoints

2858

# value = dataOut.nFFTPoints

2858

# else:

2859

# else:

2859

# value = parm_value

2860

# value = parm_value

2860

2861

header_data.header[parm_name] = parm_value

2862

header_data.header[parm_name] = parm_value

2862

2863

2864

header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())

2865

header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())

2865

header_data.header['CHANNELLIST'] = str(dataOut.channelList)

2866

header_data.header['CHANNELLIST'] = str(dataOut.channelList)

2866

header_data.header['NCHANNELS'] = dataOut.nChannels

2867

header_data.header['NCHANNELS'] = dataOut.nChannels

2867

#header_data.header['HEIGHTS'] = dataOut.heightList

2868

#header_data.header['HEIGHTS'] = dataOut.heightList

2868

header_data.header['NHEIGHTS'] = dataOut.nHeights

2869

header_data.header['NHEIGHTS'] = dataOut.nHeights

2869

2870

header_data.header['IPPSECONDS'] = dataOut.ippSeconds

2871

header_data.header['IPPSECONDS'] = dataOut.ippSeconds

2871

header_data.header['NCOHINT'] = dataOut.nCohInt

2872

header_data.header['NCOHINT'] = dataOut.nCohInt

2872

header_data.header['NINCOHINT'] = dataOut.nIncohInt

2873

header_data.header['NINCOHINT'] = dataOut.nIncohInt

2873

header_data.header['TIMEZONE'] = dataOut.timeZone

2874

header_data.header['TIMEZONE'] = dataOut.timeZone

2874

header_data.header['NBLOCK'] = self.blockIndex

2875

header_data.header['NBLOCK'] = self.blockIndex

2875

2876

header_data.writeto(self.filename)

2877

header_data.writeto(self.filename)

2877

2878

self.addExtension(dataOut.heightList,'HEIGHTLIST')

2879

self.addExtension(dataOut.heightList,'HEIGHTLIST')

2879

2880

2881

def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):

2882

def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):

2882

2883

self.path = path

2884

self.path = path

2884

self.dataOut = dataOut

2885

self.dataOut = dataOut

2885

self.metadatafile = metadatafile

2886

self.metadatafile = metadatafile

2886

self.dataBlocksPerFile = dataBlocksPerFile

2887

self.dataBlocksPerFile = dataBlocksPerFile

2887

2888

def open(self):

2889

def open(self):

2889

self.fitsObj = pyfits.open(self.filename, mode='update')

2890

self.fitsObj = pyfits.open(self.filename, mode='update')

2890

2891

2892

def addExtension(self, data, tagname):

2893

def addExtension(self, data, tagname):

2893

self.open()

2894

self.open()

2894

extension = pyfits.ImageHDU(data=data, name=tagname)

2895

extension = pyfits.ImageHDU(data=data, name=tagname)

2895

#extension.header['TAG'] = tagname

2896

#extension.header['TAG'] = tagname

2896

self.fitsObj.append(extension)

2897

self.fitsObj.append(extension)

2897

self.write()

2898

self.write()

2898

2899

def addData(self, data):

2900

def addData(self, data):

2900

self.open()

2901

self.open()

2901

extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])

2902

extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])

2902

extension.header['UTCTIME'] = self.dataOut.utctime

2903

extension.header['UTCTIME'] = self.dataOut.utctime

2903

self.fitsObj.append(extension)

2904

self.fitsObj.append(extension)

2904

self.blockIndex += 1

2905

self.blockIndex += 1

2905

self.fitsObj[0].header['NBLOCK'] = self.blockIndex

2906

self.fitsObj[0].header['NBLOCK'] = self.blockIndex

2906

2907

self.write()

2908

self.write()

2908

2909

def write(self):

2910

def write(self):

2910

2911

self.fitsObj.flush(verbose=True)

2912

self.fitsObj.flush(verbose=True)

2912

self.fitsObj.close()

2913

self.fitsObj.close()

2913

2914

2915

def setNextFile(self):

2916

def setNextFile(self):

2916

2917

ext = self.ext

2918

ext = self.ext

2918

path = self.path

2919

path = self.path

2919

2920

timeTuple = time.localtime( self.dataOut.utctime)

2921

timeTuple = time.localtime( self.dataOut.utctime)

2921

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

2922

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

2922

2923

fullpath = os.path.join( path, subfolder )

2924

fullpath = os.path.join( path, subfolder )

2924

if not( os.path.exists(fullpath) ):

2925

if not( os.path.exists(fullpath) ):

2925

os.mkdir(fullpath)

2926

os.mkdir(fullpath)

2926

self.setFile = -1 #inicializo mi contador de seteo

2927

self.setFile = -1 #inicializo mi contador de seteo

2927

else:

2928

else:

2928

filesList = os.listdir( fullpath )

2929

filesList = os.listdir( fullpath )

2929

if len( filesList ) > 0:

2930

if len( filesList ) > 0:

2930

filesList = sorted( filesList, key=str.lower )

2931

filesList = sorted( filesList, key=str.lower )

2931

filen = filesList[-1]

2932

filen = filesList[-1]

2932

2933

if isNumber( filen[8:11] ):

2934

if isNumber( filen[8:11] ):

2934

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

2935

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

2935

else:

2936

else:

2936

self.setFile = -1

2937

self.setFile = -1

2937

else:

2938

else:

2938

self.setFile = -1 #inicializo mi contador de seteo

2939

self.setFile = -1 #inicializo mi contador de seteo

2939

2940

setFile = self.setFile

2941

setFile = self.setFile

2941

setFile += 1

2942

setFile += 1

2942

2943

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

2944

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

2944

timeTuple.tm_year,

2945

timeTuple.tm_year,

2945

timeTuple.tm_yday,

2946

timeTuple.tm_yday,

2946

setFile,

2947

setFile,

2947

ext )

2948

ext )

2948

2949

filename = os.path.join( path, subfolder, file )

2950

filename = os.path.join( path, subfolder, file )

2950

2951

self.blockIndex = 0

2952

self.blockIndex = 0

2952

self.filename = filename

2953

self.filename = filename

2953

self.setFile = setFile

2954

self.setFile = setFile

2954

self.flagIsNewFile = 1

2955

self.flagIsNewFile = 1

2955

2956

print 'Writing the file: %s'%self.filename

2957

print 'Writing the file: %s'%self.filename

2957

2958

self.setFitsHeader(self.dataOut, self.metadatafile)

2959

self.setFitsHeader(self.dataOut, self.metadatafile)

2959

2960

return 1

2961

return 1

2961

2962

def writeBlock(self):

2963

def writeBlock(self):

2963

self.addData(self.dataOut.data_spc)

2964

self.addData(self.dataOut.data_spc)

2964

self.flagIsNewFile = 0

2965

self.flagIsNewFile = 0

2965

2966

2967

def __setNewBlock(self):

2968

def __setNewBlock(self):

2968

2969

if self.flagIsNewFile:

2970

if self.flagIsNewFile:

2970

return 1

2971

return 1

2971

2972

if self.blockIndex < self.dataBlocksPerFile:

2973

if self.blockIndex < self.dataBlocksPerFile:

2973

return 1

2974

return 1

2974

2975

if not( self.setNextFile() ):

2976

if not( self.setNextFile() ):

2976

return 0

2977

return 0

2977

2978

return 1

2979

return 1

2979

2980

def writeNextBlock(self):

2981

def writeNextBlock(self):

2981

if not( self.__setNewBlock() ):

2982

if not( self.__setNewBlock() ):

2982

return 0

2983

return 0

2983

self.writeBlock()

2984

self.writeBlock()

2984

return 1

2985

return 1

2985

2986

def putData(self):

2987

def putData(self):

2987

if self.flagIsNewFile:

2988

if self.flagIsNewFile:

2988

self.setNextFile()

2989

self.setNextFile()

2989

self.writeNextBlock()

2990

self.writeNextBlock()

2990

2991

def run(self, dataOut, **kwargs):

2992

def run(self, dataOut, **kwargs):

2992

if not(self.isConfig):

2993

if not(self.isConfig):

2993

self.setup(dataOut, **kwargs)

2994

self.setup(dataOut, **kwargs)

2994

self.isConfig = True

2995

self.isConfig = True

2995

self.putData()

2996

self.putData()

2996

2997

2998

class FitsReader(ProcessingUnit):

2999

class FitsReader(ProcessingUnit):

2999

3000

# __TIMEZONE = time.timezone

3001

# __TIMEZONE = time.timezone

3001

3002

expName = None

3003

expName = None

3003

datetimestr = None

3004

datetimestr = None

3004

utc = None

3005

utc = None

3005

nChannels = None

3006

nChannels = None

3006

nSamples = None

3007

nSamples = None

3007

dataBlocksPerFile = None

3008

dataBlocksPerFile = None

3008

comments = None

3009

comments = None

3009

lastUTTime = None

3010

lastUTTime = None

3010

header_dict = None

3011

header_dict = None

3011

data = None

3012

data = None

3012

data_header_dict = None

3013

data_header_dict = None

3013

3014

def __init__(self):

3015

def __init__(self):

3015

self.isConfig = False

3016

self.isConfig = False

3016

self.ext = '.fits'

3017

self.ext = '.fits'

3017

self.setFile = 0

3018

self.setFile = 0

3018

self.flagNoMoreFiles = 0

3019

self.flagNoMoreFiles = 0

3019

self.flagIsNewFile = 1

3020

self.flagIsNewFile = 1

3020

self.flagTimeBlock = None

3021

self.flagTimeBlock = None

3021

self.fileIndex = None

3022

self.fileIndex = None

3022

self.filename = None

3023

self.filename = None

3023

self.fileSize = None

3024

self.fileSize = None

3024

self.fitsObj = None

3025

self.fitsObj = None

3025

self.timeZone = None

3026

self.timeZone = None

3026

self.nReadBlocks = 0

3027

self.nReadBlocks = 0

3027

self.nTotalBlocks = 0

3028

self.nTotalBlocks = 0

3028

self.dataOut = self.createObjByDefault()

3029

self.dataOut = self.createObjByDefault()

3029

self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()

3030

self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()

3030

self.blockIndex = 1

3031

self.blockIndex = 1

3031

3032

def createObjByDefault(self):

3033

def createObjByDefault(self):

3033

3034

dataObj = Fits()

3035

dataObj = Fits()

3035

3036

return dataObj

3037

return dataObj

3037

3038

def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):

3039

def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):

3039

try:

3040

try:

3040

fitsObj = pyfits.open(filename,'readonly')

3041

fitsObj = pyfits.open(filename,'readonly')

3041

except:

3042

except:

3042

raise IOError, "The file %s can't be opened" %(filename)

3043

raise IOError, "The file %s can't be opened" %(filename)

3043

3044

header = fitsObj[0].header

3045

header = fitsObj[0].header

3045

struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")

3046

struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")

3046

utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS

3047

utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS

3047

3048

ltc = utc

3049

ltc = utc

3049

if useLocalTime:

3050

if useLocalTime:

3050

ltc -= time.timezone

3051

ltc -= time.timezone

3051

thisDatetime = datetime.datetime.utcfromtimestamp(ltc)

3052

thisDatetime = datetime.datetime.utcfromtimestamp(ltc)

3052

thisTime = thisDatetime.time()

3053

thisTime = thisDatetime.time()

3053

3054

if not ((startTime <= thisTime) and (endTime > thisTime)):

3055

if not ((startTime <= thisTime) and (endTime > thisTime)):

3055

return None

3056

return None

3056

3057

return thisDatetime

3058

return thisDatetime

3058

3059

def __setNextFileOnline(self):

3060

def __setNextFileOnline(self):

3060

raise ValueError, "No implemented"

3061

raise ValueError, "No implemented"

3061

3062

def __setNextFileOffline(self):

3063

def __setNextFileOffline(self):

3063

idFile = self.fileIndex

3064

idFile = self.fileIndex

3064

3065

while (True):

3066

while (True):

3066

idFile += 1

3067

idFile += 1

3067

if not(idFile < len(self.filenameList)):

3068

if not(idFile < len(self.filenameList)):

3068

self.flagNoMoreFiles = 1

3069

self.flagNoMoreFiles = 1

3069

print "No more Files"

3070

print "No more Files"

3070

return 0

3071

return 0

3071

3072

filename = self.filenameList[idFile]

3073

filename = self.filenameList[idFile]

3073

3074

# if not(self.__verifyFile(filename)):

3075

# if not(self.__verifyFile(filename)):

3075

# continue

3076

# continue

3076

3077

fileSize = os.path.getsize(filename)

3078

fileSize = os.path.getsize(filename)

3078

fitsObj = pyfits.open(filename,'readonly')

3079

fitsObj = pyfits.open(filename,'readonly')

3079

break

3080

break

3080

3081

self.flagIsNewFile = 1

3082

self.flagIsNewFile = 1

3082

self.fileIndex = idFile

3083

self.fileIndex = idFile

3083

self.filename = filename

3084

self.filename = filename

3084

self.fileSize = fileSize

3085

self.fileSize = fileSize

3085

self.fitsObj = fitsObj

3086

self.fitsObj = fitsObj

3086

self.blockIndex = 0

3087

self.blockIndex = 0

3087

print "Setting the file: %s"%self.filename

3088

print "Setting the file: %s"%self.filename

3088

3089

return 1

3090

return 1

3090

3091

def readHeader(self):

3092

def readHeader(self):

3092

headerObj = self.fitsObj[0]

3093

headerObj = self.fitsObj[0]

3093

3094

self.header_dict = headerObj.header

3095

self.header_dict = headerObj.header

3095

if 'EXPNAME' in headerObj.header.keys():

3096

if 'EXPNAME' in headerObj.header.keys():

3096

self.expName = headerObj.header['EXPNAME']

3097

self.expName = headerObj.header['EXPNAME']

3097

3098

if 'DATATYPE' in headerObj.header.keys():

3099

if 'DATATYPE' in headerObj.header.keys():

3099

self.dataType = headerObj.header['DATATYPE']

3100

self.dataType = headerObj.header['DATATYPE']

3100

3101

self.datetimestr = headerObj.header['DATETIME']

3102

self.datetimestr = headerObj.header['DATETIME']

3102

channelList = headerObj.header['CHANNELLIST']

3103

channelList = headerObj.header['CHANNELLIST']

3103

channelList = channelList.split('[')

3104

channelList = channelList.split('[')

3104

channelList = channelList[1].split(']')

3105

channelList = channelList[1].split(']')

3105

channelList = channelList[0].split(',')

3106

channelList = channelList[0].split(',')

3106

channelList = [int(ch) for ch in channelList]

3107

channelList = [int(ch) for ch in channelList]

3107

self.channelList = channelList

3108

self.channelList = channelList

3108

self.nChannels = headerObj.header['NCHANNELS']

3109

self.nChannels = headerObj.header['NCHANNELS']

3109

self.nHeights = headerObj.header['NHEIGHTS']

3110

self.nHeights = headerObj.header['NHEIGHTS']

3110

self.ippSeconds = headerObj.header['IPPSECONDS']

3111

self.ippSeconds = headerObj.header['IPPSECONDS']

3111

self.nCohInt = headerObj.header['NCOHINT']

3112

self.nCohInt = headerObj.header['NCOHINT']

3112

self.nIncohInt = headerObj.header['NINCOHINT']

3113

self.nIncohInt = headerObj.header['NINCOHINT']

3113

self.dataBlocksPerFile = headerObj.header['NBLOCK']

3114

self.dataBlocksPerFile = headerObj.header['NBLOCK']

3114

self.timeZone = headerObj.header['TIMEZONE']

3115

self.timeZone = headerObj.header['TIMEZONE']

3115

3116

self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt

3117

self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt

3117

3118

if 'COMMENT' in headerObj.header.keys():

3119

if 'COMMENT' in headerObj.header.keys():

3119

self.comments = headerObj.header['COMMENT']

3120

self.comments = headerObj.header['COMMENT']

3120

3121

self.readHeightList()

3122

self.readHeightList()

3122

3123

def readHeightList(self):

3124

def readHeightList(self):

3124

self.blockIndex = self.blockIndex + 1

3125

self.blockIndex = self.blockIndex + 1

3125

obj = self.fitsObj[self.blockIndex]

3126

obj = self.fitsObj[self.blockIndex]

3126

self.heightList = obj.data

3127

self.heightList = obj.data

3127

self.blockIndex = self.blockIndex + 1

3128

self.blockIndex = self.blockIndex + 1

3128

3129

def readExtension(self):

3130

def readExtension(self):

3130

obj = self.fitsObj[self.blockIndex]

3131

obj = self.fitsObj[self.blockIndex]

3131

self.heightList = obj.data

3132

self.heightList = obj.data

3132

self.blockIndex = self.blockIndex + 1

3133

self.blockIndex = self.blockIndex + 1

3133

3134

def setNextFile(self):

3135

def setNextFile(self):

3135

3136

if self.online:

3137

if self.online:

3137

newFile = self.__setNextFileOnline()

3138

newFile = self.__setNextFileOnline()

3138

else:

3139

else:

3139

newFile = self.__setNextFileOffline()

3140

newFile = self.__setNextFileOffline()

3140

3141

if not(newFile):

3142

if not(newFile):

3142

return 0

3143

return 0

3143

3144

self.readHeader()

3145

self.readHeader()

3145

3146

self.nReadBlocks = 0

3147

self.nReadBlocks = 0

3147

# self.blockIndex = 1

3148

# self.blockIndex = 1

3148

return 1

3149

return 1

3149

3150

def __searchFilesOffLine(self,

3151

def __searchFilesOffLine(self,

3151

path,

3152

path,

3152

startDate,

3153

startDate,

3153

endDate,

3154

endDate,

3154

startTime=datetime.time(0,0,0),

3155

startTime=datetime.time(0,0,0),

3155

endTime=datetime.time(23,59,59),

3156

endTime=datetime.time(23,59,59),

3156

set=None,

3157

set=None,

3157

expLabel='',

3158

expLabel='',

3158

ext='.fits',

3159

ext='.fits',

3159

walk=True):

3160

walk=True):

3160

3161

pathList = []

3162

pathList = []

3162

3163

if not walk:

3164

if not walk:

3164

pathList.append(path)

3165

pathList.append(path)

3165

3166

else:

3167

else:

3167

dirList = []

3168

dirList = []

3168

for thisPath in os.listdir(path):

3169

for thisPath in os.listdir(path):

3169

if not os.path.isdir(os.path.join(path,thisPath)):

3170

if not os.path.isdir(os.path.join(path,thisPath)):

3170

continue

3171

continue

3171

if not isDoyFolder(thisPath):

3172

if not isDoyFolder(thisPath):

3172

continue

3173

continue

3173

3174

dirList.append(thisPath)

3175

dirList.append(thisPath)

3175

3176

if not(dirList):

3177

if not(dirList):

3177

return None, None

3178

return None, None

3178

3179

thisDate = startDate

3180

thisDate = startDate

3180

3181

while(thisDate <= endDate):

3182

while(thisDate <= endDate):

3182

year = thisDate.timetuple().tm_year

3183

year = thisDate.timetuple().tm_year

3183

doy = thisDate.timetuple().tm_yday

3184

doy = thisDate.timetuple().tm_yday

3184

3185

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

3186

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

3186

if len(matchlist) == 0:

3187

if len(matchlist) == 0:

3187

thisDate += datetime.timedelta(1)

3188

thisDate += datetime.timedelta(1)

3188

continue

3189

continue

3189

for match in matchlist:

3190

for match in matchlist:

3190

pathList.append(os.path.join(path,match,expLabel))

3191

pathList.append(os.path.join(path,match,expLabel))

3191

3192

thisDate += datetime.timedelta(1)

3193

thisDate += datetime.timedelta(1)

3193

3194

if pathList == []:

3195

if pathList == []:

3195

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

3196

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

3196

return None, None

3197

return None, None

3197

3198

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

3199

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

3199

3200

filenameList = []

3201

filenameList = []

3201

datetimeList = []

3202

datetimeList = []

3202

3203

for i in range(len(pathList)):

3204

for i in range(len(pathList)):

3204

3205

thisPath = pathList[i]

3206

thisPath = pathList[i]

3206

3207

fileList = glob.glob1(thisPath, "*%s" %ext)

3208

fileList = glob.glob1(thisPath, "*%s" %ext)

3208

fileList.sort()

3209

fileList.sort()

3209

3210

for file in fileList:

3211

for file in fileList:

3211

3212

filename = os.path.join(thisPath,file)

3213

filename = os.path.join(thisPath,file)

3213

thisDatetime = self.isFileinThisTime(filename, startTime, endTime)

3214

thisDatetime = self.isFileinThisTime(filename, startTime, endTime)

3214

3215

if not(thisDatetime):

3216

if not(thisDatetime):

3216

continue

3217

continue

3217

3218

filenameList.append(filename)

3219

filenameList.append(filename)

3219

datetimeList.append(thisDatetime)

3220

datetimeList.append(thisDatetime)

3220

3221

if not(filenameList):

3222

if not(filenameList):

3222

print "Any file was found for the time range %s - %s" %(startTime, endTime)

3223

print "Any file was found for the time range %s - %s" %(startTime, endTime)

3223

return None, None

3224

return None, None

3224

3225

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

3226

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

3226

print

3227

print

3227

3228

for i in range(len(filenameList)):

3229

for i in range(len(filenameList)):

3229

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

3230

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

3230

3231

self.filenameList = filenameList

3232

self.filenameList = filenameList

3232

self.datetimeList = datetimeList

3233

self.datetimeList = datetimeList

3233

3234

return pathList, filenameList

3235

return pathList, filenameList

3235

3236

def setup(self, path=None,

3237

def setup(self, path=None,

3237

startDate=None,

3238

startDate=None,

3238

endDate=None,

3239

endDate=None,

3239

startTime=datetime.time(0,0,0),

3240

startTime=datetime.time(0,0,0),

3240

endTime=datetime.time(23,59,59),

3241

endTime=datetime.time(23,59,59),

3241

set=0,

3242

set=0,

3242

expLabel = "",

3243

expLabel = "",

3243

ext = None,

3244

ext = None,

3244

online = False,

3245

online = False,

3245

delay = 60,

3246

delay = 60,

3246

walk = True):

3247

walk = True):

3247

3248

if path == None:

3249

if path == None:

3249

raise ValueError, "The path is not valid"

3250

raise ValueError, "The path is not valid"

3250

3251

if ext == None:

3252

if ext == None:

3252

ext = self.ext

3253

ext = self.ext

3253

3254

if not(online):

3255

if not(online):

3255

print "Searching files in offline mode ..."

3256

print "Searching files in offline mode ..."

3256

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

3257

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

3257

startTime=startTime, endTime=endTime,

3258

startTime=startTime, endTime=endTime,

3258

set=set, expLabel=expLabel, ext=ext,

3259

set=set, expLabel=expLabel, ext=ext,

3259

walk=walk)

3260

walk=walk)

3260

3261

if not(pathList):

3262

if not(pathList):

3262

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

3263

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

3263

datetime.datetime.combine(startDate,startTime).ctime(),

3264

datetime.datetime.combine(startDate,startTime).ctime(),

3264

datetime.datetime.combine(endDate,endTime).ctime())

3265

datetime.datetime.combine(endDate,endTime).ctime())

3265

3266

sys.exit(-1)

3267

sys.exit(-1)

3267

3268

self.fileIndex = -1

3269

self.fileIndex = -1

3269

self.pathList = pathList

3270

self.pathList = pathList

3270

self.filenameList = filenameList

3271

self.filenameList = filenameList

3271

3272

self.online = online

3273

self.online = online

3273

self.delay = delay

3274

self.delay = delay

3274

ext = ext.lower()

3275

ext = ext.lower()

3275

self.ext = ext

3276

self.ext = ext

3276

3277

if not(self.setNextFile()):

3278

if not(self.setNextFile()):

3278

if (startDate!=None) and (endDate!=None):

3279

if (startDate!=None) and (endDate!=None):

3279

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

3280

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

3280

elif startDate != None:

3281

elif startDate != None:

3281

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

3282

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

3282

else:

3283

else:

3283

print "No files"

3284

print "No files"

3284

3285

sys.exit(-1)

3286

sys.exit(-1)

3286

3287

3288

3289

def readBlock(self):

3290

def readBlock(self):

3290

dataObj = self.fitsObj[self.blockIndex]

3291

dataObj = self.fitsObj[self.blockIndex]

3291

3292

self.data = dataObj.data

3293

self.data = dataObj.data

3293

self.data_header_dict = dataObj.header

3294

self.data_header_dict = dataObj.header

3294

self.utc = self.data_header_dict['UTCTIME']

3295

self.utc = self.data_header_dict['UTCTIME']

3295

3296

self.flagIsNewFile = 0

3297

self.flagIsNewFile = 0

3297

self.blockIndex += 1

3298

self.blockIndex += 1

3298

self.nTotalBlocks += 1

3299

self.nTotalBlocks += 1

3299

self.nReadBlocks += 1

3300

self.nReadBlocks += 1

3300

3301

return 1

3302

return 1

3302

3303

def __jumpToLastBlock(self):

3304

def __jumpToLastBlock(self):

3304

raise ValueError, "No implemented"

3305

raise ValueError, "No implemented"

3305

3306

def __waitNewBlock(self):

3307

def __waitNewBlock(self):

3307

"""

3308

"""

3308

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

3309

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

3309

3310

Si el modo de lectura es OffLine siempre retorn 0

3311

Si el modo de lectura es OffLine siempre retorn 0

3311

"""

3312

"""

3312

if not self.online:

3313

if not self.online:

3313

return 0

3314

return 0

3314

3315

if (self.nReadBlocks >= self.dataBlocksPerFile):

3316

if (self.nReadBlocks >= self.dataBlocksPerFile):

3316

return 0

3317

return 0

3317

3318

currentPointer = self.fp.tell()

3319

currentPointer = self.fp.tell()

3319

3320

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

3321

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

3321

3322

for nTries in range( self.nTries ):

3323

for nTries in range( self.nTries ):

3323

3324

self.fp.close()

3325

self.fp.close()

3325

self.fp = open( self.filename, 'rb' )

3326

self.fp = open( self.filename, 'rb' )

3326

self.fp.seek( currentPointer )

3327

self.fp.seek( currentPointer )

3327

3328

self.fileSize = os.path.getsize( self.filename )

3329

self.fileSize = os.path.getsize( self.filename )

3329

currentSize = self.fileSize - currentPointer

3330

currentSize = self.fileSize - currentPointer

3330

3331

if ( currentSize >= neededSize ):

3332

if ( currentSize >= neededSize ):

3332

self.__rdBasicHeader()

3333

self.__rdBasicHeader()

3333

return 1

3334

return 1

3334

3335

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

3336

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

3336

time.sleep( self.delay )

3337

time.sleep( self.delay )

3337

3338

3339

return 0

3340

return 0

3340

3341

def __setNewBlock(self):

3342

def __setNewBlock(self):

3342

3343

if self.online:

3344

if self.online:

3344

self.__jumpToLastBlock()

3345

self.__jumpToLastBlock()

3345

3346

if self.flagIsNewFile:

3347

if self.flagIsNewFile:

3347

return 1

3348

return 1

3348

3349

self.lastUTTime = self.utc

3350

self.lastUTTime = self.utc

3350

3351

if self.online:

3352

if self.online:

3352

if self.__waitNewBlock():

3353

if self.__waitNewBlock():

3353

return 1

3354

return 1

3354

3355

if self.nReadBlocks < self.dataBlocksPerFile:

3356

if self.nReadBlocks < self.dataBlocksPerFile:

3356

return 1

3357

return 1

3357

3358

if not(self.setNextFile()):

3359

if not(self.setNextFile()):

3359

return 0

3360

return 0

3360

3361

deltaTime = self.utc - self.lastUTTime

3362

deltaTime = self.utc - self.lastUTTime

3362

3363

self.flagTimeBlock = 0

3364

self.flagTimeBlock = 0

3364

3365

if deltaTime > self.maxTimeStep:

3366

if deltaTime > self.maxTimeStep:

3366

self.flagTimeBlock = 1

3367

self.flagTimeBlock = 1

3367

3368

return 1

3369

return 1

3369

3370

3371

def readNextBlock(self):

3372

def readNextBlock(self):

3372

if not(self.__setNewBlock()):

3373

if not(self.__setNewBlock()):

3373

return 0

3374

return 0

3374

3375

if not(self.readBlock()):

3376

if not(self.readBlock()):

3376

return 0

3377

return 0

3377

3378

return 1

3379

return 1

3379

3380

3381

def getData(self):

3382

def getData(self):

3382

3383

if self.flagNoMoreFiles:

3384

if self.flagNoMoreFiles:

3384

self.dataOut.flagNoData = True

3385

self.dataOut.flagNoData = True

3385

print 'Process finished'

3386

print 'Process finished'

3386

return 0

3387

return 0

3387

3388

self.flagTimeBlock = 0

3389

self.flagTimeBlock = 0

3389

self.flagIsNewBlock = 0

3390

self.flagIsNewBlock = 0

3390

3391

if not(self.readNextBlock()):

3392

if not(self.readNextBlock()):

3392

return 0

3393

return 0

3393

3394

if self.data == None:

3395

if self.data == None:

3395

self.dataOut.flagNoData = True

3396

self.dataOut.flagNoData = True

3396

return 0

3397

return 0

3397

3398

self.dataOut.data = self.data

3399

self.dataOut.data = self.data

3399

self.dataOut.data_header = self.data_header_dict

3400

self.dataOut.data_header = self.data_header_dict

3400

self.dataOut.utctime = self.utc

3401

self.dataOut.utctime = self.utc

3401

3402

self.dataOut.header = self.header_dict

3403

self.dataOut.header = self.header_dict

3403

self.dataOut.expName = self.expName

3404

self.dataOut.expName = self.expName

3404

self.dataOut.nChannels = self.nChannels

3405

self.dataOut.nChannels = self.nChannels

3405

self.dataOut.timeZone = self.timeZone

3406

self.dataOut.timeZone = self.timeZone

3406

self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile

3407

self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile

3407

self.dataOut.comments = self.comments

3408

self.dataOut.comments = self.comments

3408

self.dataOut.timeInterval = self.timeInterval

3409

self.dataOut.timeInterval = self.timeInterval

3409

self.dataOut.channelList = self.channelList

3410

self.dataOut.channelList = self.channelList

3410

self.dataOut.heightList = self.heightList

3411

self.dataOut.heightList = self.heightList

3411

self.dataOut.flagNoData = False

3412

self.dataOut.flagNoData = False

3412

3413

return self.dataOut.data

3414

return self.dataOut.data

3414

3415

def run(self, **kwargs):

3416

def run(self, **kwargs):

3416

3417

if not(self.isConfig):

3418

if not(self.isConfig):

3418

self.setup(**kwargs)

3419

self.setup(**kwargs)

3419

self.isConfig = True

3420

self.isConfig = True

3420

3421

self.getData() No newline at end of file

3422

self.getData()

3423

3424

3425

class RadacHeader():

3426

def __init__(self, fp):

3427

header = 'Raw11/Data/RadacHeader'

3428

self.beamCode = fp.get(header+'/BeamCode')

3429

self.code = fp.get(header+'/Code')

3430

self.frameCount = fp.get(header+'/FrameCount')

3431

self.modeGroup = fp.get(header+'/ModeGroup')

3432

self.nsamplesPulse = fp.get(header+'/NSamplesPulse')

3433

self.pulseCount = fp.get(header+'/PulseCount')

3434

self.radacTime = fp.get(header+'/RadacTime')

3435

self.timeCount = fp.get(header+'/TimeCount')

3436

self.timeStatus = fp.get(header+'/TimeStatus')

3437

3438

self.nrecords = self.pulseCount.shape[0] #numero de bloques

3439

self.npulses = self.pulseCount.shape[1] #numero de perfiles

3440

self.nsamples = self.nsamplesPulse[0,0] #numero de alturas

3441

3442

3443

def getIndexRangeToPulse(self, idrecord=0):

3444

indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)

3445

startPulseCountId = indexToZero[0][0]

3446

endPulseCountId = startPulseCountId - 1

3447

range1 = numpy.arange(startPulseCountId,self.npulses,1)

3448

range2 = numpy.arange(0,startPulseCountId,1)

3449

return range1, range2

3450

3451

3452

class AMISRReader(ProcessingUnit):

3453

3454

path = None

3455

startDate = None

3456

endDate = None

3457

startTime = None

3458

endTime = None

3459

walk = None

3460

isConfig = False

3461

3462

def __init__(self):

3463

self.set = None

3464

self.subset = None

3465

self.extension_file = '.h5'

3466

self.dtc_str = 'dtc'

3467

self.dtc_id = 0

3468

self.status = True

3469

self.isConfig = False

3470

self.dirnameList = []

3471

self.filenameList = []

3472

self.fileIndex = None

3473

self.flagNoMoreFiles = False

3474

self.flagIsNewFile = 0

3475

self.filename = ''

3476

self.amisrFilePointer = None

3477

self.radacHeaderObj = None

3478

self.dataOut = self.__createObjByDefault()

3479

self.datablock = None

3480

self.rest_datablock = None

3481

self.range = None

3482

self.idrecord_count = 0

3483

self.profileIndex = 0

3484

self.idpulse_range1 = None

3485

self.idpulse_range2 = None

3486

self.beamCodeByFrame = None

3487

self.radacTimeByFrame = None

3488

#atributos originales tal y como esta en el archivo de datos

3489

self.beamCodesFromFile = None

3490

self.radacTimeFromFile = None

3491

self.rangeFromFile = None

3492

self.dataByFrame = None

3493

self.dataset = None

3494

3495

3496

def __createObjByDefault(self):

3497

3498

dataObj = AMISR()

3499

3500

return dataObj

3501

3502

def __setParameters(self,path,startDate,endDate,startTime,endTime,walk):

3503

self.path = path

3504

self.startDate = startDate

3505

self.endDate = endDate

3506

self.startTime = startTime

3507

self.endTime = endTime

3508

self.walk = walk

3509

3510

def __checkPath(self):

3511

if os.path.exists(self.path):

3512

self.status = 1

3513

else:

3514

self.status = 0

3515

print 'Path:%s does not exists'%self.path

3516

3517

return

3518

3519

def __selDates(self, amisr_dirname_format):

3520

year = int(amisr_dirname_format[0:4])

3521

month = int(amisr_dirname_format[4:6])

3522

dom = int(amisr_dirname_format[6:8])

3523

thisDate = datetime.date(year,month,dom)

3524

3525

if (thisDate>=self.startDate and thisDate <= self.endDate):

3526

return amisr_dirname_format

3527

3528

def __findDataForDates(self):

3529

3530

import re

3531

3532

if not(self.status):

3533

return None

3534

3535

pat = '\d+.\d+'

3536

dirnameList = [re.search(pat,x).string for x in os.listdir(self.path)]

3537

dirnameList = [self.__selDates(x) for x in dirnameList]

3538

dirnameList = filter(lambda x:x!=None,dirnameList)

3539

if len(dirnameList)>0:

3540

self.status = 1

3541

self.dirnameList = dirnameList

3542

self.dirnameList.sort()

3543

else:

3544

self.status = 0

3545

return None

3546

3547

def __getTimeFromData(self):

3548

pass

3549

3550

def __filterByGlob1(self, dirName):

3551

filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)

3552

filterDict = {}

3553

filterDict.setdefault(dirName)

3554

filterDict[dirName] = filter_files

3555

return filterDict

3556

3557

def __getFilenameList(self, fileListInKeys, dirList):

3558

for value in fileListInKeys:

3559

dirName = value.keys()[0]

3560

for file in value[dirName]:

3561

filename = os.path.join(dirName, file)

3562

self.filenameList.append(filename)

3563

3564

3565

def __selectDataForTimes(self):

3566

#aun no esta implementado el filtro for tiempo

3567

if not(self.status):

3568

return None

3569

3570

dirList = [os.path.join(self.path,x) for x in self.dirnameList]

3571

3572

fileListInKeys = [self.__filterByGlob1(x) for x in dirList]

3573

3574

self.__getFilenameList(fileListInKeys, dirList)

3575

3576

if len(self.filenameList)>0:

3577

self.status = 1

3578

self.filenameList.sort()

3579

else:

3580

self.status = 0

3581

return None

3582

3583

3584

def __searchFilesOffline(self,

3585

path,

3586

startDate,

3587

endDate,

3588

startTime=datetime.time(0,0,0),

3589

endTime=datetime.time(23,59,59),

3590

walk=True):

3591

3592

self.__setParameters(path, startDate, endDate, startTime, endTime, walk)

3593

3594

self.__checkPath()

3595

3596

self.__findDataForDates()

3597

3598

self.__selectDataForTimes()

3599

3600

for i in range(len(self.filenameList)):

3601

print "%s" %(self.filenameList[i])

3602

3603

return

3604

3605

def __setNextFileOffline(self):

3606

idFile = self.fileIndex

3607

3608

while (True):

3609

idFile += 1

3610

if not(idFile < len(self.filenameList)):

3611

self.flagNoMoreFiles = 1

3612

print "No more Files"

3613

return 0

3614

3615

filename = self.filenameList[idFile]

3616

3617

amisrFilePointer = h5py.File(filename,'r')

3618

3619

break

3620

3621

self.flagIsNewFile = 1

3622

self.fileIndex = idFile

3623

self.filename = filename

3624

3625

self.amisrFilePointer = amisrFilePointer

3626

3627

print "Setting the file: %s"%self.filename

3628

3629

return 1

3630

3631

def __readHeader(self):

3632

self.radacHeaderObj = RadacHeader(self.amisrFilePointer)

3633

self.flagIsNewFile = 1

3634

3635

3636

3637

def __setNextFile(self):

3638

3639

newFile = self.__setNextFileOffline()

3640

3641

if not(newFile):

3642

return 0

3643

3644

self.__readHeader()

3645

3646

self.readDataBlock()

3647

3648

3649

def setup(self,path=None,

3650

startDate=None,

3651

endDate=None,

3652

startTime=datetime.time(0,0,0),

3653

endTime=datetime.time(23,59,59),

3654

walk=True):

3655

3656

#Busqueda de archivos offline

3657

self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)

3658

3659

if not(self.filenameList):

3660

print "There is no files into the folder: %s"%(path)

3661

3662

sys.exit(-1)

3663

3664

self.fileIndex = -1

3665

3666

self.__setNextFile()

3667

3668

def readRanges(self):

3669

dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')

3670

self.rangeFromFile = dataset.values()

3671

return range

3672

3673

3674

def readRadacTime(self,idrecord, range1, range2):

3675

self.radacTimeFromFile = self.radacHeaderObj.radacTime.value

3676

3677

radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))

3678

#radacTimeByFrame = dataset[idrecord - 1,range1]

3679

#radacTimeByFrame = dataset[idrecord,range2]

3680

3681

return radacTimeByFrame

3682

3683

def readBeamCode(self, idrecord, range1, range2):

3684

dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')

3685

beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))

3686

self.beamCodesFromFile = dataset.value

3687

3688

#beamcodeByFrame[range1] = dataset[idrecord - 1, range1]

3689

#beamcodeByFrame[range2] = dataset[idrecord, range2]

3690

beamcodeByFrame[range1] = dataset[idrecord, range1]

3691

beamcodeByFrame[range2] = dataset[idrecord, range2]

3692

3693

return beamcodeByFrame

3694

3695

3696

def __setDataByFrame(self):

3697

ndata = 2 # porque es complejo

3698

dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))

3699

return dataByFrame

3700

3701

def __readDataSet(self):

3702

dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')

3703

return dataset

3704

3705

def __setDataBlock(self,):

3706

real = self.dataByFrame[:,:,0] #asumo que 0 es real

3707

imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario

3708

datablock = real + imag*1j #armo el complejo

3709

return datablock

3710

3711

def readSamples_version1(self,idrecord):

3712

#estas tres primeras lineas solo se deben ejecutar una vez

3713

if self.flagIsNewFile:

3714

self.idpulse_range1, self.idpulse_range2 = self.radacHeaderObj.getIndexRangeToPulse(0)

3715

self.dataByFrame = self.__setDataByFrame()

3716

self.beamCodeByFrame = self.readBeamCode(idrecord, self.idpulse_range1, self.idpulse_range2)

3717

self.radacTimeByFrame = self.readRadacTime(idrecord, self.idpulse_range1, self.idpulse_range2)

3718

#reading dataset

3719

self.dataset = self.__readDataSet()

3720

3721

if idrecord == 0:

3722

3723

if len(numpy.where(self.dataByFrame!=0.0)[0]) or len(numpy.where(self.dataByFrame!=0.0)[1]) or len(numpy.where(self.dataByFrame!=0.0)[2]):

3724

#falta agregar una condicion para datos discontinuos

3725

#por defecto une los datos del record anterior

3726

self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :]

3727

#timepulse

3728

self.radacTimeByFrame[self.idpulse_range2] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range2]

3729

else:

3730

self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :]

3731

3732

self.radacTimeByFrame[self.idpulse_range1] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range1]

3733

3734

datablock = self.__setDataBlock()

3735

3736

return datablock

3737

3738

self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord - 1,self.idpulse_range1, :, :]

3739

self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :]

3740

datablock = self.__setDataBlock()

3741

self.flagIsNewFile = 0

3742

3743

self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :]

3744

3745

3746

return datablock

3747

3748

3749

def readSamples(self,idrecord):

3750

if self.flagIsNewFile:

3751

self.dataByFrame = self.__setDataByFrame()

3752

self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]

3753

3754

#reading dataset

3755

self.dataset = self.__readDataSet()

3756

3757

self.flagIsNewFile = 0

3758

self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]

3759

self.dataByFrame = self.dataset[idrecord, :, :, :]

3760

datablock = self.__setDataBlock()

3761

return datablock

3762

3763

3764

def readDataBlock(self):

3765

3766

#self.datablock = self.readSamples(self.idrecord_count)

3767

self.datablock = self.readSamples(self.idrecord_count)

3768

#print 'record:', self.idrecord_count

3769

3770

self.idrecord_count += 1

3771

self.profileIndex = 0

3772

3773

if self.idrecord_count >= self.radacHeaderObj.nrecords:

3774

self.idrecord_count = 0

3775

self.flagIsNewFile = 1

3776

3777

def readNextBlock(self):

3778

3779

self.readDataBlock()

3780

3781

if self.flagIsNewFile:

3782

self.__setNextFile()

3783

pass

3784

3785

def __hasNotDataInBuffer(self):

3786

#self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record

3787

if self.profileIndex >= self.radacHeaderObj.npulses:

3788

return 1

3789

return 0

3790

3791

3792

def getData(self):

3793

3794

if self.flagNoMoreFiles:

3795

self.dataOut.flagNoData = True

3796

print 'Process finished'

3797

return 0

3798

3799

if self.__hasNotDataInBuffer():

3800

self.readNextBlock()

3801

# if not( self.readNextBlock() ):

3802

# return 0

3803

# self.getFirstHeader()

3804

3805

if self.datablock == None: # setear esta condicion cuando no hayan datos por leers

3806

self.dataOut.flagNoData = True

3807

return 0

3808

3809

self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))

3810

3811

self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]

3812

3813

self.dataOut.flagNoData = False

3814

3815

self.profileIndex += 1

3816

3817

return self.dataOut.data

3818

3819

3820

def run(self, **kwargs):

3821

if not(self.isConfig):

3822

self.setup(**kwargs)

3823

self.isConfig = True

3824

3825

self.getData()

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages