schain Commit - r351:a731a256fc18 · Jicamarca Repository

             '''
             $Author: murco $
             $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
             '''
             import os, sys
             import glob
             import time
             import numpy
             import fnmatch
             import time, datetime
             try:
                 import pyfits
             except:
                 print "pyfits module has not been imported, it should be installed to save files in fits format"
             from jrodata import *
             from jroheaderIO import *
             from jroprocessing import *
             LOCALTIME = True #-18000
             def isNumber(str):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float( str )
                     return True
                 except:
                     return False
             def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename,'rb')
                 except:
                     raise IOError, "The file %s can't be opened" %(filename)
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isFileinThisTime(filename, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename,'rb')
                 except:
                     raise IOError, "The file %s can't be opened" %(filename)
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = basicHeaderObj.datatime
                 thisTime = basicHeaderObj.datatime.time()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return None
                 if not ((startTime <= thisTime) and (endTime > thisTime)):
                     return None
                 return thisDatetime
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for file in fileList:
                     try:
                         year = int(file[1:5])
                         doy  = int(file[5:8])
                     except:
                         continue
                     if (os.path.splitext(file)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(file)
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def checkForRealPath(path, foldercounter, year, doy, set, ext):
                 """
                 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
                 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
                 el path exacto de un determinado file.
                 Example    :
                     nombre correcto del file es  .../.../D2009307/P2009307367.ext
                     Entonces la funcion prueba con las siguientes combinaciones
                         .../.../y2009307367.ext
                         .../.../Y2009307367.ext
                         .../.../x2009307/y2009307367.ext
                         .../.../x2009307/Y2009307367.ext
                         .../.../X2009307/y2009307367.ext
                         .../.../X2009307/Y2009307367.ext
                     siendo para este caso, la ultima combinacion de letras, identica al file buscado
                 Return:
                     Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
                     caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
                     para el filename
                 """
                 fullfilename = None
                 find_flag = False
                 filename = None
                 prefixDirList = [None,'d','D']
                 if ext.lower() == ".r": #voltage
                     prefixFileList = ['d','D']
                 elif ext.lower() == ".pdata": #spectra
                     prefixFileList = ['p','P']
                 else:
                     return None, filename
                 #barrido por las combinaciones posibles
                 for prefixDir in prefixDirList:
                     thispath = path
                     if prefixDir != None:
                         #formo el nombre del directorio xYYYYDDD (x=d o x=D)
                         if foldercounter == 0:
                             thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
                         else:
                             thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
                     for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
                         filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
                         fullfilename = os.path.join( thispath, filename ) #formo el path completo
                         if os.path.exists( fullfilename ): #verifico que exista
                             find_flag = True
                             break
                     if find_flag:
                         break
                 if not(find_flag):
                     return None, filename
                 return fullfilename, filename
             def isDoyFolder(folder):
                 try:
                     year = int(folder[1:5])
                 except:
                     return 0
                 try:
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
             class JRODataIO:
                 c = 3E8
                 isConfig = False
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
                 processingHeaderObj = ProcessingHeader()
                 online = 0
                 dtype = None
                 pathList = []
                 filenameList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagTimeBlock = 0
                 flagIsNewBlock = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
                 ippSeconds = None
                 fileSizeByHeader = None
                 fileIndex = None
                 profileIndex = None
                 blockIndex = None
                 nTotalBlocks = None
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
                 blocksize = None
                 def __init__(self):
                     raise ValueError, "Not implemented"
                 def run(self):
                     raise ValueError, "Not implemented"
                 def getOutput(self):
                     return self.dataOut
             class JRODataReader(JRODataIO, ProcessingUnit):
                 nReadBlocks = 0
                 delay  = 10   #number of seconds waiting a new file
                 nTries  = 3  #quantity tries
                 nFiles = 3   #number of files for searching
                 path = None
                 foldercounter = 0
                 flagNoMoreFiles = 0
                 datetimeList = []
                 __isFirstTimeOnline = 1
                 __printInfo = True
                 profileIndex = None
                 def __init__(self):
                     """
                     """
                     raise ValueError, "This method has not been implemented"
                 def createObjByDefault(self):
                     """
                     """
                     raise ValueError, "This method has not been implemented"
                 def getBlockDimension(self):
                     raise ValueError, "No implemented"
                 def __searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.r',
                                         walk=True):
                     pathList = []
                     if not walk:
                         pathList.append(path)
                     else:
                         dirList = []
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isDoyFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None
                         thisDate = startDate
                         while(thisDate <= endDate):
                             year = thisDate.timetuple().tm_year
                             doy = thisDate.timetuple().tm_yday
                             matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
                             if len(matchlist) == 0:
                                 thisDate += datetime.timedelta(1)
                                 continue
                             for match in matchlist:
                                 pathList.append(os.path.join(path,match,expLabel))
                             thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
                         return None, None
                     print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
                     filenameList = []
                     datetimeList = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             thisDatetime = isFileinThisTime(filename, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "Any file was found for the time range %s - %s" %(startTime, endTime)
                         return None, None
                     print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True):
                     """
                     Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
                     devuelve el archivo encontrado ademas de otros datos.
                     Input:
                         path           :    carpeta donde estan contenidos los files que contiene data
                         expLabel        :     Nombre del subexperimento (subfolder)
                         ext              :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
                         directory   :    eL directorio donde esta el file encontrado
                         filename    :    el ultimo file de una determinada carpeta
                         year        :    el anho
                         doy         :    el numero de dia del anho
                         set         :    el set del archivo
                     """
                     dirList = []
                     if not walk:
                         fullpath = path
                     else:
                         #Filtra solo los directorios
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isDoyFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None, None, None, None
                         dirList = sorted( dirList, key=str.lower )
                         doypath = dirList[-1]
                         foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
                         fullpath = os.path.join(path, doypath, expLabel)
                     print "%s folder was found: " %(fullpath )
                     filename = getlastFileFromPath(fullpath, ext)
                     if not(filename):
                         return None, None, None, None, None
                     print "%s file was found" %(filename)
                     if not(self.__verifyFile(os.path.join(fullpath, filename))):
                         return None, None, None, None, None
                     year = int( filename[1:5] )
                     doy  = int( filename[5:8] )
                     set  = int( filename[8:11] )
                     return fullpath, foldercounter, filename, year, doy, set
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print "No more Files"
                             return 0
                         filename = self.filenameList[idFile]
                         if not(self.__verifyFile(filename)):
                             continue
                         fileSize = os.path.getsize(filename)
                         fp = open(filename,'rb')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fp = fp
                     print "Setting the file: %s"%self.filename
                     return 1
                 def __setNextFileOnline(self):
                     """
                     Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
                     no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
                     siguientes.
                     Affected:
                         self.flagIsNewFile
                         self.filename
                         self.fileSize
                         self.fp
                         self.set
                         self.flagNoMoreFiles
                     Return:
 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
 : si el file fue abierto con exito y esta listo a ser leido
                     Excepciones:
                         Si un determinado file no puede ser abierto
                     """
                     nFiles = 0
                     fileOk_flag = False
                     firstTime_flag = True
                     self.set += 1
                     if self.set > 999:
                         self.set = 0
                         self.foldercounter += 1
                     #busca el 1er file disponible
                     fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                     if fullfilename:
                         if self.__verifyFile(fullfilename, False):
                             fileOk_flag = True
                     #si no encuentra un file entonces espera y vuelve a buscar
                     if not(fileOk_flag):
                         for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
                             if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
                                 tries = self.nTries
                             else:
                                 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
                             for nTries in range( tries ):
                                 if firstTime_flag:
                                     print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
                                     time.sleep( self.delay )
                                 else:
                                     print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
                                 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                                 if fullfilename:
                                     if self.__verifyFile(fullfilename):
                                         fileOk_flag = True
                                         break
                             if fileOk_flag:
                                 break
                             firstTime_flag = False
                             print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename
                             self.set += 1
                             if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
                                 self.set = 0
                                 self.doy += 1
                                 self.foldercounter = 0
                     if fileOk_flag:
                         self.fileSize = os.path.getsize( fullfilename )
                         self.filename = fullfilename
                         self.flagIsNewFile = 1
                         if self.fp != None: self.fp.close()
                         self.fp = open(fullfilename, 'rb')
                         self.flagNoMoreFiles = 0
                         print 'Setting the file: %s' % fullfilename
                     else:
                         self.fileSize = 0
                         self.filename = None
                         self.flagIsNewFile = 0
                         self.fp = None
                         self.flagNoMoreFiles = 1
                         print 'No more Files'
                     return fileOk_flag
                 def setNextFile(self):
                     if self.fp != None:
                         self.fp.close()
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         return 0
                     self.__readFirstHeader()
                     self.nReadBlocks = 0
                     return 1
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.__rdBasicHeader()
                             return 1
                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         time.sleep( self.delay )
                     return 0
                 def __jumpToLastBlock(self):
                     if not(self.__isFirstTimeOnline):
                         return
                     csize = self.fileSize - self.fp.tell()
                     #sata el primer bloque de datos
                     if csize > self.processingHeaderObj.blockSize:
                         self.fp.seek(self.fp.tell() + self.processingHeaderObj.blockSize)
                     else:
                         return
                     csize = self.fileSize - self.fp.tell()
                     neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     factor = int(csize/neededsize)
                     if factor > 0:
                         self.fp.seek(self.fp.tell() + factor*neededsize)
                     self.flagIsNewFile = 0
                     self.__isFirstTimeOnline = 0
                 def __setNewBlock(self):
                     if self.fp == None:
                         return 0
                     if self.online:
                         self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         return 1
                     self.lastUTTime = self.basicHeaderObj.utc
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.__rdBasicHeader()
                         return 1
                     if self.__waitNewBlock():
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
                     self.flagTimeBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagTimeBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def __rdProcessingHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.processingHeaderObj.read(fp)
                 def __rdRadarControllerHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.radarControllerHeaderObj.read(fp)
                 def __rdSystemHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.systemHeaderObj.read(fp)
                 def __rdBasicHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.basicHeaderObj.read(fp)
                 def __readFirstHeader(self):
                     self.__rdBasicHeader()
                     self.__rdSystemHeader()
                     self.__rdRadarControllerHeader()
                     self.__rdProcessingHeader()
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
                     else:
                         raise ValueError, 'Data type was not defined'
                     self.dtype = datatype_str
                     self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
             #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
             #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def __verifyFile(self, filename, msgFlag=True):
                     msg = None
                     try:
                         fp = open(filename, 'rb')
                         currentPosition = fp.tell()
                     except:
                         if msgFlag:
                             print "The file %s can't be opened" % (filename)
                         return False
                     neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                     if neededSize == 0:
                         basicHeaderObj = BasicHeader(LOCALTIME)
                         systemHeaderObj = SystemHeader()
                         radarControllerHeaderObj = RadarControllerHeader()
                         processingHeaderObj = ProcessingHeader()
                         try:
                             if not( basicHeaderObj.read(fp) ): raise IOError
                             if not( systemHeaderObj.read(fp) ): raise IOError
                             if not( radarControllerHeaderObj.read(fp) ): raise IOError
                             if not( processingHeaderObj.read(fp) ): raise IOError
                             data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                             neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
                         except:
                             if msgFlag:
                                 print "\tThe file %s is empty or it hasn't enough data" % filename
                             fp.close()
                             return False
                     else:
                         msg = "\tSkipping the file %s due to it hasn't enough data" %filename
                     fp.close()
                     fileSize = os.path.getsize(filename)
                     currentSize = fileSize - currentPosition
                     if currentSize < neededSize:
                         if msgFlag and (msg != None):
                             print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
                         return False
                     return True
                 def setup(self,
                             path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=0,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True):
                     if path == None:
                         raise ValueError, "The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if online:
                         print "Searching files in online mode..."
                         for nTries in range( self.nTries ):
                             fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk)
                             if fullpath:
                                 break
                             print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
                             time.sleep( self.delay )
                         if not(fullpath):
                             print "There 'isn't valied files in %s" % path
                             return None
                         self.year = year
                         self.doy  = doy
                         self.set  = set - 1
                         self.path = path
                         self.foldercounter = foldercounter
                     else:
                         print "Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
                             print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime())
                             sys.exit(-1)
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                     self.online = online
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "No files"
                         sys.exit(-1)
             #        self.updateDataHeader()
                     return self.dataOut
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds
                     self.dataOut.flagTimeBlock = self.flagTimeBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                 def getFirstHeader(self):
                     raise ValueError, "This method has not been implemented"
                 def getData():
                     raise ValueError, "This method has not been implemented"
                 def hasNotDataInBuffer():
                     raise ValueError, "This method has not been implemented"
                 def readBlock():
                     raise ValueError, "This method has not been implemented"
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print "Number of read blocks per file %04d" %self.nReadBlocks
                 def printTotalBlocks(self):
                     print "Number of read blocks %04d" %self.nTotalBlocks
                 def printNumberOfBlock(self):
                     if self.flagIsNewBlock:
                         print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
                 def printInfo(self):
                     if self.__printInfo == False:
                         return
                     self.basicHeaderObj.printInfo()
                     self.systemHeaderObj.printInfo()
                     self.radarControllerHeaderObj.printInfo()
                     self.processingHeaderObj.printInfo()
                     self.__printInfo = False
                 def run(self, **kwargs):
                     if not(self.isConfig):
             #            self.dataOut = dataOut
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             class JRODataWriter(JRODataIO, Operation):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 blockIndex = 0
                 path = None
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 def __init__(self, dataOut=None):
                     raise ValueError, "Not implemented"
                 def hasAllDataInBuffer(self):
                     raise ValueError, "Not implemented"
                 def setBlockDimension(self):
                     raise ValueError, "Not implemented"
                 def writeBlock(self):
                     raise ValueError, "No implemented"
                 def putData(self):
                     raise ValueError, "No implemented"
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize #bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond  = (self.dataOut.utctime - utc)* 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
                     raise ValueError, "No implemented"
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                     self.dtype = self.dataOut.dtype
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def setNextFile(self):
                     """
                     Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, file )
                     fp = open( filename,'wb' )
                     self.blockIndex = 0
                     #guardando atributos
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.setFirstHeader()
                     print 'Writing the file: %s'%self.filename
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=None, set=0, ext=None):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path      :    el path destino en el cual se escribiran los files a crear
                         format    :    formato en el cual sera salvado un file
                         set       :    el setebo del file
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     ext = ext.lower()
                     self.ext = ext
                     self.path = path
                     self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     if not(self.setNextFile()):
                         print "There isn't a next file"
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class VoltageReader(JRODataReader):
                 """
                 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
                 perfiles*alturas*canales) son almacenados en la variable "buffer".
                                          perfiles * alturas * canales
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = VoltageReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         #to get one profile
                         profile =  readerObj.getData()
                         #print the profile
                         print profile
                         #If you want to see all datablock
                         print readerObj.datablock
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 ext = ".r"
                 optchar = "D"
                 dataOut = None
                 def __init__(self):
                     """
                     Inicializador de la clase VoltageReader para la lectura de datos de voltage.
                     Input:
                         dataOut    :    Objeto de la clase Voltage. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Variables afectadas:
                         self.dataOut
                     Return:
                         None
                     """
                     self.isConfig = False
                     self.datablock = None
                     self.utc = 0
                     self.ext = ".r"
                     self.optchar = "D"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.online = 0
                     self.fp = None
                     self.idFile = None
                     self.dtype = None
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.filename = None
                     self.fileSize = None
                     self.firstHeaderSize = 0
                     self.basicHeaderSize = 24
                     self.pathList = []
                     self.filenameList = []
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
                     self.flagNoMoreFiles = 0
                     self.set = 0
                     self.path = None
                     self.profileIndex = 2**32-1
                     self.delay  = 3   #seconds
                     self.nTries  = 3  #quantity tries
                     self.nFiles = 3   #number of files for searching
                     self.nReadBlocks = 0
                     self.flagIsNewFile = 1
                     self.__isFirstTimeOnline = 1
                     self.ippSeconds = 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     self.nTotalBlocks = 0
                     self.blocksize = 0
                     self.dataOut = self.createObjByDefault()
                 def createObjByDefault(self):
                     dataObj = Voltage()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
                         return 1
                     return 0
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.blocksize
                     Return:
                         None
                     """
                     pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
                     self.blocksize = pts2read
                 def readBlock(self):
                     """
                     readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Inputs:
                         None
                     Return:
                         None
                     Affected:
                         self.profileIndex
                         self.datablock
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
                     junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
                     try:
                         junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
                     except:
                         print "The read block (%3d) has not enough data" %self.nReadBlocks
                         return 0
                     junk = numpy.transpose(junk, (2,0,1))
                     self.datablock = junk['real'] + junk['imag']*1j
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.dataOut.dtype = self.dtype
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
                     self.dataOut.ippSeconds = self.ippSeconds
                     self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.flagShiftFFT = False
                     if self.radarControllerHeaderObj.code != None:
                         self.dataOut.nCode = self.radarControllerHeaderObj.nCode
                         self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
                         self.dataOut.code = self.radarControllerHeaderObj.code
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip
                     self.dataOut.flagShiftFFT = False
                 def getData(self):
                     """
                     getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Ademas incrementa el contador del buffer en 1.
                     Return:
                         data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                      buffer. Si no hay mas archivos a leer retorna None.
                     Variables afectadas:
                         self.dataOut
                         self.profileIndex
                     Affected:
                         self.dataOut
                         self.profileIndex
                         self.flagTimeBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             return 0
                         self.getFirstHeader()
                     if self.datablock == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.dataOut.data = self.datablock[:,self.profileIndex,:]
                     self.dataOut.flagNoData = False
                     self.getBasicHeader()
                     self.profileIndex += 1
                     return self.dataOut.data
             class VoltageWriter(JRODataWriter):
                 """
                 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".r"
                 optchar = "D"
                 shapeBuffer = None
                 def __init__(self):
                     """
                     Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                     Return: None
                     """
                     self.nTotalBlocks = 0
                     self.profileIndex = 0
                     self.isConfig = False
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.nTotalBlocks = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
                         return 1
                     return 0
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
                                         self.processingHeaderObj.nHeights,
                                         self.systemHeaderObj.nChannels)
                     self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
                                                  self.processingHeaderObj.profilesPerBlock,
                                                  self.processingHeaderObj.nHeights),
                                                  dtype=numpy.dtype('complex64'))
                 def writeBlock(self):
                     """
                     Escribe el buffer en el file designado
                     Affected:
                         self.profileIndex
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.blockIndex
                     Return: None
                     """
                     data = numpy.zeros( self.shapeBuffer, self.dtype )
                     junk = numpy.transpose(self.datablock, (1,2,0))
                     data['real'] = junk.real
                     data['imag'] = junk.imag
                     data = data.reshape( (-1) )
                     data.tofile( self.fp )
                     self.datablock.fill(0)
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.flagIsNewBlock
                         self.profileIndex
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagTimeBlock:
                         self.datablock.fill(0)
                         self.profileIndex = 0
                         self.setNextFile()
                     if self.profileIndex == 0:
                         self.setBasicHeader()
                     self.datablock[:,self.profileIndex,:] = self.dataOut.data
                     self.profileIndex += 1
                     if self.hasAllDataInBuffer():
                         #if self.flagIsNewFile:
                         self.writeNextBlock()
             #            self.setFirstHeader()
                     return 1
                 def __getProcessFlags(self):
                     processFlags = 0
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList =  [PROCFLAG.DATATYPE_CHAR,
                                        PROCFLAG.DATATYPE_SHORT,
                                        PROCFLAG.DATATYPE_LONG,
                                        PROCFLAG.DATATYPE_INT64,
                                        PROCFLAG.DATATYPE_FLOAT,
                                        PROCFLAG.DATATYPE_DOUBLE]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             dtypeValue = datatypeValueList[index]
                             break
                     processFlags += dtypeValue
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code != None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nCohInt > 1:
                         processFlags += PROCFLAG.COHERENT_INTEGRATION
                     return processFlags
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
                     '''
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList = [1,2,4,8,4,8]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             datatypeValue = datatypeValueList[index]
                             break
                     blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.dataOut.nProfiles * datatypeValue * 2)
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.setBasicHeader()
                     processingHeaderSize = 40 # bytes
                     self.processingHeaderObj.dtype = 0 # Voltage
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.processFlags = self.__getProcessFlags()
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
                     self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
                     self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
                     if self.dataOut.code != None:
                         self.processingHeaderObj.code = self.dataOut.code
                         self.processingHeaderObj.nCode = self.dataOut.nCode
                         self.processingHeaderObj.nBaud = self.dataOut.nBaud
                         codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)
                         processingHeaderSize += codesize
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                         processingHeaderSize += 12
                     self.processingHeaderObj.size = processingHeaderSize
             class SpectraReader(JRODataReader):
                 """
                 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
                 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
                                     paresCanalesIguales    * alturas * perfiles  (Self Spectra)
                                     paresCanalesDiferentes * alturas * perfiles  (Cross Spectra)
                                     canales * alturas                            (DC Channels)
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = SpectraReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         readerObj.getData()
                         print readerObj.data_spc
                         print readerObj.data_cspc
                         print readerObj.data_dc
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 pts2read_SelfSpectra = 0
                 pts2read_CrossSpectra = 0
                 pts2read_DCchannels = 0
                 ext = ".pdata"
                 optchar = "P"
                 dataOut = None
                 nRdChannels = None
                 nRdPairs = None
                 rdPairList = []
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraReader para la lectura de datos de espectros.
                     Inputs:
                         dataOut    :    Objeto de la clase Spectra. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Affected:
                         self.dataOut
                     Return      : None
                     """
                     self.isConfig = False
                     self.pts2read_SelfSpectra = 0
                     self.pts2read_CrossSpectra = 0
                     self.pts2read_DCchannels = 0
                     self.datablock = None
                     self.utc = None
                     self.ext = ".pdata"
                     self.optchar = "P"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.online = 0
                     self.fp = None
                     self.idFile = None
                     self.dtype = None
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.filename = None
                     self.fileSize = None
                     self.firstHeaderSize = 0
                     self.basicHeaderSize = 24
                     self.pathList = []
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
                     self.flagNoMoreFiles = 0
                     self.set = 0
                     self.path = None
                     self.delay  = 60   #seconds
                     self.nTries = 3  #quantity tries
                     self.nFiles = 3   #number of files for searching
                     self.nReadBlocks = 0
                     self.flagIsNewFile = 1
                     self.__isFirstTimeOnline = 1
                     self.ippSeconds = 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     self.nTotalBlocks = 0
                     self.blocksize = 0
                     self.dataOut = self.createObjByDefault()
                     self.profileIndex = 1 #Always
                 def createObjByDefault(self):
                     dataObj = Spectra()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     return 1
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.nRdChannels
                         self.nRdPairs
                         self.pts2read_SelfSpectra
                         self.pts2read_CrossSpectra
                         self.pts2read_DCchannels
                         self.blocksize
                         self.dataOut.nChannels
                         self.dataOut.nPairs
                     Return:
                         None
                     """
                     self.nRdChannels = 0
                     self.nRdPairs = 0
                     self.rdPairList = []
                     for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
                         if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
                             self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
                         else:
                             self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
                             self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
                     pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
                     self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
                     self.blocksize = self.pts2read_SelfSpectra
                     if self.processingHeaderObj.flag_cspc:
                         self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
                         self.blocksize += self.pts2read_CrossSpectra
                     if self.processingHeaderObj.flag_dc:
                         self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
                         self.blocksize += self.pts2read_DCchannels
             #        self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
                 def readBlock(self):
                     """
                     Lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Return: None
                     Variables afectadas:
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
                     blockOk_flag = False
                     fpointer = self.fp.tell()
                     spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
                     spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
                         cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_dc:
                         dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
                         dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
                     if not(self.processingHeaderObj.shif_fft):
                         #desplaza a la derecha en el eje 2 determinadas posiciones
                         shift = int(self.processingHeaderObj.profilesPerBlock/2)
                         spc = numpy.roll( spc, shift , axis=2 )
                         if self.processingHeaderObj.flag_cspc:
                             #desplaza a la derecha en el eje 2 determinadas posiciones
                             cspc = numpy.roll( cspc, shift, axis=2 )
             #            self.processingHeaderObj.shif_fft = True
                     spc = numpy.transpose( spc, (0,2,1) )
                     self.data_spc = spc
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.transpose( cspc, (0,2,1) )
                         self.data_cspc = cspc['real'] + cspc['imag']*1j
                     else:
                         self.data_cspc = None
                     if self.processingHeaderObj.flag_dc:
                         self.data_dc = dc['real'] + dc['imag']*1j
                     else:
                         self.data_dc = None
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.dataOut.dtype = self.dtype
                     self.dataOut.nPairs = self.nRdPairs
                     self.dataOut.pairsList = self.rdPairList
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
                     self.dataOut.ippSeconds = self.ippSeconds
                     self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
                     self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip
                     if self.processingHeaderObj.code != None:
                         self.dataOut.nCode = self.processingHeaderObj.nCode
                         self.dataOut.nBaud = self.processingHeaderObj.nBaud
                         self.dataOut.code = self.processingHeaderObj.code
                         self.dataOut.flagDecodeData = True
                 def getData(self):
                     """
                     Copia el buffer de lectura a la clase "Spectra",
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Return:
 :    Si no hay mas archivos disponibles
 :    Si hizo una buena copia del buffer
                     Affected:
                         self.dataOut
                         self.flagTimeBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             self.dataOut.flagNoData = True
                             return 0
                     #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
                     if self.data_dc == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.getBasicHeader()
                     self.getFirstHeader()
                     self.dataOut.data_spc = self.data_spc
                     self.dataOut.data_cspc = self.data_cspc
                     self.dataOut.data_dc = self.data_dc
                     self.dataOut.flagNoData = False
                     return self.dataOut.data_spc
             class SpectraWriter(JRODataWriter):
                 """
                 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".pdata"
                 optchar = "P"
                 shape_spc_Buffer = None
                 shape_cspc_Buffer = None
                 shape_dc_Buffer = None
                 data_spc = None
                 data_cspc = None
                 data_dc = None
             #    dataOut = None
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj
                     Return: None
                     """
                     self.isConfig = False
                     self.nTotalBlocks = 0
                     self.data_spc = None
                     self.data_cspc = None
                     self.data_dc = None
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.nTotalBlocks = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.noMoreFiles = 0
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     return 1
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shape_spc_Buffer = (self.dataOut.nChannels,
                                              self.processingHeaderObj.nHeights,
                                              self.processingHeaderObj.profilesPerBlock)
                     self.shape_cspc_Buffer = (self.dataOut.nPairs,
                                               self.processingHeaderObj.nHeights,
                                               self.processingHeaderObj.profilesPerBlock)
                     self.shape_dc_Buffer = (self.dataOut.nChannels,
                                             self.processingHeaderObj.nHeights)
                 def writeBlock(self):
                     """
                     Escribe el buffer en el file designado
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.nWriteBlocks
                     Return: None
                     """
                     spc = numpy.transpose( self.data_spc, (0,2,1) )
                     if not( self.processingHeaderObj.shif_fft ):
                         spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                     data = spc.reshape((-1))
                     data = data.astype(self.dtype[0])
                     data.tofile(self.fp)
                     if self.data_cspc != None:
                         data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
                         cspc = numpy.transpose( self.data_cspc, (0,2,1) )
                         if not( self.processingHeaderObj.shif_fft ):
                             cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                         data['real'] = cspc.real
                         data['imag'] = cspc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     if self.data_dc != None:
                         data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
                         dc = self.data_dc
                         data['real'] = dc.real
                         data['imag'] = dc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     self.data_spc.fill(0)
                     self.data_dc.fill(0)
                     if self.data_cspc != None:
                         self.data_cspc.fill(0)
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nWriteBlocks += 1
                     self.blockIndex += 1
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagTimeBlock:
                         self.data_spc.fill(0)
                         self.data_cspc.fill(0)
                         self.data_dc.fill(0)
                         self.setNextFile()
                     if self.flagIsNewFile == 0:
                         self.setBasicHeader()
                     self.data_spc = self.dataOut.data_spc.copy()
                     self.data_cspc = self.dataOut.data_cspc.copy()
                     self.data_dc = self.dataOut.data_dc.copy()
                     # #self.processingHeaderObj.dataBlocksPerFile)
                     if self.hasAllDataInBuffer():
             #            self.setFirstHeader()
                         self.writeNextBlock()
                     return 1
                 def __getProcessFlags(self):
                     processFlags = 0
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList =  [PROCFLAG.DATATYPE_CHAR,
                                        PROCFLAG.DATATYPE_SHORT,
                                        PROCFLAG.DATATYPE_LONG,
                                        PROCFLAG.DATATYPE_INT64,
                                        PROCFLAG.DATATYPE_FLOAT,
                                        PROCFLAG.DATATYPE_DOUBLE]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             dtypeValue = datatypeValueList[index]
                             break
                     processFlags += dtypeValue
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code != None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nIncohInt > 1:
                         processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                     if self.dataOut.data_dc != None:
                         processFlags += PROCFLAG.SAVE_CHANNELS_DC
                     return processFlags
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
                     '''
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList = [1,2,4,8,4,8]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             datatypeValue = datatypeValueList[index]
                             break
                     pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
                     pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
                     blocksize = (pts2write_SelfSpectra*datatypeValue)
                     if self.dataOut.data_cspc != None:
                         pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
                         blocksize += (pts2write_CrossSpectra*datatypeValue*2)
                     if self.dataOut.data_dc != None:
                         pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
                         blocksize += (pts2write_DCchannels*datatypeValue*2)
                     blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.setBasicHeader()
                     processingHeaderSize = 40 # bytes
                     self.processingHeaderObj.dtype = 1 # Spectra
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.processFlags = self.__getProcessFlags()
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
                     self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
                     self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
                     self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
                     if self.processingHeaderObj.totalSpectra > 0:
                         channelList = []
                         for channel in range(self.dataOut.nChannels):
                             channelList.append(channel)
                             channelList.append(channel)
                         pairsList = []
                         for pair in self.dataOut.pairsList:
                             pairsList.append(pair[0])
                             pairsList.append(pair[1])
                         spectraComb = channelList + pairsList
                         spectraComb = numpy.array(spectraComb,dtype="u1")
                         self.processingHeaderObj.spectraComb = spectraComb
                         sizeOfSpcComb = len(spectraComb)
                         processingHeaderSize += sizeOfSpcComb
             #        The processing header should not have information about code
             #        if self.dataOut.code != None:
             #            self.processingHeaderObj.code = self.dataOut.code
             #            self.processingHeaderObj.nCode = self.dataOut.nCode
             #            self.processingHeaderObj.nBaud = self.dataOut.nBaud
             #            nCodeSize = 4 # bytes
             #            nBaudSize = 4 # bytes
             #            codeSize = 4 # bytes
             #            sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)
             #            processingHeaderSize += sizeOfCode
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                         sizeOfFirstHeight = 4
                         sizeOfdeltaHeight = 4
                         sizeOfnHeights = 4
                         sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows
                         processingHeaderSize += sizeOfWindows
                     self.processingHeaderObj.size = processingHeaderSize
             class SpectraHeisWriter(Operation):
             #    set = None
                 setFile = None
                 idblock = None
                 doypath = None
                 subfolder = None
                 def __init__(self):
                     self.wrObj = FITS()
             #        self.dataOut = dataOut
                     self.nTotalBlocks=0
             #        self.set = None
                     self.setFile = None
                     self.idblock = 0
                     self.wrpath = None
                     self.doypath = None
                     self.subfolder = None
                     self.isConfig = False
                 def isNumber(str):
                     """
                     Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                     Excepciones:
                     Si un determinado string no puede ser convertido a numero
                     Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                     Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                     """
                     try:
                         float( str )
                         return True
                     except:
                         return False
                 def setup(self, dataOut, wrpath):
                     if not(os.path.exists(wrpath)):
                         os.mkdir(wrpath)
                     self.wrpath = wrpath
             #        self.setFile = 0
                     self.dataOut = dataOut
                 def putData(self):
                     name= time.localtime( self.dataOut.utctime)
                     ext=".fits"
                     if self.doypath == None:
                        self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
                        self.doypath = os.path.join( self.wrpath, self.subfolder )
                        os.mkdir(self.doypath)
                     if self.setFile == None:
             #           self.set = self.dataOut.set
                        self.setFile = 0
             #        if self.set != self.dataOut.set:
             ##            self.set = self.dataOut.set
             #            self.setFile = 0
                     #make the filename
                     file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
                     filename = os.path.join(self.wrpath,self.subfolder, file)
                     idblock = numpy.array([self.idblock],dtype="int64")
                     header=self.wrObj.cFImage(idblock=idblock,
                                             year=time.gmtime(self.dataOut.utctime).tm_year,
                                             month=time.gmtime(self.dataOut.utctime).tm_mon,
                                             day=time.gmtime(self.dataOut.utctime).tm_mday,
                                             hour=time.gmtime(self.dataOut.utctime).tm_hour,
                                             minute=time.gmtime(self.dataOut.utctime).tm_min,
                                             second=time.gmtime(self.dataOut.utctime).tm_sec)
                     c=3E8
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
                     colList = []
                     colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
                     colList.append(colFreq)
                     nchannel=self.dataOut.nChannels
                     for i in range(nchannel):
                         col = self.wrObj.writeData(name="PCh"+str(i+1),
                                                      format=str(self.dataOut.nFFTPoints)+'E',
                                                       data=10*numpy.log10(self.dataOut.data_spc[i,:]))
                         colList.append(col)
                     data=self.wrObj.Ctable(colList=colList)
                     self.wrObj.CFile(header,data)
                     self.wrObj.wFile(filename)
                     #update the setFile
                     self.setFile += 1
                     self.idblock += 1
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class FITS:
                 name=None
                 format=None
                 array =None
                 data =None
                 thdulist=None
                 prihdr=None
                 hdu=None
                 def __init__(self):
                     pass
                 def setColF(self,name,format,array):
                     self.name=name
                     self.format=format
                     self.array=array
                     a1=numpy.array([self.array],dtype=numpy.float32)
                     self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
                     return self.col1
             #    def setColP(self,name,format,data):
             #        self.name=name
             #        self.format=format
             #        self.data=data
             #        a2=numpy.array([self.data],dtype=numpy.float32)
             #        self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
             #        return self.col2
                 def writeData(self,name,format,data):
                     self.name=name
                     self.format=format
                     self.data=data
                     a2=numpy.array([self.data],dtype=numpy.float32)
                     self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
                     return self.col2
                 def cFImage(self,idblock,year,month,day,hour,minute,second):
                     self.hdu= pyfits.PrimaryHDU(idblock)
                     self.hdu.header.set("Year",year)
                     self.hdu.header.set("Month",month)
                     self.hdu.header.set("Day",day)
                     self.hdu.header.set("Hour",hour)
                     self.hdu.header.set("Minute",minute)
                     self.hdu.header.set("Second",second)
                     return self.hdu
                 def Ctable(self,colList):
                     self.cols=pyfits.ColDefs(colList)
                     self.tbhdu = pyfits.new_table(self.cols)
                     return self.tbhdu
                 def CFile(self,hdu,tbhdu):
                     self.thdulist=pyfits.HDUList([hdu,tbhdu])
                 def wFile(self,filename):
                     if os.path.isfile(filename):
                         os.remove(filename)
                     self.thdulist.writeto(filename)
+            class ParameterConf:
+                ELEMENTNAME = 'Parameter'
+                def __init__(self):
+                    self.name = ''
+                    self.value = ''
+                def readXml(self, parmElement):
+                    self.name = parmElement.get('name')
+                    self.value = parmElement.get('value')
+                def getElementName(self):
+                    return self.ELEMENTNAME
+            class Metadata:
+                def __init__(self, filename):
+                    self.parmConfObjList = []
+                    self.readXml(filename)
+                def readXml(self, filename):
+                    self.projectElement = None
+                    self.procUnitConfObjDict = {}
+                    self.projectElement = ElementTree().parse(filename)
+                    self.project = self.projectElement.tag
+                    parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
+                    for parmElement in parmElementList:
+                        parmConfObj = ParameterConf()
+                        parmConfObj.readXml(parmElement)
+                        self.parmConfObjList.append(parmConfObj)
+            class FitsWriter(Operation):
+                def __init__(self):
+                    self.isConfig = False
+                    self.dataBlocksPerFile = None
+                    self.blockIndex = 0
+                    self.flagIsNewFile = 1
+                    self.fitsObj = None
+                    self.optchar = 'P'
+                    self.ext = '.fits'
+                    self.setFile = 0
+                def setFitsHeader(self, dataOut, metadatafile):
+                    header_data = pyfits.PrimaryHDU()
+                    metadata4fits = Metadata(metadatafile)
+                    for parameter in metadata4fits.parmConfObjList:
+                        parm_name = parameter.name
+                        parm_value = parameter.value
+                        if parm_value == 'fromdatadatetime':
+                            value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
+                        elif parm_value == 'fromdataheights':
+                            value = dataOut.nHeights
+                        elif parm_value == 'fromdatachannel':
+                            value = dataOut.nChannels
+                        elif parm_value == 'fromdatasamples':
+                            value = dataOut.nFFTPoints
+                        else:
+                            value = parm_value
+                        header_data.header[parm_name] = value
+                    header_data.header['NBLOCK'] = self.blockIndex
+                    header_data.writeto(self.filename)
+                def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):
+                    self.path = path
+                    self.dataOut = dataOut
+                    self.metadatafile = metadatafile
+                    self.dataBlocksPerFile = dataBlocksPerFile
+                def open(self):
+                    self.fitsObj = pyfits.open(self.filename, mode='update')
+                def addData(self, data):
+                    self.open()
+                    extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATA'])
+                    extension.header['UTCTIME'] = self.dataOut.utctime
+                    self.fitsObj.append(extension)
+                    self.blockIndex += 1
+                    self.fitsObj[0].header['NBLOCK'] = self.blockIndex
+                    self.write()
+                def write(self):
+                    self.fitsObj.flush(verbose=True)
+                    self.fitsObj.close()
+                def setNextFile(self):
+                    ext = self.ext
+                    path = self.path
+                    timeTuple = time.localtime( self.dataOut.utctime)
+                    subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
+                    fullpath = os.path.join( path, subfolder )
+                    if not( os.path.exists(fullpath) ):
+                        os.mkdir(fullpath)
+                        self.setFile = -1 #inicializo mi contador de seteo
+                    else:
+                        filesList = os.listdir( fullpath )
+                        if len( filesList ) > 0:
+                            filesList = sorted( filesList, key=str.lower )
+                            filen = filesList[-1]
+                            if isNumber( filen[8:11] ):
+                                self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
+                            else:
+                                self.setFile = -1
+                        else:
+                            self.setFile = -1 #inicializo mi contador de seteo
+                    setFile = self.setFile
+                    setFile += 1
+                    file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
+                                                    timeTuple.tm_year,
+                                                    timeTuple.tm_yday,
+                                                    setFile,
+                                                    ext )
+                    filename = os.path.join( path, subfolder, file )
+                    self.blockIndex = 0
+                    self.filename = filename
+                    self.setFile = setFile
+                    self.flagIsNewFile = 1
+                    print 'Writing the file: %s'%self.filename
+                    self.setFitsHeader(self.dataOut, self.metadatafile)
+                    return 1
+                def writeBlock(self):
+                    self.addData(self.dataOut.data_spc)
+                    self.flagIsNewFile = 0
+                def __setNewBlock(self):
+                    if self.flagIsNewFile:
+                        return 1
+                    if self.blockIndex < self.dataBlocksPerFile:
+                        return 1
+                    if not( self.setNextFile() ):
+                        return 0
+                    return 1
+                def writeNextBlock(self):
+                    if not( self.__setNewBlock() ):
+                        return 0
+                    self.writeBlock()
+                    return 1
+                def putData(self):
+                    if self.flagIsNewFile:
+                        self.setNextFile()
+                    self.writeNextBlock()
+                def run(self, dataOut, **kwargs):
+                    if not(self.isConfig):
+                        self.setup(dataOut, **kwargs)
+                        self.isConfig = True
+                    self.putData()

1

'''

1

'''

2

3

$Author: murco $

3

$Author: murco $

4

$Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $

4

$Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $

5

'''

5

'''

6

7

import os, sys

7

import os, sys

8

import glob

8

import glob

9

import time

9

import time

10

import numpy

10

import numpy

11

import fnmatch

11

import fnmatch

12

import time, datetime

12

import time, datetime

13

try:

13

try:

14

import pyfits

14

import pyfits

15

except:

15

except:

16

print "pyfits module has not been imported, it should be installed to save files in fits format"

16

print "pyfits module has not been imported, it should be installed to save files in fits format"

17

18

from jrodata import *

18

from jrodata import *

19

from jroheaderIO import *

19

from jroheaderIO import *

20

from jroprocessing import *

20

from jroprocessing import *

21

22

LOCALTIME = True #-18000

22

LOCALTIME = True #-18000

23

24

def isNumber(str):

24

def isNumber(str):

25

"""

25

"""

26

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

26

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

27

28

Excepciones:

28

Excepciones:

29

Si un determinado string no puede ser convertido a numero

29

Si un determinado string no puede ser convertido a numero

30

Input:

30

Input:

31

str, string al cual se le analiza para determinar si convertible a un numero o no

31

str, string al cual se le analiza para determinar si convertible a un numero o no

32

33

Return:

33

Return:

34

True : si el string es uno numerico

34

True : si el string es uno numerico

35

False : no es un string numerico

35

False : no es un string numerico

36

"""

36

"""

37

try:

37

try:

38

float( str )

38

float( str )

39

return True

39

return True

40

except:

40

except:

41

return False

41

return False

42

43

def isThisFileinRange(filename, startUTSeconds, endUTSeconds):

43

def isThisFileinRange(filename, startUTSeconds, endUTSeconds):

44

"""

44

"""

45

Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.

45

Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.

46

47

Inputs:

47

Inputs:

48

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

48

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

49

50

startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en

50

startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en

51

segundos contados desde 01/01/1970.

51

segundos contados desde 01/01/1970.

52

endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en

52

endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en

53

segundos contados desde 01/01/1970.

53

segundos contados desde 01/01/1970.

54

55

Return:

55

Return:

56

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

56

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

57

fecha especificado, de lo contrario retorna False.

57

fecha especificado, de lo contrario retorna False.

58

59

Excepciones:

59

Excepciones:

60

Si el archivo no existe o no puede ser abierto

60

Si el archivo no existe o no puede ser abierto

61

Si la cabecera no puede ser leida.

61

Si la cabecera no puede ser leida.

62

63

"""

63

"""

64

basicHeaderObj = BasicHeader(LOCALTIME)

64

basicHeaderObj = BasicHeader(LOCALTIME)

65

66

try:

66

try:

67

fp = open(filename,'rb')

67

fp = open(filename,'rb')

68

except:

68

except:

69

raise IOError, "The file %s can't be opened" %(filename)

69

raise IOError, "The file %s can't be opened" %(filename)

70

71

sts = basicHeaderObj.read(fp)

71

sts = basicHeaderObj.read(fp)

72

fp.close()

72

fp.close()

73

74

if not(sts):

74

if not(sts):

75

print "Skipping the file %s because it has not a valid header" %(filename)

75

print "Skipping the file %s because it has not a valid header" %(filename)

76

return 0

76

return 0

77

78

if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):

78

if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):

79

return 0

79

return 0

80

81

return 1

81

return 1

82

83

def isFileinThisTime(filename, startTime, endTime):

83

def isFileinThisTime(filename, startTime, endTime):

84

"""

84

"""

85

Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.

85

Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.

86

87

Inputs:

87

Inputs:

88

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

88

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

89

90

startTime : tiempo inicial del rango seleccionado en formato datetime.time

90

startTime : tiempo inicial del rango seleccionado en formato datetime.time

91

92

endTime : tiempo final del rango seleccionado en formato datetime.time

92

endTime : tiempo final del rango seleccionado en formato datetime.time

93

94

Return:

94

Return:

95

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

95

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

96

fecha especificado, de lo contrario retorna False.

96

fecha especificado, de lo contrario retorna False.

97

98

Excepciones:

98

Excepciones:

99

Si el archivo no existe o no puede ser abierto

99

Si el archivo no existe o no puede ser abierto

100

Si la cabecera no puede ser leida.

100

Si la cabecera no puede ser leida.

101

102

"""

102

"""

103

104

105

try:

105

try:

106

fp = open(filename,'rb')

106

fp = open(filename,'rb')

107

except:

107

except:

108

raise IOError, "The file %s can't be opened" %(filename)

108

raise IOError, "The file %s can't be opened" %(filename)

109

110

basicHeaderObj = BasicHeader(LOCALTIME)

110

basicHeaderObj = BasicHeader(LOCALTIME)

111

sts = basicHeaderObj.read(fp)

111

sts = basicHeaderObj.read(fp)

112

fp.close()

112

fp.close()

113

114

thisDatetime = basicHeaderObj.datatime

114

thisDatetime = basicHeaderObj.datatime

115

thisTime = basicHeaderObj.datatime.time()

115

thisTime = basicHeaderObj.datatime.time()

116

117

if not(sts):

117

if not(sts):

118

print "Skipping the file %s because it has not a valid header" %(filename)

118

print "Skipping the file %s because it has not a valid header" %(filename)

119

return None

119

return None

120

121

if not ((startTime <= thisTime) and (endTime > thisTime)):

121

if not ((startTime <= thisTime) and (endTime > thisTime)):

122

return None

122

return None

123

124

return thisDatetime

124

return thisDatetime

125

126

def getlastFileFromPath(path, ext):

126

def getlastFileFromPath(path, ext):

127

"""

127

"""

128

Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"

128

Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"

129

al final de la depuracion devuelve el ultimo file de la lista que quedo.

129

al final de la depuracion devuelve el ultimo file de la lista que quedo.

130

131

Input:

131

Input:

132

fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta

132

fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta

133

ext : extension de los files contenidos en una carpeta

133

ext : extension de los files contenidos en una carpeta

134

135

Return:

135

Return:

136

El ultimo file de una determinada carpeta, no se considera el path.

136

El ultimo file de una determinada carpeta, no se considera el path.

137

"""

137

"""

138

validFilelist = []

138

validFilelist = []

139

fileList = os.listdir(path)

139

fileList = os.listdir(path)

140

141

# 0 1234 567 89A BCDE

141

# 0 1234 567 89A BCDE

142

# H YYYY DDD SSS .ext

142

# H YYYY DDD SSS .ext

143

144

for file in fileList:

144

for file in fileList:

145

try:

145

try:

146

year = int(file[1:5])

146

year = int(file[1:5])

147

doy = int(file[5:8])

147

doy = int(file[5:8])

148

149

150

except:

150

except:

151

continue

151

continue

152

153

if (os.path.splitext(file)[-1].lower() != ext.lower()):

153

if (os.path.splitext(file)[-1].lower() != ext.lower()):

154

continue

154

continue

155

156

validFilelist.append(file)

156

validFilelist.append(file)

157

158

if validFilelist:

158

if validFilelist:

159

validFilelist = sorted( validFilelist, key=str.lower )

159

validFilelist = sorted( validFilelist, key=str.lower )

160

return validFilelist[-1]

160

return validFilelist[-1]

161

162

return None

162

return None

163

164

def checkForRealPath(path, foldercounter, year, doy, set, ext):

164

def checkForRealPath(path, foldercounter, year, doy, set, ext):

165

"""

165

"""

166

Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,

166

Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,

167

Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar

167

Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar

168

el path exacto de un determinado file.

168

el path exacto de un determinado file.

169

170

Example :

170

Example :

171

nombre correcto del file es .../.../D2009307/P2009307367.ext

171

nombre correcto del file es .../.../D2009307/P2009307367.ext

172

173

Entonces la funcion prueba con las siguientes combinaciones

173

Entonces la funcion prueba con las siguientes combinaciones

174

.../.../y2009307367.ext

174

.../.../y2009307367.ext

175

.../.../Y2009307367.ext

175

.../.../Y2009307367.ext

176

.../.../x2009307/y2009307367.ext

176

.../.../x2009307/y2009307367.ext

177

.../.../x2009307/Y2009307367.ext

177

.../.../x2009307/Y2009307367.ext

178

.../.../X2009307/y2009307367.ext

178

.../.../X2009307/y2009307367.ext

179

.../.../X2009307/Y2009307367.ext

179

.../.../X2009307/Y2009307367.ext

180

siendo para este caso, la ultima combinacion de letras, identica al file buscado

180

siendo para este caso, la ultima combinacion de letras, identica al file buscado

181

182

Return:

182

Return:

183

Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file

183

Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file

184

caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas

184

caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas

185

para el filename

185

para el filename

186

"""

186

"""

187

fullfilename = None

187

fullfilename = None

188

find_flag = False

188

find_flag = False

189

filename = None

189

filename = None

190

191

prefixDirList = [None,'d','D']

191

prefixDirList = [None,'d','D']

192

if ext.lower() == ".r": #voltage

192

if ext.lower() == ".r": #voltage

193

prefixFileList = ['d','D']

193

prefixFileList = ['d','D']

194

elif ext.lower() == ".pdata": #spectra

194

elif ext.lower() == ".pdata": #spectra

195

prefixFileList = ['p','P']

195

prefixFileList = ['p','P']

196

else:

196

else:

197

return None, filename

197

return None, filename

198

199

#barrido por las combinaciones posibles

199

#barrido por las combinaciones posibles

200

for prefixDir in prefixDirList:

200

for prefixDir in prefixDirList:

201

thispath = path

201

thispath = path

202

if prefixDir != None:

202

if prefixDir != None:

203

#formo el nombre del directorio xYYYYDDD (x=d o x=D)

203

#formo el nombre del directorio xYYYYDDD (x=d o x=D)

204

if foldercounter == 0:

204

if foldercounter == 0:

205

thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))

205

thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))

206

else:

206

else:

207

thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))

207

thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))

208

for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"

208

for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"

209

filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext

209

filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext

210

fullfilename = os.path.join( thispath, filename ) #formo el path completo

210

fullfilename = os.path.join( thispath, filename ) #formo el path completo

211

212

if os.path.exists( fullfilename ): #verifico que exista

212

if os.path.exists( fullfilename ): #verifico que exista

213

find_flag = True

213

find_flag = True

214

break

214

break

215

if find_flag:

215

if find_flag:

216

break

216

break

217

218

if not(find_flag):

218

if not(find_flag):

219

return None, filename

219

return None, filename

220

221

return fullfilename, filename

221

return fullfilename, filename

222

223

def isDoyFolder(folder):

223

def isDoyFolder(folder):

224

try:

224

try:

225

year = int(folder[1:5])

225

year = int(folder[1:5])

226

except:

226

except:

227

return 0

227

return 0

228

229

try:

229

try:

230

doy = int(folder[5:8])

230

doy = int(folder[5:8])

231

except:

231

except:

232

return 0

232

return 0

233

234

return 1

234

return 1

235

236

class JRODataIO:

236

class JRODataIO:

237

238

c = 3E8

238

c = 3E8

239

240

isConfig = False

240

isConfig = False

241

242

basicHeaderObj = BasicHeader(LOCALTIME)

242

basicHeaderObj = BasicHeader(LOCALTIME)

243

244

systemHeaderObj = SystemHeader()

244

systemHeaderObj = SystemHeader()

245

246

radarControllerHeaderObj = RadarControllerHeader()

246

radarControllerHeaderObj = RadarControllerHeader()

247

248

processingHeaderObj = ProcessingHeader()

248

processingHeaderObj = ProcessingHeader()

249

250

online = 0

250

online = 0

251

252

dtype = None

252

dtype = None

253

254

pathList = []

254

pathList = []

255

256

filenameList = []

256

filenameList = []

257

258

filename = None

258

filename = None

259

260

ext = None

260

ext = None

261

262

flagIsNewFile = 1

262

flagIsNewFile = 1

263

264

flagTimeBlock = 0

264

flagTimeBlock = 0

265

266

flagIsNewBlock = 0

266

flagIsNewBlock = 0

267

268

fp = None

268

fp = None

269

270

firstHeaderSize = 0

270

firstHeaderSize = 0

271

272

basicHeaderSize = 24

272

basicHeaderSize = 24

273

274

versionFile = 1103

274

versionFile = 1103

275

276

fileSize = None

276

fileSize = None

277

278

ippSeconds = None

278

ippSeconds = None

279

280

fileSizeByHeader = None

280

fileSizeByHeader = None

281

282

fileIndex = None

282

fileIndex = None

283

284

profileIndex = None

284

profileIndex = None

285

286

blockIndex = None

286

blockIndex = None

287

288

nTotalBlocks = None

288

nTotalBlocks = None

289

290

maxTimeStep = 30

290

maxTimeStep = 30

291

292

lastUTTime = None

292

lastUTTime = None

293

294

datablock = None

294

datablock = None

295

296

dataOut = None

296

dataOut = None

297

298

blocksize = None

298

blocksize = None

299

300

def __init__(self):

300

def __init__(self):

301

302

raise ValueError, "Not implemented"

302

raise ValueError, "Not implemented"

303

304

def run(self):

304

def run(self):

305

306

raise ValueError, "Not implemented"

306

raise ValueError, "Not implemented"

307

308

def getOutput(self):

308

def getOutput(self):

309

310

return self.dataOut

310

return self.dataOut

311

312

class JRODataReader(JRODataIO, ProcessingUnit):

312

class JRODataReader(JRODataIO, ProcessingUnit):

313

314

nReadBlocks = 0

314

nReadBlocks = 0

315

316

delay = 10 #number of seconds waiting a new file

316

delay = 10 #number of seconds waiting a new file

317

318

nTries = 3 #quantity tries

318

nTries = 3 #quantity tries

319

320

nFiles = 3 #number of files for searching

320

nFiles = 3 #number of files for searching

321

322

path = None

322

path = None

323

324

foldercounter = 0

324

foldercounter = 0

325

326

flagNoMoreFiles = 0

326

flagNoMoreFiles = 0

327

328

datetimeList = []

328

datetimeList = []

329

330

__isFirstTimeOnline = 1

330

__isFirstTimeOnline = 1

331

332

__printInfo = True

332

__printInfo = True

333

334

profileIndex = None

334

profileIndex = None

335

336

def __init__(self):

336

def __init__(self):

337

338

"""

338

"""

339

340

"""

340

"""

341

342

raise ValueError, "This method has not been implemented"

342

raise ValueError, "This method has not been implemented"

343

344

345

def createObjByDefault(self):

345

def createObjByDefault(self):

346

"""

346

"""

347

348

"""

348

"""

349

raise ValueError, "This method has not been implemented"

349

raise ValueError, "This method has not been implemented"

350

351

def getBlockDimension(self):

351

def getBlockDimension(self):

352

353

raise ValueError, "No implemented"

353

raise ValueError, "No implemented"

354

355

def __searchFilesOffLine(self,

355

def __searchFilesOffLine(self,

356

path,

356

path,

357

startDate,

357

startDate,

358

endDate,

358

endDate,

359

startTime=datetime.time(0,0,0),

359

startTime=datetime.time(0,0,0),

360

endTime=datetime.time(23,59,59),

360

endTime=datetime.time(23,59,59),

361

set=None,

361

set=None,

362

expLabel='',

362

expLabel='',

363

ext='.r',

363

ext='.r',

364

walk=True):

364

walk=True):

365

366

pathList = []

366

pathList = []

367

368

if not walk:

368

if not walk:

369

pathList.append(path)

369

pathList.append(path)

370

371

else:

371

else:

372

dirList = []

372

dirList = []

373

for thisPath in os.listdir(path):

373

for thisPath in os.listdir(path):

374

if not os.path.isdir(os.path.join(path,thisPath)):

374

if not os.path.isdir(os.path.join(path,thisPath)):

375

continue

375

continue

376

if not isDoyFolder(thisPath):

376

if not isDoyFolder(thisPath):

377

continue

377

continue

378

379

dirList.append(thisPath)

379

dirList.append(thisPath)

380

381

if not(dirList):

381

if not(dirList):

382

return None, None

382

return None, None

383

384

thisDate = startDate

384

thisDate = startDate

385

386

while(thisDate <= endDate):

386

while(thisDate <= endDate):

387

year = thisDate.timetuple().tm_year

387

year = thisDate.timetuple().tm_year

388

doy = thisDate.timetuple().tm_yday

388

doy = thisDate.timetuple().tm_yday

389

390

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

390

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

391

if len(matchlist) == 0:

391

if len(matchlist) == 0:

392

thisDate += datetime.timedelta(1)

392

thisDate += datetime.timedelta(1)

393

continue

393

continue

394

for match in matchlist:

394

for match in matchlist:

395

pathList.append(os.path.join(path,match,expLabel))

395

pathList.append(os.path.join(path,match,expLabel))

396

397

thisDate += datetime.timedelta(1)

397

thisDate += datetime.timedelta(1)

398

399

if pathList == []:

399

if pathList == []:

400

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

400

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

401

return None, None

401

return None, None

402

403

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

403

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

404

405

filenameList = []

405

filenameList = []

406

datetimeList = []

406

datetimeList = []

407

408

for i in range(len(pathList)):

408

for i in range(len(pathList)):

409

410

thisPath = pathList[i]

410

thisPath = pathList[i]

411

412

fileList = glob.glob1(thisPath, "*%s" %ext)

412

fileList = glob.glob1(thisPath, "*%s" %ext)

413

fileList.sort()

413

fileList.sort()

414

415

for file in fileList:

415

for file in fileList:

416

417

filename = os.path.join(thisPath,file)

417

filename = os.path.join(thisPath,file)

418

thisDatetime = isFileinThisTime(filename, startTime, endTime)

418

thisDatetime = isFileinThisTime(filename, startTime, endTime)

419

420

if not(thisDatetime):

420

if not(thisDatetime):

421

continue

421

continue

422

423

filenameList.append(filename)

423

filenameList.append(filename)

424

datetimeList.append(thisDatetime)

424

datetimeList.append(thisDatetime)

425

426

if not(filenameList):

426

if not(filenameList):

427

print "Any file was found for the time range %s - %s" %(startTime, endTime)

427

print "Any file was found for the time range %s - %s" %(startTime, endTime)

428

return None, None

428

return None, None

429

430

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

430

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

431

print

431

print

432

433

for i in range(len(filenameList)):

433

for i in range(len(filenameList)):

434

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

434

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

435

436

self.filenameList = filenameList

436

self.filenameList = filenameList

437

self.datetimeList = datetimeList

437

self.datetimeList = datetimeList

438

439

return pathList, filenameList

439

return pathList, filenameList

440

441

def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True):

441

def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True):

442

443

"""

443

"""

444

Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y

444

Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y

445

devuelve el archivo encontrado ademas de otros datos.

445

devuelve el archivo encontrado ademas de otros datos.

446

447

Input:

447

Input:

448

path : carpeta donde estan contenidos los files que contiene data

448

path : carpeta donde estan contenidos los files que contiene data

449

450

expLabel : Nombre del subexperimento (subfolder)

450

expLabel : Nombre del subexperimento (subfolder)

451

452

ext : extension de los files

452

ext : extension de los files

453

454

walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)

454

walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)

455

456

Return:

456

Return:

457

directory : eL directorio donde esta el file encontrado

457

directory : eL directorio donde esta el file encontrado

458

filename : el ultimo file de una determinada carpeta

458

filename : el ultimo file de una determinada carpeta

459

year : el anho

459

year : el anho

460

doy : el numero de dia del anho

460

doy : el numero de dia del anho

461

set : el set del archivo

461

set : el set del archivo

462

463

464

"""

464

"""

465

dirList = []

465

dirList = []

466

467

if not walk:

467

if not walk:

468

fullpath = path

468

fullpath = path

469

470

else:

470

else:

471

#Filtra solo los directorios

471

#Filtra solo los directorios

472

for thisPath in os.listdir(path):

472

for thisPath in os.listdir(path):

473

if not os.path.isdir(os.path.join(path,thisPath)):

473

if not os.path.isdir(os.path.join(path,thisPath)):

474

continue

474

continue

475

if not isDoyFolder(thisPath):

475

if not isDoyFolder(thisPath):

476

continue

476

continue

477

478

dirList.append(thisPath)

478

dirList.append(thisPath)

479

480

if not(dirList):

480

if not(dirList):

481

return None, None, None, None, None

481

return None, None, None, None, None

482

483

dirList = sorted( dirList, key=str.lower )

483

dirList = sorted( dirList, key=str.lower )

484

485

doypath = dirList[-1]

485

doypath = dirList[-1]

486

foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0

486

foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0

487

fullpath = os.path.join(path, doypath, expLabel)

487

fullpath = os.path.join(path, doypath, expLabel)

488

489

490

print "%s folder was found: " %(fullpath )

490

print "%s folder was found: " %(fullpath )

491

492

filename = getlastFileFromPath(fullpath, ext)

492

filename = getlastFileFromPath(fullpath, ext)

493

494

if not(filename):

494

if not(filename):

495

return None, None, None, None, None

495

return None, None, None, None, None

496

497

print "%s file was found" %(filename)

497

print "%s file was found" %(filename)

498

499

if not(self.__verifyFile(os.path.join(fullpath, filename))):

499

if not(self.__verifyFile(os.path.join(fullpath, filename))):

500

return None, None, None, None, None

500

return None, None, None, None, None

501

502

year = int( filename[1:5] )

502

year = int( filename[1:5] )

503

doy = int( filename[5:8] )

503

doy = int( filename[5:8] )

504

set = int( filename[8:11] )

504

set = int( filename[8:11] )

505

506

return fullpath, foldercounter, filename, year, doy, set

506

return fullpath, foldercounter, filename, year, doy, set

507

508

def __setNextFileOffline(self):

508

def __setNextFileOffline(self):

509

510

idFile = self.fileIndex

510

idFile = self.fileIndex

511

512

while (True):

512

while (True):

513

idFile += 1

513

idFile += 1

514

if not(idFile < len(self.filenameList)):

514

if not(idFile < len(self.filenameList)):

515

self.flagNoMoreFiles = 1

515

self.flagNoMoreFiles = 1

516

print "No more Files"

516

print "No more Files"

517

return 0

517

return 0

518

519

filename = self.filenameList[idFile]

519

filename = self.filenameList[idFile]

520

521

if not(self.__verifyFile(filename)):

521

if not(self.__verifyFile(filename)):

522

continue

522

continue

523

524

fileSize = os.path.getsize(filename)

524

fileSize = os.path.getsize(filename)

525

fp = open(filename,'rb')

525

fp = open(filename,'rb')

526

break

526

break

527

528

self.flagIsNewFile = 1

528

self.flagIsNewFile = 1

529

self.fileIndex = idFile

529

self.fileIndex = idFile

530

self.filename = filename

530

self.filename = filename

531

self.fileSize = fileSize

531

self.fileSize = fileSize

532

self.fp = fp

532

self.fp = fp

533

534

print "Setting the file: %s"%self.filename

534

print "Setting the file: %s"%self.filename

535

536

return 1

536

return 1

537

538

def __setNextFileOnline(self):

538

def __setNextFileOnline(self):

539

"""

539

"""

540

Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si

540

Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si

541

no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files

541

no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files

542

siguientes.

542

siguientes.

543

544

Affected:

544

Affected:

545

self.flagIsNewFile

545

self.flagIsNewFile

546

self.filename

546

self.filename

547

self.fileSize

547

self.fileSize

548

self.fp

548

self.fp

549

self.set

549

self.set

550

self.flagNoMoreFiles

550

self.flagNoMoreFiles

551

552

Return:

552

Return:

553

0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado

553

0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado

554

1 : si el file fue abierto con exito y esta listo a ser leido

554

1 : si el file fue abierto con exito y esta listo a ser leido

555

556

Excepciones:

556

Excepciones:

557

Si un determinado file no puede ser abierto

557

Si un determinado file no puede ser abierto

558

"""

558

"""

559

nFiles = 0

559

nFiles = 0

560

fileOk_flag = False

560

fileOk_flag = False

561

firstTime_flag = True

561

firstTime_flag = True

562

563

self.set += 1

563

self.set += 1

564

565

if self.set > 999:

565

if self.set > 999:

566

self.set = 0

566

self.set = 0

567

self.foldercounter += 1

567

self.foldercounter += 1

568

569

#busca el 1er file disponible

569

#busca el 1er file disponible

570

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

570

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

571

if fullfilename:

571

if fullfilename:

572

if self.__verifyFile(fullfilename, False):

572

if self.__verifyFile(fullfilename, False):

573

fileOk_flag = True

573

fileOk_flag = True

574

575

#si no encuentra un file entonces espera y vuelve a buscar

575

#si no encuentra un file entonces espera y vuelve a buscar

576

if not(fileOk_flag):

576

if not(fileOk_flag):

577

for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles

577

for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles

578

579

if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces

579

if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces

580

tries = self.nTries

580

tries = self.nTries

581

else:

581

else:

582

tries = 1 #si no es la 1era vez entonces solo lo hace una vez

582

tries = 1 #si no es la 1era vez entonces solo lo hace una vez

583

584

for nTries in range( tries ):

584

for nTries in range( tries ):

585

if firstTime_flag:

585

if firstTime_flag:

586

print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )

586

print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )

587

time.sleep( self.delay )

587

time.sleep( self.delay )

588

else:

588

else:

589

print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)

589

print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)

590

591

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

591

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

592

if fullfilename:

592

if fullfilename:

593

if self.__verifyFile(fullfilename):

593

if self.__verifyFile(fullfilename):

594

fileOk_flag = True

594

fileOk_flag = True

595

break

595

break

596

597

if fileOk_flag:

597

if fileOk_flag:

598

break

598

break

599

600

firstTime_flag = False

600

firstTime_flag = False

601

602

print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename

602

print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename

603

self.set += 1

603

self.set += 1

604

605

if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta

605

if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta

606

self.set = 0

606

self.set = 0

607

self.doy += 1

607

self.doy += 1

608

self.foldercounter = 0

608

self.foldercounter = 0

609

610

if fileOk_flag:

610

if fileOk_flag:

611

self.fileSize = os.path.getsize( fullfilename )

611

self.fileSize = os.path.getsize( fullfilename )

612

self.filename = fullfilename

612

self.filename = fullfilename

613

self.flagIsNewFile = 1

613

self.flagIsNewFile = 1

614

if self.fp != None: self.fp.close()

614

if self.fp != None: self.fp.close()

615

self.fp = open(fullfilename, 'rb')

615

self.fp = open(fullfilename, 'rb')

616

self.flagNoMoreFiles = 0

616

self.flagNoMoreFiles = 0

617

print 'Setting the file: %s' % fullfilename

617

print 'Setting the file: %s' % fullfilename

618

else:

618

else:

619

self.fileSize = 0

619

self.fileSize = 0

620

self.filename = None

620

self.filename = None

621

self.flagIsNewFile = 0

621

self.flagIsNewFile = 0

622

self.fp = None

622

self.fp = None

623

self.flagNoMoreFiles = 1

623

self.flagNoMoreFiles = 1

624

print 'No more Files'

624

print 'No more Files'

625

626

return fileOk_flag

626

return fileOk_flag

627

628

629

def setNextFile(self):

629

def setNextFile(self):

630

if self.fp != None:

630

if self.fp != None:

631

self.fp.close()

631

self.fp.close()

632

633

if self.online:

633

if self.online:

634

newFile = self.__setNextFileOnline()

634

newFile = self.__setNextFileOnline()

635

else:

635

else:

636

newFile = self.__setNextFileOffline()

636

newFile = self.__setNextFileOffline()

637

638

if not(newFile):

638

if not(newFile):

639

return 0

639

return 0

640

641

self.__readFirstHeader()

641

self.__readFirstHeader()

642

self.nReadBlocks = 0

642

self.nReadBlocks = 0

643

return 1

643

return 1

644

645

def __waitNewBlock(self):

645

def __waitNewBlock(self):

646

"""

646

"""

647

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

647

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

648

649

Si el modo de lectura es OffLine siempre retorn 0

649

Si el modo de lectura es OffLine siempre retorn 0

650

"""

650

"""

651

if not self.online:

651

if not self.online:

652

return 0

652

return 0

653

654

if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):

654

if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):

655

return 0

655

return 0

656

657

currentPointer = self.fp.tell()

657

currentPointer = self.fp.tell()

658

659

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

659

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

660

661

for nTries in range( self.nTries ):

661

for nTries in range( self.nTries ):

662

663

self.fp.close()

663

self.fp.close()

664

self.fp = open( self.filename, 'rb' )

664

self.fp = open( self.filename, 'rb' )

665

self.fp.seek( currentPointer )

665

self.fp.seek( currentPointer )

666

667

self.fileSize = os.path.getsize( self.filename )

667

self.fileSize = os.path.getsize( self.filename )

668

currentSize = self.fileSize - currentPointer

668

currentSize = self.fileSize - currentPointer

669

670

if ( currentSize >= neededSize ):

670

if ( currentSize >= neededSize ):

671

self.__rdBasicHeader()

671

self.__rdBasicHeader()

672

return 1

672

return 1

673

674

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

674

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

675

time.sleep( self.delay )

675

time.sleep( self.delay )

676

677

678

return 0

678

return 0

679

680

def __jumpToLastBlock(self):

680

def __jumpToLastBlock(self):

681

682

if not(self.__isFirstTimeOnline):

682

if not(self.__isFirstTimeOnline):

683

return

683

return

684

685

csize = self.fileSize - self.fp.tell()

685

csize = self.fileSize - self.fp.tell()

686

687

#sata el primer bloque de datos

687

#sata el primer bloque de datos

688

if csize > self.processingHeaderObj.blockSize:

688

if csize > self.processingHeaderObj.blockSize:

689

self.fp.seek(self.fp.tell() + self.processingHeaderObj.blockSize)

689

self.fp.seek(self.fp.tell() + self.processingHeaderObj.blockSize)

690

else:

690

else:

691

return

691

return

692

693

csize = self.fileSize - self.fp.tell()

693

csize = self.fileSize - self.fp.tell()

694

neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

694

neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

695

factor = int(csize/neededsize)

695

factor = int(csize/neededsize)

696

if factor > 0:

696

if factor > 0:

697

self.fp.seek(self.fp.tell() + factor*neededsize)

697

self.fp.seek(self.fp.tell() + factor*neededsize)

698

699

self.flagIsNewFile = 0

699

self.flagIsNewFile = 0

700

self.__isFirstTimeOnline = 0

700

self.__isFirstTimeOnline = 0

701

702

703

def __setNewBlock(self):

703

def __setNewBlock(self):

704

705

if self.fp == None:

705

if self.fp == None:

706

return 0

706

return 0

707

708

if self.online:

708

if self.online:

709

self.__jumpToLastBlock()

709

self.__jumpToLastBlock()

710

711

if self.flagIsNewFile:

711

if self.flagIsNewFile:

712

return 1

712

return 1

713

714

self.lastUTTime = self.basicHeaderObj.utc

714

self.lastUTTime = self.basicHeaderObj.utc

715

currentSize = self.fileSize - self.fp.tell()

715

currentSize = self.fileSize - self.fp.tell()

716

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

716

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

717

718

if (currentSize >= neededSize):

718

if (currentSize >= neededSize):

719

self.__rdBasicHeader()

719

self.__rdBasicHeader()

720

return 1

720

return 1

721

722

if self.__waitNewBlock():

722

if self.__waitNewBlock():

723

return 1

723

return 1

724

725

if not(self.setNextFile()):

725

if not(self.setNextFile()):

726

return 0

726

return 0

727

728

deltaTime = self.basicHeaderObj.utc - self.lastUTTime #

728

deltaTime = self.basicHeaderObj.utc - self.lastUTTime #

729

730

self.flagTimeBlock = 0

730

self.flagTimeBlock = 0

731

732

if deltaTime > self.maxTimeStep:

732

if deltaTime > self.maxTimeStep:

733

self.flagTimeBlock = 1

733

self.flagTimeBlock = 1

734

735

return 1

735

return 1

736

737

738

def readNextBlock(self):

738

def readNextBlock(self):

739

if not(self.__setNewBlock()):

739

if not(self.__setNewBlock()):

740

return 0

740

return 0

741

742

if not(self.readBlock()):

742

if not(self.readBlock()):

743

return 0

743

return 0

744

745

return 1

745

return 1

746

747

def __rdProcessingHeader(self, fp=None):

747

def __rdProcessingHeader(self, fp=None):

748

if fp == None:

748

if fp == None:

749

fp = self.fp

749

fp = self.fp

750

751

self.processingHeaderObj.read(fp)

751

self.processingHeaderObj.read(fp)

752

753

def __rdRadarControllerHeader(self, fp=None):

753

def __rdRadarControllerHeader(self, fp=None):

754

if fp == None:

754

if fp == None:

755

fp = self.fp

755

fp = self.fp

756

757

self.radarControllerHeaderObj.read(fp)

757

self.radarControllerHeaderObj.read(fp)

758

759

def __rdSystemHeader(self, fp=None):

759

def __rdSystemHeader(self, fp=None):

760

if fp == None:

760

if fp == None:

761

fp = self.fp

761

fp = self.fp

762

763

self.systemHeaderObj.read(fp)

763

self.systemHeaderObj.read(fp)

764

765

def __rdBasicHeader(self, fp=None):

765

def __rdBasicHeader(self, fp=None):

766

if fp == None:

766

if fp == None:

767

fp = self.fp

767

fp = self.fp

768

769

self.basicHeaderObj.read(fp)

769

self.basicHeaderObj.read(fp)

770

771

772

def __readFirstHeader(self):

772

def __readFirstHeader(self):

773

self.__rdBasicHeader()

773

self.__rdBasicHeader()

774

self.__rdSystemHeader()

774

self.__rdSystemHeader()

775

self.__rdRadarControllerHeader()

775

self.__rdRadarControllerHeader()

776

self.__rdProcessingHeader()

776

self.__rdProcessingHeader()

777

778

self.firstHeaderSize = self.basicHeaderObj.size

778

self.firstHeaderSize = self.basicHeaderObj.size

779

780

datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

780

datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

781

if datatype == 0:

781

if datatype == 0:

782

datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])

782

datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])

783

elif datatype == 1:

783

elif datatype == 1:

784

datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])

784

datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])

785

elif datatype == 2:

785

elif datatype == 2:

786

datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])

786

datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])

787

elif datatype == 3:

787

elif datatype == 3:

788

datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])

788

datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])

789

elif datatype == 4:

789

elif datatype == 4:

790

datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])

790

datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])

791

elif datatype == 5:

791

elif datatype == 5:

792

datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])

792

datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])

793

else:

793

else:

794

raise ValueError, 'Data type was not defined'

794

raise ValueError, 'Data type was not defined'

795

796

self.dtype = datatype_str

796

self.dtype = datatype_str

797

self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c

797

self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c

798

self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)

798

self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)

799

# self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)

799

# self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)

800

# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)

800

# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)

801

self.getBlockDimension()

801

self.getBlockDimension()

802

803

804

def __verifyFile(self, filename, msgFlag=True):

804

def __verifyFile(self, filename, msgFlag=True):

805

msg = None

805

msg = None

806

try:

806

try:

807

fp = open(filename, 'rb')

807

fp = open(filename, 'rb')

808

currentPosition = fp.tell()

808

currentPosition = fp.tell()

809

except:

809

except:

810

if msgFlag:

810

if msgFlag:

811

print "The file %s can't be opened" % (filename)

811

print "The file %s can't be opened" % (filename)

812

return False

812

return False

813

814

neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize

814

neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize

815

816

if neededSize == 0:

816

if neededSize == 0:

817

basicHeaderObj = BasicHeader(LOCALTIME)

817

basicHeaderObj = BasicHeader(LOCALTIME)

818

systemHeaderObj = SystemHeader()

818

systemHeaderObj = SystemHeader()

819

radarControllerHeaderObj = RadarControllerHeader()

819

radarControllerHeaderObj = RadarControllerHeader()

820

processingHeaderObj = ProcessingHeader()

820

processingHeaderObj = ProcessingHeader()

821

822

try:

822

try:

823

if not( basicHeaderObj.read(fp) ): raise IOError

823

if not( basicHeaderObj.read(fp) ): raise IOError

824

if not( systemHeaderObj.read(fp) ): raise IOError

824

if not( systemHeaderObj.read(fp) ): raise IOError

825

if not( radarControllerHeaderObj.read(fp) ): raise IOError

825

if not( radarControllerHeaderObj.read(fp) ): raise IOError

826

if not( processingHeaderObj.read(fp) ): raise IOError

826

if not( processingHeaderObj.read(fp) ): raise IOError

827

data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

827

data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

828

829

neededSize = processingHeaderObj.blockSize + basicHeaderObj.size

829

neededSize = processingHeaderObj.blockSize + basicHeaderObj.size

830

831

except:

831

except:

832

if msgFlag:

832

if msgFlag:

833

print "\tThe file %s is empty or it hasn't enough data" % filename

833

print "\tThe file %s is empty or it hasn't enough data" % filename

834

835

fp.close()

835

fp.close()

836

return False

836

return False

837

else:

837

else:

838

msg = "\tSkipping the file %s due to it hasn't enough data" %filename

838

msg = "\tSkipping the file %s due to it hasn't enough data" %filename

839

840

fp.close()

840

fp.close()

841

fileSize = os.path.getsize(filename)

841

fileSize = os.path.getsize(filename)

842

currentSize = fileSize - currentPosition

842

currentSize = fileSize - currentPosition

843

if currentSize < neededSize:

843

if currentSize < neededSize:

844

if msgFlag and (msg != None):

844

if msgFlag and (msg != None):

845

print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename

845

print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename

846

return False

846

return False

847

848

return True

848

return True

849

850

def setup(self,

850

def setup(self,

851

path=None,

851

path=None,

852

startDate=None,

852

startDate=None,

853

endDate=None,

853

endDate=None,

854

startTime=datetime.time(0,0,0),

854

startTime=datetime.time(0,0,0),

855

endTime=datetime.time(23,59,59),

855

endTime=datetime.time(23,59,59),

856

set=0,

856

set=0,

857

expLabel = "",

857

expLabel = "",

858

ext = None,

858

ext = None,

859

online = False,

859

online = False,

860

delay = 60,

860

delay = 60,

861

walk = True):

861

walk = True):

862

863

if path == None:

863

if path == None:

864

raise ValueError, "The path is not valid"

864

raise ValueError, "The path is not valid"

865

866

if ext == None:

866

if ext == None:

867

ext = self.ext

867

ext = self.ext

868

869

if online:

869

if online:

870

print "Searching files in online mode..."

870

print "Searching files in online mode..."

871

872

for nTries in range( self.nTries ):

872

for nTries in range( self.nTries ):

873

fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk)

873

fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk)

874

875

if fullpath:

875

if fullpath:

876

break

876

break

877

878

print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)

878

print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)

879

time.sleep( self.delay )

879

time.sleep( self.delay )

880

881

if not(fullpath):

881

if not(fullpath):

882

print "There 'isn't valied files in %s" % path

882

print "There 'isn't valied files in %s" % path

883

return None

883

return None

884

885

self.year = year

885

self.year = year

886

self.doy = doy

886

self.doy = doy

887

self.set = set - 1

887

self.set = set - 1

888

self.path = path

888

self.path = path

889

self.foldercounter = foldercounter

889

self.foldercounter = foldercounter

890

891

else:

891

else:

892

print "Searching files in offline mode ..."

892

print "Searching files in offline mode ..."

893

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

893

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

894

startTime=startTime, endTime=endTime,

894

startTime=startTime, endTime=endTime,

895

set=set, expLabel=expLabel, ext=ext,

895

set=set, expLabel=expLabel, ext=ext,

896

walk=walk)

896

walk=walk)

897

898

if not(pathList):

898

if not(pathList):

899

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

899

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

900

datetime.datetime.combine(startDate,startTime).ctime(),

900

datetime.datetime.combine(startDate,startTime).ctime(),

901

datetime.datetime.combine(endDate,endTime).ctime())

901

datetime.datetime.combine(endDate,endTime).ctime())

902

903

sys.exit(-1)

903

sys.exit(-1)

904

905

906

self.fileIndex = -1

906

self.fileIndex = -1

907

self.pathList = pathList

907

self.pathList = pathList

908

self.filenameList = filenameList

908

self.filenameList = filenameList

909

910

self.online = online

910

self.online = online

911

self.delay = delay

911

self.delay = delay

912

ext = ext.lower()

912

ext = ext.lower()

913

self.ext = ext

913

self.ext = ext

914

915

if not(self.setNextFile()):

915

if not(self.setNextFile()):

916

if (startDate!=None) and (endDate!=None):

916

if (startDate!=None) and (endDate!=None):

917

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

917

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

918

elif startDate != None:

918

elif startDate != None:

919

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

919

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

920

else:

920

else:

921

print "No files"

921

print "No files"

922

923

sys.exit(-1)

923

sys.exit(-1)

924

925

# self.updateDataHeader()

925

# self.updateDataHeader()

926

927

return self.dataOut

927

return self.dataOut

928

929

def getBasicHeader(self):

929

def getBasicHeader(self):

930

931

self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds

931

self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds

932

933

self.dataOut.flagTimeBlock = self.flagTimeBlock

933

self.dataOut.flagTimeBlock = self.flagTimeBlock

934

935

self.dataOut.timeZone = self.basicHeaderObj.timeZone

935

self.dataOut.timeZone = self.basicHeaderObj.timeZone

936

937

self.dataOut.dstFlag = self.basicHeaderObj.dstFlag

937

self.dataOut.dstFlag = self.basicHeaderObj.dstFlag

938

939

self.dataOut.errorCount = self.basicHeaderObj.errorCount

939

self.dataOut.errorCount = self.basicHeaderObj.errorCount

940

941

self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime

941

self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime

942

943

def getFirstHeader(self):

943

def getFirstHeader(self):

944

945

raise ValueError, "This method has not been implemented"

945

raise ValueError, "This method has not been implemented"

946

947

def getData():

947

def getData():

948

949

raise ValueError, "This method has not been implemented"

949

raise ValueError, "This method has not been implemented"

950

951

def hasNotDataInBuffer():

951

def hasNotDataInBuffer():

952

953

raise ValueError, "This method has not been implemented"

953

raise ValueError, "This method has not been implemented"

954

955

def readBlock():

955

def readBlock():

956

957

raise ValueError, "This method has not been implemented"

957

raise ValueError, "This method has not been implemented"

958

959

def isEndProcess(self):

959

def isEndProcess(self):

960

961

return self.flagNoMoreFiles

961

return self.flagNoMoreFiles

962

963

def printReadBlocks(self):

963

def printReadBlocks(self):

964

965

print "Number of read blocks per file %04d" %self.nReadBlocks

965

print "Number of read blocks per file %04d" %self.nReadBlocks

966

967

def printTotalBlocks(self):

967

def printTotalBlocks(self):

968

969

print "Number of read blocks %04d" %self.nTotalBlocks

969

print "Number of read blocks %04d" %self.nTotalBlocks

970

971

def printNumberOfBlock(self):

971

def printNumberOfBlock(self):

972

973

if self.flagIsNewBlock:

973

if self.flagIsNewBlock:

974

print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())

974

print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())

975

976

def printInfo(self):

976

def printInfo(self):

977

978

if self.__printInfo == False:

978

if self.__printInfo == False:

979

return

979

return

980

981

self.basicHeaderObj.printInfo()

981

self.basicHeaderObj.printInfo()

982

self.systemHeaderObj.printInfo()

982

self.systemHeaderObj.printInfo()

983

self.radarControllerHeaderObj.printInfo()

983

self.radarControllerHeaderObj.printInfo()

984

self.processingHeaderObj.printInfo()

984

self.processingHeaderObj.printInfo()

985

986

self.__printInfo = False

986

self.__printInfo = False

987

988

989

def run(self, **kwargs):

989

def run(self, **kwargs):

990

991

if not(self.isConfig):

991

if not(self.isConfig):

992

993

# self.dataOut = dataOut

993

# self.dataOut = dataOut

994

self.setup(**kwargs)

994

self.setup(**kwargs)

995

self.isConfig = True

995

self.isConfig = True

996

997

self.getData()

997

self.getData()

998

999

class JRODataWriter(JRODataIO, Operation):

999

class JRODataWriter(JRODataIO, Operation):

1000

1001

"""

1001

"""

1002

Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura

1002

Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura

1003

de los datos siempre se realiza por bloques.

1003

de los datos siempre se realiza por bloques.

1004

"""

1004

"""

1005

1006

blockIndex = 0

1006

blockIndex = 0

1007

1008

path = None

1008

path = None

1009

1010

setFile = None

1010

setFile = None

1011

1012

profilesPerBlock = None

1012

profilesPerBlock = None

1013

1014

blocksPerFile = None

1014

blocksPerFile = None

1015

1016

nWriteBlocks = 0

1016

nWriteBlocks = 0

1017

1018

def __init__(self, dataOut=None):

1018

def __init__(self, dataOut=None):

1019

raise ValueError, "Not implemented"

1019

raise ValueError, "Not implemented"

1020

1021

1022

def hasAllDataInBuffer(self):

1022

def hasAllDataInBuffer(self):

1023

raise ValueError, "Not implemented"

1023

raise ValueError, "Not implemented"

1024

1025

1026

def setBlockDimension(self):

1026

def setBlockDimension(self):

1027

raise ValueError, "Not implemented"

1027

raise ValueError, "Not implemented"

1028

1029

1030

def writeBlock(self):

1030

def writeBlock(self):

1031

raise ValueError, "No implemented"

1031

raise ValueError, "No implemented"

1032

1033

1034

def putData(self):

1034

def putData(self):

1035

raise ValueError, "No implemented"

1035

raise ValueError, "No implemented"

1036

1037

1038

def setBasicHeader(self):

1038

def setBasicHeader(self):

1039

1040

self.basicHeaderObj.size = self.basicHeaderSize #bytes

1040

self.basicHeaderObj.size = self.basicHeaderSize #bytes

1041

self.basicHeaderObj.version = self.versionFile

1041

self.basicHeaderObj.version = self.versionFile

1042

self.basicHeaderObj.dataBlock = self.nTotalBlocks

1042

self.basicHeaderObj.dataBlock = self.nTotalBlocks

1043

1044

utc = numpy.floor(self.dataOut.utctime)

1044

utc = numpy.floor(self.dataOut.utctime)

1045

milisecond = (self.dataOut.utctime - utc)* 1000.0

1045

milisecond = (self.dataOut.utctime - utc)* 1000.0

1046

1047

self.basicHeaderObj.utc = utc

1047

self.basicHeaderObj.utc = utc

1048

self.basicHeaderObj.miliSecond = milisecond

1048

self.basicHeaderObj.miliSecond = milisecond

1049

self.basicHeaderObj.timeZone = self.dataOut.timeZone

1049

self.basicHeaderObj.timeZone = self.dataOut.timeZone

1050

self.basicHeaderObj.dstFlag = self.dataOut.dstFlag

1050

self.basicHeaderObj.dstFlag = self.dataOut.dstFlag

1051

self.basicHeaderObj.errorCount = self.dataOut.errorCount

1051

self.basicHeaderObj.errorCount = self.dataOut.errorCount

1052

1053

def setFirstHeader(self):

1053

def setFirstHeader(self):

1054

"""

1054

"""

1055

Obtiene una copia del First Header

1055

Obtiene una copia del First Header

1056

1057

Affected:

1057

Affected:

1058

1059

self.basicHeaderObj

1059

self.basicHeaderObj

1060

self.systemHeaderObj

1060

self.systemHeaderObj

1061

self.radarControllerHeaderObj

1061

self.radarControllerHeaderObj

1062

self.processingHeaderObj self.

1062

self.processingHeaderObj self.

1063

1064

Return:

1064

Return:

1065

None

1065

None

1066

"""

1066

"""

1067

1068

raise ValueError, "No implemented"

1068

raise ValueError, "No implemented"

1069

1070

def __writeFirstHeader(self):

1070

def __writeFirstHeader(self):

1071

"""

1071

"""

1072

Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)

1072

Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)

1073

1074

Affected:

1074

Affected:

1075

__dataType

1075

__dataType

1076

1077

Return:

1077

Return:

1078

None

1078

None

1079

"""

1079

"""

1080

1081

# CALCULAR PARAMETROS

1081

# CALCULAR PARAMETROS

1082

1083

sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size

1083

sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size

1084

self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader

1084

self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader

1085

1086

self.basicHeaderObj.write(self.fp)

1086

self.basicHeaderObj.write(self.fp)

1087

self.systemHeaderObj.write(self.fp)

1087

self.systemHeaderObj.write(self.fp)

1088

self.radarControllerHeaderObj.write(self.fp)

1088

self.radarControllerHeaderObj.write(self.fp)

1089

self.processingHeaderObj.write(self.fp)

1089

self.processingHeaderObj.write(self.fp)

1090

1091

self.dtype = self.dataOut.dtype

1091

self.dtype = self.dataOut.dtype

1092

1093

def __setNewBlock(self):

1093

def __setNewBlock(self):

1094

"""

1094

"""

1095

Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header

1095

Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header

1096

1097

Return:

1097

Return:

1098

0 : si no pudo escribir nada

1098

0 : si no pudo escribir nada

1099

1 : Si escribio el Basic el First Header

1099

1 : Si escribio el Basic el First Header

1100

"""

1100

"""

1101

if self.fp == None:

1101

if self.fp == None:

1102

self.setNextFile()

1102

self.setNextFile()

1103

1104

if self.flagIsNewFile:

1104

if self.flagIsNewFile:

1105

return 1

1105

return 1

1106

1107

if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:

1107

if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:

1108

self.basicHeaderObj.write(self.fp)

1108

self.basicHeaderObj.write(self.fp)

1109

return 1

1109

return 1

1110

1111

if not( self.setNextFile() ):

1111

if not( self.setNextFile() ):

1112

return 0

1112

return 0

1113

1114

return 1

1114

return 1

1115

1116

1117

def writeNextBlock(self):

1117

def writeNextBlock(self):

1118

"""

1118

"""

1119

Selecciona el bloque siguiente de datos y los escribe en un file

1119

Selecciona el bloque siguiente de datos y los escribe en un file

1120

1121

Return:

1121

Return:

1122

0 : Si no hizo pudo escribir el bloque de datos

1122

0 : Si no hizo pudo escribir el bloque de datos

1123

1 : Si no pudo escribir el bloque de datos

1123

1 : Si no pudo escribir el bloque de datos

1124

"""

1124

"""

1125

if not( self.__setNewBlock() ):

1125

if not( self.__setNewBlock() ):

1126

return 0

1126

return 0

1127

1128

self.writeBlock()

1128

self.writeBlock()

1129

1130

return 1

1130

return 1

1131

1132

def setNextFile(self):

1132

def setNextFile(self):

1133

"""

1133

"""

1134

Determina el siguiente file que sera escrito

1134

Determina el siguiente file que sera escrito

1135

1136

Affected:

1136

Affected:

1137

self.filename

1137

self.filename

1138

self.subfolder

1138

self.subfolder

1139

self.fp

1139

self.fp

1140

self.setFile

1140

self.setFile

1141

self.flagIsNewFile

1141

self.flagIsNewFile

1142

1143

Return:

1143

Return:

1144

0 : Si el archivo no puede ser escrito

1144

0 : Si el archivo no puede ser escrito

1145

1 : Si el archivo esta listo para ser escrito

1145

1 : Si el archivo esta listo para ser escrito

1146

"""

1146

"""

1147

ext = self.ext

1147

ext = self.ext

1148

path = self.path

1148

path = self.path

1149

1150

if self.fp != None:

1150

if self.fp != None:

1151

self.fp.close()

1151

self.fp.close()

1152

1153

timeTuple = time.localtime( self.dataOut.utctime)

1153

timeTuple = time.localtime( self.dataOut.utctime)

1154

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

1154

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

1155

1156

fullpath = os.path.join( path, subfolder )

1156

fullpath = os.path.join( path, subfolder )

1157

if not( os.path.exists(fullpath) ):

1157

if not( os.path.exists(fullpath) ):

1158

os.mkdir(fullpath)

1158

os.mkdir(fullpath)

1159

self.setFile = -1 #inicializo mi contador de seteo

1159

self.setFile = -1 #inicializo mi contador de seteo

1160

else:

1160

else:

1161

filesList = os.listdir( fullpath )

1161

filesList = os.listdir( fullpath )

1162

if len( filesList ) > 0:

1162

if len( filesList ) > 0:

1163

filesList = sorted( filesList, key=str.lower )

1163

filesList = sorted( filesList, key=str.lower )

1164

filen = filesList[-1]

1164

filen = filesList[-1]

1165

# el filename debera tener el siguiente formato

1165

# el filename debera tener el siguiente formato

1166

# 0 1234 567 89A BCDE (hex)

1166

# 0 1234 567 89A BCDE (hex)

1167

# x YYYY DDD SSS .ext

1167

# x YYYY DDD SSS .ext

1168

if isNumber( filen[8:11] ):

1168

if isNumber( filen[8:11] ):

1169

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

1169

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

1170

else:

1170

else:

1171

self.setFile = -1

1171

self.setFile = -1

1172

else:

1172

else:

1173

self.setFile = -1 #inicializo mi contador de seteo

1173

self.setFile = -1 #inicializo mi contador de seteo

1174

1175

setFile = self.setFile

1175

setFile = self.setFile

1176

setFile += 1

1176

setFile += 1

1177

1178

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

1178

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

1179

timeTuple.tm_year,

1179

timeTuple.tm_year,

1180

timeTuple.tm_yday,

1180

timeTuple.tm_yday,

1181

setFile,

1181

setFile,

1182

ext )

1182

ext )

1183

1184

filename = os.path.join( path, subfolder, file )

1184

filename = os.path.join( path, subfolder, file )

1185

1186

fp = open( filename,'wb' )

1186

fp = open( filename,'wb' )

1187

1188

self.blockIndex = 0

1188

self.blockIndex = 0

1189

1190

#guardando atributos

1190

#guardando atributos

1191

self.filename = filename

1191

self.filename = filename

1192

self.subfolder = subfolder

1192

self.subfolder = subfolder

1193

self.fp = fp

1193

self.fp = fp

1194

self.setFile = setFile

1194

self.setFile = setFile

1195

self.flagIsNewFile = 1

1195

self.flagIsNewFile = 1

1196

1197

self.setFirstHeader()

1197

self.setFirstHeader()

1198

1199

print 'Writing the file: %s'%self.filename

1199

print 'Writing the file: %s'%self.filename

1200

1201

self.__writeFirstHeader()

1201

self.__writeFirstHeader()

1202

1203

return 1

1203

return 1

1204

1205

def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=None, set=0, ext=None):

1205

def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=None, set=0, ext=None):

1206

"""

1206

"""

1207

Setea el tipo de formato en la cual sera guardada la data y escribe el First Header

1207

Setea el tipo de formato en la cual sera guardada la data y escribe el First Header

1208

1209

Inputs:

1209

Inputs:

1210

path : el path destino en el cual se escribiran los files a crear

1210

path : el path destino en el cual se escribiran los files a crear

1211

format : formato en el cual sera salvado un file

1211

format : formato en el cual sera salvado un file

1212

set : el setebo del file

1212

set : el setebo del file

1213

1214

Return:

1214

Return:

1215

0 : Si no realizo un buen seteo

1215

0 : Si no realizo un buen seteo

1216

1 : Si realizo un buen seteo

1216

1 : Si realizo un buen seteo

1217

"""

1217

"""

1218

1219

if ext == None:

1219

if ext == None:

1220

ext = self.ext

1220

ext = self.ext

1221

1222

ext = ext.lower()

1222

ext = ext.lower()

1223

1224

self.ext = ext

1224

self.ext = ext

1225

1226

self.path = path

1226

self.path = path

1227

1228

self.setFile = set - 1

1228

self.setFile = set - 1

1229

1230

self.blocksPerFile = blocksPerFile

1230

self.blocksPerFile = blocksPerFile

1231

1232

self.profilesPerBlock = profilesPerBlock

1232

self.profilesPerBlock = profilesPerBlock

1233

1234

self.dataOut = dataOut

1234

self.dataOut = dataOut

1235

1236

if not(self.setNextFile()):

1236

if not(self.setNextFile()):

1237

print "There isn't a next file"

1237

print "There isn't a next file"

1238

return 0

1238

return 0

1239

1240

self.setBlockDimension()

1240

self.setBlockDimension()

1241

1242

return 1

1242

return 1

1243

1244

def run(self, dataOut, **kwargs):

1244

def run(self, dataOut, **kwargs):

1245

1246

if not(self.isConfig):

1246

if not(self.isConfig):

1247

1248

self.setup(dataOut, **kwargs)

1248

self.setup(dataOut, **kwargs)

1249

self.isConfig = True

1249

self.isConfig = True

1250

1251

self.putData()

1251

self.putData()

1252

1253

class VoltageReader(JRODataReader):

1253

class VoltageReader(JRODataReader):

1254

"""

1254

"""

1255

Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura

1255

Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura

1256

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:

1256

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:

1257

perfiles*alturas*canales) son almacenados en la variable "buffer".

1257

perfiles*alturas*canales) son almacenados en la variable "buffer".

1258

1259

perfiles * alturas * canales

1259

perfiles * alturas * canales

1260

1261

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1261

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1262

RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la

1262

RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la

1263

cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de

1263

cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de

1264

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1264

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1265

1266

Example:

1266

Example:

1267

1268

dpath = "/home/myuser/data"

1268

dpath = "/home/myuser/data"

1269

1270

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1270

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1271

1272

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1272

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1273

1274

readerObj = VoltageReader()

1274

readerObj = VoltageReader()

1275

1276

readerObj.setup(dpath, startTime, endTime)

1276

readerObj.setup(dpath, startTime, endTime)

1277

1278

while(True):

1278

while(True):

1279

1280

#to get one profile

1280

#to get one profile

1281

profile = readerObj.getData()

1281

profile = readerObj.getData()

1282

1283

#print the profile

1283

#print the profile

1284

print profile

1284

print profile

1285

1286

#If you want to see all datablock

1286

#If you want to see all datablock

1287

print readerObj.datablock

1287

print readerObj.datablock

1288

1289

if readerObj.flagNoMoreFiles:

1289

if readerObj.flagNoMoreFiles:

1290

break

1290

break

1291

1292

"""

1292

"""

1293

1294

ext = ".r"

1294

ext = ".r"

1295

1296

optchar = "D"

1296

optchar = "D"

1297

dataOut = None

1297

dataOut = None

1298

1299

1300

def __init__(self):

1300

def __init__(self):

1301

"""

1301

"""

1302

Inicializador de la clase VoltageReader para la lectura de datos de voltage.

1302

Inicializador de la clase VoltageReader para la lectura de datos de voltage.

1303

1304

Input:

1304

Input:

1305

dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para

1305

dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para

1306

almacenar un perfil de datos cada vez que se haga un requerimiento

1306

almacenar un perfil de datos cada vez que se haga un requerimiento

1307

(getData). El perfil sera obtenido a partir del buffer de datos,

1307

(getData). El perfil sera obtenido a partir del buffer de datos,

1308

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1308

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1309

bloque de datos.

1309

bloque de datos.

1310

Si este parametro no es pasado se creara uno internamente.

1310

Si este parametro no es pasado se creara uno internamente.

1311

1312

Variables afectadas:

1312

Variables afectadas:

1313

self.dataOut

1313

self.dataOut

1314

1315

Return:

1315

Return:

1316

None

1316

None

1317

"""

1317

"""

1318

1319

self.isConfig = False

1319

self.isConfig = False

1320

1321

self.datablock = None

1321

self.datablock = None

1322

1323

self.utc = 0

1323

self.utc = 0

1324

1325

self.ext = ".r"

1325

self.ext = ".r"

1326

1327

self.optchar = "D"

1327

self.optchar = "D"

1328

1329

self.basicHeaderObj = BasicHeader(LOCALTIME)

1329

self.basicHeaderObj = BasicHeader(LOCALTIME)

1330

1331

self.systemHeaderObj = SystemHeader()

1331

self.systemHeaderObj = SystemHeader()

1332

1333

self.radarControllerHeaderObj = RadarControllerHeader()

1333

self.radarControllerHeaderObj = RadarControllerHeader()

1334

1335

self.processingHeaderObj = ProcessingHeader()

1335

self.processingHeaderObj = ProcessingHeader()

1336

1337

self.online = 0

1337

self.online = 0

1338

1339

self.fp = None

1339

self.fp = None

1340

1341

self.idFile = None

1341

self.idFile = None

1342

1343

self.dtype = None

1343

self.dtype = None

1344

1345

self.fileSizeByHeader = None

1345

self.fileSizeByHeader = None

1346

1347

self.filenameList = []

1347

self.filenameList = []

1348

1349

self.filename = None

1349

self.filename = None

1350

1351

self.fileSize = None

1351

self.fileSize = None

1352

1353

self.firstHeaderSize = 0

1353

self.firstHeaderSize = 0

1354

1355

self.basicHeaderSize = 24

1355

self.basicHeaderSize = 24

1356

1357

self.pathList = []

1357

self.pathList = []

1358

1359

self.filenameList = []

1359

self.filenameList = []

1360

1361

self.lastUTTime = 0

1361

self.lastUTTime = 0

1362

1363

self.maxTimeStep = 30

1363

self.maxTimeStep = 30

1364

1365

self.flagNoMoreFiles = 0

1365

self.flagNoMoreFiles = 0

1366

1367

self.set = 0

1367

self.set = 0

1368

1369

self.path = None

1369

self.path = None

1370

1371

self.profileIndex = 2**32-1

1371

self.profileIndex = 2**32-1

1372

1373

self.delay = 3 #seconds

1373

self.delay = 3 #seconds

1374

1375

self.nTries = 3 #quantity tries

1375

self.nTries = 3 #quantity tries

1376

1377

self.nFiles = 3 #number of files for searching

1377

self.nFiles = 3 #number of files for searching

1378

1379

self.nReadBlocks = 0

1379

self.nReadBlocks = 0

1380

1381

self.flagIsNewFile = 1

1381

self.flagIsNewFile = 1

1382

1383

self.__isFirstTimeOnline = 1

1383

self.__isFirstTimeOnline = 1

1384

1385

self.ippSeconds = 0

1385

self.ippSeconds = 0

1386

1387

self.flagTimeBlock = 0

1387

self.flagTimeBlock = 0

1388

1389

self.flagIsNewBlock = 0

1389

self.flagIsNewBlock = 0

1390

1391

self.nTotalBlocks = 0

1391

self.nTotalBlocks = 0

1392

1393

self.blocksize = 0

1393

self.blocksize = 0

1394

1395

self.dataOut = self.createObjByDefault()

1395

self.dataOut = self.createObjByDefault()

1396

1397

def createObjByDefault(self):

1397

def createObjByDefault(self):

1398

1399

dataObj = Voltage()

1399

dataObj = Voltage()

1400

1401

return dataObj

1401

return dataObj

1402

1403

def __hasNotDataInBuffer(self):

1403

def __hasNotDataInBuffer(self):

1404

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1404

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1405

return 1

1405

return 1

1406

return 0

1406

return 0

1407

1408

1409

def getBlockDimension(self):

1409

def getBlockDimension(self):

1410

"""

1410

"""

1411

Obtiene la cantidad de puntos a leer por cada bloque de datos

1411

Obtiene la cantidad de puntos a leer por cada bloque de datos

1412

1413

Affected:

1413

Affected:

1414

self.blocksize

1414

self.blocksize

1415

1416

Return:

1416

Return:

1417

None

1417

None

1418

"""

1418

"""

1419

pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels

1419

pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels

1420

self.blocksize = pts2read

1420

self.blocksize = pts2read

1421

1422

1423

def readBlock(self):

1423

def readBlock(self):

1424

"""

1424

"""

1425

readBlock lee el bloque de datos desde la posicion actual del puntero del archivo

1425

readBlock lee el bloque de datos desde la posicion actual del puntero del archivo

1426

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

1426

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

1427

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

1427

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

1428

es seteado a 0

1428

es seteado a 0

1429

1430

Inputs:

1430

Inputs:

1431

None

1431

None

1432

1433

Return:

1433

Return:

1434

None

1434

None

1435

1436

Affected:

1436

Affected:

1437

self.profileIndex

1437

self.profileIndex

1438

self.datablock

1438

self.datablock

1439

self.flagIsNewFile

1439

self.flagIsNewFile

1440

self.flagIsNewBlock

1440

self.flagIsNewBlock

1441

self.nTotalBlocks

1441

self.nTotalBlocks

1442

1443

Exceptions:

1443

Exceptions:

1444

Si un bloque leido no es un bloque valido

1444

Si un bloque leido no es un bloque valido

1445

"""

1445

"""

1446

1447

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1447

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1448

1449

try:

1449

try:

1450

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1450

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1451

except:

1451

except:

1452

print "The read block (%3d) has not enough data" %self.nReadBlocks

1452

print "The read block (%3d) has not enough data" %self.nReadBlocks

1453

return 0

1453

return 0

1454

1455

junk = numpy.transpose(junk, (2,0,1))

1455

junk = numpy.transpose(junk, (2,0,1))

1456

self.datablock = junk['real'] + junk['imag']*1j

1456

self.datablock = junk['real'] + junk['imag']*1j

1457

1458

self.profileIndex = 0

1458

self.profileIndex = 0

1459

1460

self.flagIsNewFile = 0

1460

self.flagIsNewFile = 0

1461

self.flagIsNewBlock = 1

1461

self.flagIsNewBlock = 1

1462

1463

self.nTotalBlocks += 1

1463

self.nTotalBlocks += 1

1464

self.nReadBlocks += 1

1464

self.nReadBlocks += 1

1465

1466

return 1

1466

return 1

1467

1468

def getFirstHeader(self):

1468

def getFirstHeader(self):

1469

1470

self.dataOut.dtype = self.dtype

1470

self.dataOut.dtype = self.dtype

1471

1472

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

1472

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

1473

1474

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

1474

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

1475

1476

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

1476

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

1477

1478

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

1478

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

1479

1480

self.dataOut.ippSeconds = self.ippSeconds

1480

self.dataOut.ippSeconds = self.ippSeconds

1481

1482

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt

1482

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt

1483

1484

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

1484

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

1485

1486

self.dataOut.flagShiftFFT = False

1486

self.dataOut.flagShiftFFT = False

1487

1488

if self.radarControllerHeaderObj.code != None:

1488

if self.radarControllerHeaderObj.code != None:

1489

1490

self.dataOut.nCode = self.radarControllerHeaderObj.nCode

1490

self.dataOut.nCode = self.radarControllerHeaderObj.nCode

1491

1492

self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud

1492

self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud

1493

1494

self.dataOut.code = self.radarControllerHeaderObj.code

1494

self.dataOut.code = self.radarControllerHeaderObj.code

1495

1496

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

1496

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

1497

1498

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

1498

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

1499

1500

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

1500

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

1501

1502

self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip

1502

self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip

1503

1504

self.dataOut.flagShiftFFT = False

1504

self.dataOut.flagShiftFFT = False

1505

1506

def getData(self):

1506

def getData(self):

1507

"""

1507

"""

1508

getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"

1508

getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"

1509

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

1509

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

1510

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

1510

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

1511

1512

Ademas incrementa el contador del buffer en 1.

1512

Ademas incrementa el contador del buffer en 1.

1513

1514

Return:

1514

Return:

1515

data : retorna un perfil de voltages (alturas * canales) copiados desde el

1515

data : retorna un perfil de voltages (alturas * canales) copiados desde el

1516

buffer. Si no hay mas archivos a leer retorna None.

1516

buffer. Si no hay mas archivos a leer retorna None.

1517

1518

Variables afectadas:

1518

Variables afectadas:

1519

self.dataOut

1519

self.dataOut

1520

self.profileIndex

1520

self.profileIndex

1521

1522

Affected:

1522

Affected:

1523

self.dataOut

1523

self.dataOut

1524

self.profileIndex

1524

self.profileIndex

1525

self.flagTimeBlock

1525

self.flagTimeBlock

1526

self.flagIsNewBlock

1526

self.flagIsNewBlock

1527

"""

1527

"""

1528

1529

if self.flagNoMoreFiles:

1529

if self.flagNoMoreFiles:

1530

self.dataOut.flagNoData = True

1530

self.dataOut.flagNoData = True

1531

print 'Process finished'

1531

print 'Process finished'

1532

return 0

1532

return 0

1533

1534

self.flagTimeBlock = 0

1534

self.flagTimeBlock = 0

1535

self.flagIsNewBlock = 0

1535

self.flagIsNewBlock = 0

1536

1537

if self.__hasNotDataInBuffer():

1537

if self.__hasNotDataInBuffer():

1538

1539

if not( self.readNextBlock() ):

1539

if not( self.readNextBlock() ):

1540

return 0

1540

return 0

1541

1542

self.getFirstHeader()

1542

self.getFirstHeader()

1543

1544

if self.datablock == None:

1544

if self.datablock == None:

1545

self.dataOut.flagNoData = True

1545

self.dataOut.flagNoData = True

1546

return 0

1546

return 0

1547

1548

self.dataOut.data = self.datablock[:,self.profileIndex,:]

1548

self.dataOut.data = self.datablock[:,self.profileIndex,:]

1549

1550

self.dataOut.flagNoData = False

1550

self.dataOut.flagNoData = False

1551

1552

self.getBasicHeader()

1552

self.getBasicHeader()

1553

1554

self.profileIndex += 1

1554

self.profileIndex += 1

1555

1556

return self.dataOut.data

1556

return self.dataOut.data

1557

1558

1559

class VoltageWriter(JRODataWriter):

1559

class VoltageWriter(JRODataWriter):

1560

"""

1560

"""

1561

Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura

1561

Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura

1562

de los datos siempre se realiza por bloques.

1562

de los datos siempre se realiza por bloques.

1563

"""

1563

"""

1564

1565

ext = ".r"

1565

ext = ".r"

1566

1567

optchar = "D"

1567

optchar = "D"

1568

1569

shapeBuffer = None

1569

shapeBuffer = None

1570

1571

1572

def __init__(self):

1572

def __init__(self):

1573

"""

1573

"""

1574

Inicializador de la clase VoltageWriter para la escritura de datos de espectros.

1574

Inicializador de la clase VoltageWriter para la escritura de datos de espectros.

1575

1576

Affected:

1576

Affected:

1577

self.dataOut

1577

self.dataOut

1578

1579

Return: None

1579

Return: None

1580

"""

1580

"""

1581

1582

self.nTotalBlocks = 0

1582

self.nTotalBlocks = 0

1583

1584

self.profileIndex = 0

1584

self.profileIndex = 0

1585

1586

self.isConfig = False

1586

self.isConfig = False

1587

1588

self.fp = None

1588

self.fp = None

1589

1590

self.flagIsNewFile = 1

1590

self.flagIsNewFile = 1

1591

1592

self.nTotalBlocks = 0

1592

self.nTotalBlocks = 0

1593

1594

self.flagIsNewBlock = 0

1594

self.flagIsNewBlock = 0

1595

1596

self.setFile = None

1596

self.setFile = None

1597

1598

self.dtype = None

1598

self.dtype = None

1599

1600

self.path = None

1600

self.path = None

1601

1602

self.filename = None

1602

self.filename = None

1603

1604

self.basicHeaderObj = BasicHeader(LOCALTIME)

1604

self.basicHeaderObj = BasicHeader(LOCALTIME)

1605

1606

self.systemHeaderObj = SystemHeader()

1606

self.systemHeaderObj = SystemHeader()

1607

1608

self.radarControllerHeaderObj = RadarControllerHeader()

1608

self.radarControllerHeaderObj = RadarControllerHeader()

1609

1610

self.processingHeaderObj = ProcessingHeader()

1610

self.processingHeaderObj = ProcessingHeader()

1611

1612

def hasAllDataInBuffer(self):

1612

def hasAllDataInBuffer(self):

1613

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1613

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1614

return 1

1614

return 1

1615

return 0

1615

return 0

1616

1617

1618

def setBlockDimension(self):

1618

def setBlockDimension(self):

1619

"""

1619

"""

1620

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

1620

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

1621

1622

Affected:

1622

Affected:

1623

self.shape_spc_Buffer

1623

self.shape_spc_Buffer

1624

self.shape_cspc_Buffer

1624

self.shape_cspc_Buffer

1625

self.shape_dc_Buffer

1625

self.shape_dc_Buffer

1626

1627

Return: None

1627

Return: None

1628

"""

1628

"""

1629

self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,

1629

self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,

1630

self.processingHeaderObj.nHeights,

1630

self.processingHeaderObj.nHeights,

1631

self.systemHeaderObj.nChannels)

1631

self.systemHeaderObj.nChannels)

1632

1633

self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,

1633

self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,

1634

self.processingHeaderObj.profilesPerBlock,

1634

self.processingHeaderObj.profilesPerBlock,

1635

self.processingHeaderObj.nHeights),

1635

self.processingHeaderObj.nHeights),

1636

dtype=numpy.dtype('complex64'))

1636

dtype=numpy.dtype('complex64'))

1637

1638

1639

def writeBlock(self):

1639

def writeBlock(self):

1640

"""

1640

"""

1641

Escribe el buffer en el file designado

1641

Escribe el buffer en el file designado

1642

1643

Affected:

1643

Affected:

1644

self.profileIndex

1644

self.profileIndex

1645

self.flagIsNewFile

1645

self.flagIsNewFile

1646

self.flagIsNewBlock

1646

self.flagIsNewBlock

1647

self.nTotalBlocks

1647

self.nTotalBlocks

1648

self.blockIndex

1648

self.blockIndex

1649

1650

Return: None

1650

Return: None

1651

"""

1651

"""

1652

data = numpy.zeros( self.shapeBuffer, self.dtype )

1652

data = numpy.zeros( self.shapeBuffer, self.dtype )

1653

1654

junk = numpy.transpose(self.datablock, (1,2,0))

1654

junk = numpy.transpose(self.datablock, (1,2,0))

1655

1656

data['real'] = junk.real

1656

data['real'] = junk.real

1657

data['imag'] = junk.imag

1657

data['imag'] = junk.imag

1658

1659

data = data.reshape( (-1) )

1659

data = data.reshape( (-1) )

1660

1661

data.tofile( self.fp )

1661

data.tofile( self.fp )

1662

1663

self.datablock.fill(0)

1663

self.datablock.fill(0)

1664

1665

self.profileIndex = 0

1665

self.profileIndex = 0

1666

self.flagIsNewFile = 0

1666

self.flagIsNewFile = 0

1667

self.flagIsNewBlock = 1

1667

self.flagIsNewBlock = 1

1668

1669

self.blockIndex += 1

1669

self.blockIndex += 1

1670

self.nTotalBlocks += 1

1670

self.nTotalBlocks += 1

1671

1672

def putData(self):

1672

def putData(self):

1673

"""

1673

"""

1674

Setea un bloque de datos y luego los escribe en un file

1674

Setea un bloque de datos y luego los escribe en un file

1675

1676

Affected:

1676

Affected:

1677

self.flagIsNewBlock

1677

self.flagIsNewBlock

1678

self.profileIndex

1678

self.profileIndex

1679

1680

Return:

1680

Return:

1681

0 : Si no hay data o no hay mas files que puedan escribirse

1681

0 : Si no hay data o no hay mas files que puedan escribirse

1682

1 : Si se escribio la data de un bloque en un file

1682

1 : Si se escribio la data de un bloque en un file

1683

"""

1683

"""

1684

if self.dataOut.flagNoData:

1684

if self.dataOut.flagNoData:

1685

return 0

1685

return 0

1686

1687

self.flagIsNewBlock = 0

1687

self.flagIsNewBlock = 0

1688

1689

if self.dataOut.flagTimeBlock:

1689

if self.dataOut.flagTimeBlock:

1690

1691

self.datablock.fill(0)

1691

self.datablock.fill(0)

1692

self.profileIndex = 0

1692

self.profileIndex = 0

1693

self.setNextFile()

1693

self.setNextFile()

1694

1695

if self.profileIndex == 0:

1695

if self.profileIndex == 0:

1696

self.setBasicHeader()

1696

self.setBasicHeader()

1697

1698

self.datablock[:,self.profileIndex,:] = self.dataOut.data

1698

self.datablock[:,self.profileIndex,:] = self.dataOut.data

1699

1700

self.profileIndex += 1

1700

self.profileIndex += 1

1701

1702

if self.hasAllDataInBuffer():

1702

if self.hasAllDataInBuffer():

1703

#if self.flagIsNewFile:

1703

#if self.flagIsNewFile:

1704

self.writeNextBlock()

1704

self.writeNextBlock()

1705

# self.setFirstHeader()

1705

# self.setFirstHeader()

1706

1707

return 1

1707

return 1

1708

1709

def __getProcessFlags(self):

1709

def __getProcessFlags(self):

1710

1711

processFlags = 0

1711

processFlags = 0

1712

1713

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1713

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1714

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1714

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1715

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1715

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1716

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1716

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1717

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1717

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1718

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1718

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1719

1720

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1720

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1721

1722

1723

1724

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

1724

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

1725

PROCFLAG.DATATYPE_SHORT,

1725

PROCFLAG.DATATYPE_SHORT,

1726

PROCFLAG.DATATYPE_LONG,

1726

PROCFLAG.DATATYPE_LONG,

1727

PROCFLAG.DATATYPE_INT64,

1727

PROCFLAG.DATATYPE_INT64,

1728

PROCFLAG.DATATYPE_FLOAT,

1728

PROCFLAG.DATATYPE_FLOAT,

1729

PROCFLAG.DATATYPE_DOUBLE]

1729

PROCFLAG.DATATYPE_DOUBLE]

1730

1731

1732

for index in range(len(dtypeList)):

1732

for index in range(len(dtypeList)):

1733

if self.dataOut.dtype == dtypeList[index]:

1733

if self.dataOut.dtype == dtypeList[index]:

1734

dtypeValue = datatypeValueList[index]

1734

dtypeValue = datatypeValueList[index]

1735

break

1735

break

1736

1737

processFlags += dtypeValue

1737

processFlags += dtypeValue

1738

1739

if self.dataOut.flagDecodeData:

1739

if self.dataOut.flagDecodeData:

1740

processFlags += PROCFLAG.DECODE_DATA

1740

processFlags += PROCFLAG.DECODE_DATA

1741

1742

if self.dataOut.flagDeflipData:

1742

if self.dataOut.flagDeflipData:

1743

processFlags += PROCFLAG.DEFLIP_DATA

1743

processFlags += PROCFLAG.DEFLIP_DATA

1744

1745

if self.dataOut.code != None:

1745

if self.dataOut.code != None:

1746

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

1746

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

1747

1748

if self.dataOut.nCohInt > 1:

1748

if self.dataOut.nCohInt > 1:

1749

processFlags += PROCFLAG.COHERENT_INTEGRATION

1749

processFlags += PROCFLAG.COHERENT_INTEGRATION

1750

1751

return processFlags

1751

return processFlags

1752

1753

1754

def __getBlockSize(self):

1754

def __getBlockSize(self):

1755

'''

1755

'''

1756

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage

1756

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage

1757

'''

1757

'''

1758

1759

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1759

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1760

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1760

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1761

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1761

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1762

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1762

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1763

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1763

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1764

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1764

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1765

1766

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1766

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1767

datatypeValueList = [1,2,4,8,4,8]

1767

datatypeValueList = [1,2,4,8,4,8]

1768

for index in range(len(dtypeList)):

1768

for index in range(len(dtypeList)):

1769

if self.dataOut.dtype == dtypeList[index]:

1769

if self.dataOut.dtype == dtypeList[index]:

1770

datatypeValue = datatypeValueList[index]

1770

datatypeValue = datatypeValueList[index]

1771

break

1771

break

1772

1773

blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.dataOut.nProfiles * datatypeValue * 2)

1773

blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.dataOut.nProfiles * datatypeValue * 2)

1774

1775

return blocksize

1775

return blocksize

1776

1777

def setFirstHeader(self):

1777

def setFirstHeader(self):

1778

1779

"""

1779

"""

1780

Obtiene una copia del First Header

1780

Obtiene una copia del First Header

1781

1782

Affected:

1782

Affected:

1783

self.systemHeaderObj

1783

self.systemHeaderObj

1784

self.radarControllerHeaderObj

1784

self.radarControllerHeaderObj

1785

self.dtype

1785

self.dtype

1786

1787

Return:

1787

Return:

1788

None

1788

None

1789

"""

1789

"""

1790

1791

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

1791

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

1792

self.systemHeaderObj.nChannels = self.dataOut.nChannels

1792

self.systemHeaderObj.nChannels = self.dataOut.nChannels

1793

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

1793

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

1794

1795

self.setBasicHeader()

1795

self.setBasicHeader()

1796

1797

processingHeaderSize = 40 # bytes

1797

processingHeaderSize = 40 # bytes

1798

self.processingHeaderObj.dtype = 0 # Voltage

1798

self.processingHeaderObj.dtype = 0 # Voltage

1799

self.processingHeaderObj.blockSize = self.__getBlockSize()

1799

self.processingHeaderObj.blockSize = self.__getBlockSize()

1800

self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock

1800

self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock

1801

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

1801

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

1802

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

1802

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

1803

self.processingHeaderObj.processFlags = self.__getProcessFlags()

1803

self.processingHeaderObj.processFlags = self.__getProcessFlags()

1804

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt

1804

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt

1805

self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage

1805

self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage

1806

self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage

1806

self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage

1807

1808

if self.dataOut.code != None:

1808

if self.dataOut.code != None:

1809

self.processingHeaderObj.code = self.dataOut.code

1809

self.processingHeaderObj.code = self.dataOut.code

1810

self.processingHeaderObj.nCode = self.dataOut.nCode

1810

self.processingHeaderObj.nCode = self.dataOut.nCode

1811

self.processingHeaderObj.nBaud = self.dataOut.nBaud

1811

self.processingHeaderObj.nBaud = self.dataOut.nBaud

1812

codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)

1812

codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)

1813

processingHeaderSize += codesize

1813

processingHeaderSize += codesize

1814

1815

if self.processingHeaderObj.nWindows != 0:

1815

if self.processingHeaderObj.nWindows != 0:

1816

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

1816

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

1817

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

1817

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

1818

self.processingHeaderObj.nHeights = self.dataOut.nHeights

1818

self.processingHeaderObj.nHeights = self.dataOut.nHeights

1819

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

1819

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

1820

processingHeaderSize += 12

1820

processingHeaderSize += 12

1821

1822

self.processingHeaderObj.size = processingHeaderSize

1822

self.processingHeaderObj.size = processingHeaderSize

1823

1824

class SpectraReader(JRODataReader):

1824

class SpectraReader(JRODataReader):

1825

"""

1825

"""

1826

Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura

1826

Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura

1827

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)

1827

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)

1828

son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.

1828

son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.

1829

1830

paresCanalesIguales * alturas * perfiles (Self Spectra)

1830

paresCanalesIguales * alturas * perfiles (Self Spectra)

1831

paresCanalesDiferentes * alturas * perfiles (Cross Spectra)

1831

paresCanalesDiferentes * alturas * perfiles (Cross Spectra)

1832

canales * alturas (DC Channels)

1832

canales * alturas (DC Channels)

1833

1834

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1834

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1835

RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la

1835

RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la

1836

cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de

1836

cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de

1837

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1837

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1838

1839

Example:

1839

Example:

1840

dpath = "/home/myuser/data"

1840

dpath = "/home/myuser/data"

1841

1842

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1842

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1843

1844

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1844

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1845

1846

readerObj = SpectraReader()

1846

readerObj = SpectraReader()

1847

1848

readerObj.setup(dpath, startTime, endTime)

1848

readerObj.setup(dpath, startTime, endTime)

1849

1850

while(True):

1850

while(True):

1851

1852

readerObj.getData()

1852

readerObj.getData()

1853

1854

print readerObj.data_spc

1854

print readerObj.data_spc

1855

1856

print readerObj.data_cspc

1856

print readerObj.data_cspc

1857

1858

print readerObj.data_dc

1858

print readerObj.data_dc

1859

1860

if readerObj.flagNoMoreFiles:

1860

if readerObj.flagNoMoreFiles:

1861

break

1861

break

1862

1863

"""

1863

"""

1864

1865

pts2read_SelfSpectra = 0

1865

pts2read_SelfSpectra = 0

1866

1867

pts2read_CrossSpectra = 0

1867

pts2read_CrossSpectra = 0

1868

1869

pts2read_DCchannels = 0

1869

pts2read_DCchannels = 0

1870

1871

ext = ".pdata"

1871

ext = ".pdata"

1872

1873

optchar = "P"

1873

optchar = "P"

1874

1875

dataOut = None

1875

dataOut = None

1876

1877

nRdChannels = None

1877

nRdChannels = None

1878

1879

nRdPairs = None

1879

nRdPairs = None

1880

1881

rdPairList = []

1881

rdPairList = []

1882

1883

def __init__(self):

1883

def __init__(self):

1884

"""

1884

"""

1885

Inicializador de la clase SpectraReader para la lectura de datos de espectros.

1885

Inicializador de la clase SpectraReader para la lectura de datos de espectros.

1886

1887

Inputs:

1887

Inputs:

1888

dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para

1888

dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para

1889

almacenar un perfil de datos cada vez que se haga un requerimiento

1889

almacenar un perfil de datos cada vez que se haga un requerimiento

1890

(getData). El perfil sera obtenido a partir del buffer de datos,

1890

(getData). El perfil sera obtenido a partir del buffer de datos,

1891

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1891

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1892

bloque de datos.

1892

bloque de datos.

1893

Si este parametro no es pasado se creara uno internamente.

1893

Si este parametro no es pasado se creara uno internamente.

1894

1895

Affected:

1895

Affected:

1896

self.dataOut

1896

self.dataOut

1897

1898

Return : None

1898

Return : None

1899

"""

1899

"""

1900

1901

self.isConfig = False

1901

self.isConfig = False

1902

1903

self.pts2read_SelfSpectra = 0

1903

self.pts2read_SelfSpectra = 0

1904

1905

self.pts2read_CrossSpectra = 0

1905

self.pts2read_CrossSpectra = 0

1906

1907

self.pts2read_DCchannels = 0

1907

self.pts2read_DCchannels = 0

1908

1909

self.datablock = None

1909

self.datablock = None

1910

1911

self.utc = None

1911

self.utc = None

1912

1913

self.ext = ".pdata"

1913

self.ext = ".pdata"

1914

1915

self.optchar = "P"

1915

self.optchar = "P"

1916

1917

self.basicHeaderObj = BasicHeader(LOCALTIME)

1917

self.basicHeaderObj = BasicHeader(LOCALTIME)

1918

1919

self.systemHeaderObj = SystemHeader()

1919

self.systemHeaderObj = SystemHeader()

1920

1921

self.radarControllerHeaderObj = RadarControllerHeader()

1921

self.radarControllerHeaderObj = RadarControllerHeader()

1922

1923

self.processingHeaderObj = ProcessingHeader()

1923

self.processingHeaderObj = ProcessingHeader()

1924

1925

self.online = 0

1925

self.online = 0

1926

1927

self.fp = None

1927

self.fp = None

1928

1929

self.idFile = None

1929

self.idFile = None

1930

1931

self.dtype = None

1931

self.dtype = None

1932

1933

self.fileSizeByHeader = None

1933

self.fileSizeByHeader = None

1934

1935

self.filenameList = []

1935

self.filenameList = []

1936

1937

self.filename = None

1937

self.filename = None

1938

1939

self.fileSize = None

1939

self.fileSize = None

1940

1941

self.firstHeaderSize = 0

1941

self.firstHeaderSize = 0

1942

1943

self.basicHeaderSize = 24

1943

self.basicHeaderSize = 24

1944

1945

self.pathList = []

1945

self.pathList = []

1946

1947

self.lastUTTime = 0

1947

self.lastUTTime = 0

1948

1949

self.maxTimeStep = 30

1949

self.maxTimeStep = 30

1950

1951

self.flagNoMoreFiles = 0

1951

self.flagNoMoreFiles = 0

1952

1953

self.set = 0

1953

self.set = 0

1954

1955

self.path = None

1955

self.path = None

1956

1957

self.delay = 60 #seconds

1957

self.delay = 60 #seconds

1958

1959

self.nTries = 3 #quantity tries

1959

self.nTries = 3 #quantity tries

1960

1961

self.nFiles = 3 #number of files for searching

1961

self.nFiles = 3 #number of files for searching

1962

1963

self.nReadBlocks = 0

1963

self.nReadBlocks = 0

1964

1965

self.flagIsNewFile = 1

1965

self.flagIsNewFile = 1

1966

1967

self.__isFirstTimeOnline = 1

1967

self.__isFirstTimeOnline = 1

1968

1969

self.ippSeconds = 0

1969

self.ippSeconds = 0

1970

1971

self.flagTimeBlock = 0

1971

self.flagTimeBlock = 0

1972

1973

self.flagIsNewBlock = 0

1973

self.flagIsNewBlock = 0

1974

1975

self.nTotalBlocks = 0

1975

self.nTotalBlocks = 0

1976

1977

self.blocksize = 0

1977

self.blocksize = 0

1978

1979

self.dataOut = self.createObjByDefault()

1979

self.dataOut = self.createObjByDefault()

1980

1981

self.profileIndex = 1 #Always

1981

self.profileIndex = 1 #Always

1982

1983

1984

def createObjByDefault(self):

1984

def createObjByDefault(self):

1985

1986

dataObj = Spectra()

1986

dataObj = Spectra()

1987

1988

return dataObj

1988

return dataObj

1989

1990

def __hasNotDataInBuffer(self):

1990

def __hasNotDataInBuffer(self):

1991

return 1

1991

return 1

1992

1993

1994

def getBlockDimension(self):

1994

def getBlockDimension(self):

1995

"""

1995

"""

1996

Obtiene la cantidad de puntos a leer por cada bloque de datos

1996

Obtiene la cantidad de puntos a leer por cada bloque de datos

1997

1998

Affected:

1998

Affected:

1999

self.nRdChannels

1999

self.nRdChannels

2000

self.nRdPairs

2000

self.nRdPairs

2001

self.pts2read_SelfSpectra

2001

self.pts2read_SelfSpectra

2002

self.pts2read_CrossSpectra

2002

self.pts2read_CrossSpectra

2003

self.pts2read_DCchannels

2003

self.pts2read_DCchannels

2004

self.blocksize

2004

self.blocksize

2005

self.dataOut.nChannels

2005

self.dataOut.nChannels

2006

self.dataOut.nPairs

2006

self.dataOut.nPairs

2007

2008

Return:

2008

Return:

2009

None

2009

None

2010

"""

2010

"""

2011

self.nRdChannels = 0

2011

self.nRdChannels = 0

2012

self.nRdPairs = 0

2012

self.nRdPairs = 0

2013

self.rdPairList = []

2013

self.rdPairList = []

2014

2015

for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):

2015

for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):

2016

if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:

2016

if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:

2017

self.nRdChannels = self.nRdChannels + 1 #par de canales iguales

2017

self.nRdChannels = self.nRdChannels + 1 #par de canales iguales

2018

else:

2018

else:

2019

self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes

2019

self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes

2020

self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))

2020

self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))

2021

2022

pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock

2022

pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock

2023

2024

self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)

2024

self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)

2025

self.blocksize = self.pts2read_SelfSpectra

2025

self.blocksize = self.pts2read_SelfSpectra

2026

2027

if self.processingHeaderObj.flag_cspc:

2027

if self.processingHeaderObj.flag_cspc:

2028

self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)

2028

self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)

2029

self.blocksize += self.pts2read_CrossSpectra

2029

self.blocksize += self.pts2read_CrossSpectra

2030

2031

if self.processingHeaderObj.flag_dc:

2031

if self.processingHeaderObj.flag_dc:

2032

self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)

2032

self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)

2033

self.blocksize += self.pts2read_DCchannels

2033

self.blocksize += self.pts2read_DCchannels

2034

2035

# self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels

2035

# self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels

2036

2037

2038

def readBlock(self):

2038

def readBlock(self):

2039

"""

2039

"""

2040

Lee el bloque de datos desde la posicion actual del puntero del archivo

2040

Lee el bloque de datos desde la posicion actual del puntero del archivo

2041

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

2041

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

2042

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

2042

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

2043

es seteado a 0

2043

es seteado a 0

2044

2045

Return: None

2045

Return: None

2046

2047

Variables afectadas:

2047

Variables afectadas:

2048

2049

self.flagIsNewFile

2049

self.flagIsNewFile

2050

self.flagIsNewBlock

2050

self.flagIsNewBlock

2051

self.nTotalBlocks

2051

self.nTotalBlocks

2052

self.data_spc

2052

self.data_spc

2053

self.data_cspc

2053

self.data_cspc

2054

self.data_dc

2054

self.data_dc

2055

2056

Exceptions:

2056

Exceptions:

2057

Si un bloque leido no es un bloque valido

2057

Si un bloque leido no es un bloque valido

2058

"""

2058

"""

2059

blockOk_flag = False

2059

blockOk_flag = False

2060

fpointer = self.fp.tell()

2060

fpointer = self.fp.tell()

2061

2062

spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )

2062

spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )

2063

spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2063

spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2064

2065

if self.processingHeaderObj.flag_cspc:

2065

if self.processingHeaderObj.flag_cspc:

2066

cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )

2066

cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )

2067

cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2067

cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2068

2069

if self.processingHeaderObj.flag_dc:

2069

if self.processingHeaderObj.flag_dc:

2070

dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )

2070

dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )

2071

dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D

2071

dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D

2072

2073

2074

if not(self.processingHeaderObj.shif_fft):

2074

if not(self.processingHeaderObj.shif_fft):

2075

#desplaza a la derecha en el eje 2 determinadas posiciones

2075

#desplaza a la derecha en el eje 2 determinadas posiciones

2076

shift = int(self.processingHeaderObj.profilesPerBlock/2)

2076

shift = int(self.processingHeaderObj.profilesPerBlock/2)

2077

spc = numpy.roll( spc, shift , axis=2 )

2077

spc = numpy.roll( spc, shift , axis=2 )

2078

2079

if self.processingHeaderObj.flag_cspc:

2079

if self.processingHeaderObj.flag_cspc:

2080

#desplaza a la derecha en el eje 2 determinadas posiciones

2080

#desplaza a la derecha en el eje 2 determinadas posiciones

2081

cspc = numpy.roll( cspc, shift, axis=2 )

2081

cspc = numpy.roll( cspc, shift, axis=2 )

2082

2083

# self.processingHeaderObj.shif_fft = True

2083

# self.processingHeaderObj.shif_fft = True

2084

2085

spc = numpy.transpose( spc, (0,2,1) )

2085

spc = numpy.transpose( spc, (0,2,1) )

2086

self.data_spc = spc

2086

self.data_spc = spc

2087

2088

if self.processingHeaderObj.flag_cspc:

2088

if self.processingHeaderObj.flag_cspc:

2089

cspc = numpy.transpose( cspc, (0,2,1) )

2089

cspc = numpy.transpose( cspc, (0,2,1) )

2090

self.data_cspc = cspc['real'] + cspc['imag']*1j

2090

self.data_cspc = cspc['real'] + cspc['imag']*1j

2091

else:

2091

else:

2092

self.data_cspc = None

2092

self.data_cspc = None

2093

2094

if self.processingHeaderObj.flag_dc:

2094

if self.processingHeaderObj.flag_dc:

2095

self.data_dc = dc['real'] + dc['imag']*1j

2095

self.data_dc = dc['real'] + dc['imag']*1j

2096

else:

2096

else:

2097

self.data_dc = None

2097

self.data_dc = None

2098

2099

self.flagIsNewFile = 0

2099

self.flagIsNewFile = 0

2100

self.flagIsNewBlock = 1

2100

self.flagIsNewBlock = 1

2101

2102

self.nTotalBlocks += 1

2102

self.nTotalBlocks += 1

2103

self.nReadBlocks += 1

2103

self.nReadBlocks += 1

2104

2105

return 1

2105

return 1

2106

2107

def getFirstHeader(self):

2107

def getFirstHeader(self):

2108

2109

self.dataOut.dtype = self.dtype

2109

self.dataOut.dtype = self.dtype

2110

2111

self.dataOut.nPairs = self.nRdPairs

2111

self.dataOut.nPairs = self.nRdPairs

2112

2113

self.dataOut.pairsList = self.rdPairList

2113

self.dataOut.pairsList = self.rdPairList

2114

2115

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

2115

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

2116

2117

self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock

2117

self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock

2118

2119

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

2119

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

2120

2121

self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt

2121

self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt

2122

2123

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

2123

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

2124

2125

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

2125

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

2126

2127

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

2127

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

2128

2129

self.dataOut.ippSeconds = self.ippSeconds

2129

self.dataOut.ippSeconds = self.ippSeconds

2130

2131

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints

2131

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints

2132

2133

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

2133

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

2134

2135

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

2135

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

2136

2137

self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft

2137

self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft

2138

2139

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

2139

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

2140

2141

self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip

2141

self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip

2142

2143

if self.processingHeaderObj.code != None:

2143

if self.processingHeaderObj.code != None:

2144

2145

self.dataOut.nCode = self.processingHeaderObj.nCode

2145

self.dataOut.nCode = self.processingHeaderObj.nCode

2146

2147

self.dataOut.nBaud = self.processingHeaderObj.nBaud

2147

self.dataOut.nBaud = self.processingHeaderObj.nBaud

2148

2149

self.dataOut.code = self.processingHeaderObj.code

2149

self.dataOut.code = self.processingHeaderObj.code

2150

2151

self.dataOut.flagDecodeData = True

2151

self.dataOut.flagDecodeData = True

2152

2153

def getData(self):

2153

def getData(self):

2154

"""

2154

"""

2155

Copia el buffer de lectura a la clase "Spectra",

2155

Copia el buffer de lectura a la clase "Spectra",

2156

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

2156

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

2157

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

2157

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

2158

2159

Return:

2159

Return:

2160

0 : Si no hay mas archivos disponibles

2160

0 : Si no hay mas archivos disponibles

2161

1 : Si hizo una buena copia del buffer

2161

1 : Si hizo una buena copia del buffer

2162

2163

Affected:

2163

Affected:

2164

self.dataOut

2164

self.dataOut

2165

2166

self.flagTimeBlock

2166

self.flagTimeBlock

2167

self.flagIsNewBlock

2167

self.flagIsNewBlock

2168

"""

2168

"""

2169

2170

if self.flagNoMoreFiles:

2170

if self.flagNoMoreFiles:

2171

self.dataOut.flagNoData = True

2171

self.dataOut.flagNoData = True

2172

print 'Process finished'

2172

print 'Process finished'

2173

return 0

2173

return 0

2174

2175

self.flagTimeBlock = 0

2175

self.flagTimeBlock = 0

2176

self.flagIsNewBlock = 0

2176

self.flagIsNewBlock = 0

2177

2178

if self.__hasNotDataInBuffer():

2178

if self.__hasNotDataInBuffer():

2179

2180

if not( self.readNextBlock() ):

2180

if not( self.readNextBlock() ):

2181

self.dataOut.flagNoData = True

2181

self.dataOut.flagNoData = True

2182

return 0

2182

return 0

2183

2184

#data es un numpy array de 3 dmensiones (perfiles, alturas y canales)

2184

#data es un numpy array de 3 dmensiones (perfiles, alturas y canales)

2185

2186

if self.data_dc == None:

2186

if self.data_dc == None:

2187

self.dataOut.flagNoData = True

2187

self.dataOut.flagNoData = True

2188

return 0

2188

return 0

2189

2190

self.getBasicHeader()

2190

self.getBasicHeader()

2191

2192

self.getFirstHeader()

2192

self.getFirstHeader()

2193

2194

self.dataOut.data_spc = self.data_spc

2194

self.dataOut.data_spc = self.data_spc

2195

2196

self.dataOut.data_cspc = self.data_cspc

2196

self.dataOut.data_cspc = self.data_cspc

2197

2198

self.dataOut.data_dc = self.data_dc

2198

self.dataOut.data_dc = self.data_dc

2199

2200

self.dataOut.flagNoData = False

2200

self.dataOut.flagNoData = False

2201

2202

return self.dataOut.data_spc

2202

return self.dataOut.data_spc

2203

2204

2205

class SpectraWriter(JRODataWriter):

2205

class SpectraWriter(JRODataWriter):

2206

2207

"""

2207

"""

2208

Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura

2208

Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura

2209

de los datos siempre se realiza por bloques.

2209

de los datos siempre se realiza por bloques.

2210

"""

2210

"""

2211

2212

ext = ".pdata"

2212

ext = ".pdata"

2213

2214

optchar = "P"

2214

optchar = "P"

2215

2216

shape_spc_Buffer = None

2216

shape_spc_Buffer = None

2217

2218

shape_cspc_Buffer = None

2218

shape_cspc_Buffer = None

2219

2220

shape_dc_Buffer = None

2220

shape_dc_Buffer = None

2221

2222

data_spc = None

2222

data_spc = None

2223

2224

data_cspc = None

2224

data_cspc = None

2225

2226

data_dc = None

2226

data_dc = None

2227

2228

# dataOut = None

2228

# dataOut = None

2229

2230

def __init__(self):

2230

def __init__(self):

2231

"""

2231

"""

2232

Inicializador de la clase SpectraWriter para la escritura de datos de espectros.

2232

Inicializador de la clase SpectraWriter para la escritura de datos de espectros.

2233

2234

Affected:

2234

Affected:

2235

self.dataOut

2235

self.dataOut

2236

self.basicHeaderObj

2236

self.basicHeaderObj

2237

self.systemHeaderObj

2237

self.systemHeaderObj

2238

self.radarControllerHeaderObj

2238

self.radarControllerHeaderObj

2239

self.processingHeaderObj

2239

self.processingHeaderObj

2240

2241

Return: None

2241

Return: None

2242

"""

2242

"""

2243

2244

self.isConfig = False

2244

self.isConfig = False

2245

2246

self.nTotalBlocks = 0

2246

self.nTotalBlocks = 0

2247

2248

self.data_spc = None

2248

self.data_spc = None

2249

2250

self.data_cspc = None

2250

self.data_cspc = None

2251

2252

self.data_dc = None

2252

self.data_dc = None

2253

2254

self.fp = None

2254

self.fp = None

2255

2256

self.flagIsNewFile = 1

2256

self.flagIsNewFile = 1

2257

2258

self.nTotalBlocks = 0

2258

self.nTotalBlocks = 0

2259

2260

self.flagIsNewBlock = 0

2260

self.flagIsNewBlock = 0

2261

2262

self.setFile = None

2262

self.setFile = None

2263

2264

self.dtype = None

2264

self.dtype = None

2265

2266

self.path = None

2266

self.path = None

2267

2268

self.noMoreFiles = 0

2268

self.noMoreFiles = 0

2269

2270

self.filename = None

2270

self.filename = None

2271

2272

self.basicHeaderObj = BasicHeader(LOCALTIME)

2272

self.basicHeaderObj = BasicHeader(LOCALTIME)

2273

2274

self.systemHeaderObj = SystemHeader()

2274

self.systemHeaderObj = SystemHeader()

2275

2276

self.radarControllerHeaderObj = RadarControllerHeader()

2276

self.radarControllerHeaderObj = RadarControllerHeader()

2277

2278

self.processingHeaderObj = ProcessingHeader()

2278

self.processingHeaderObj = ProcessingHeader()

2279

2280

2281

def hasAllDataInBuffer(self):

2281

def hasAllDataInBuffer(self):

2282

return 1

2282

return 1

2283

2284

2285

def setBlockDimension(self):

2285

def setBlockDimension(self):

2286

"""

2286

"""

2287

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

2287

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

2288

2289

Affected:

2289

Affected:

2290

self.shape_spc_Buffer

2290

self.shape_spc_Buffer

2291

self.shape_cspc_Buffer

2291

self.shape_cspc_Buffer

2292

self.shape_dc_Buffer

2292

self.shape_dc_Buffer

2293

2294

Return: None

2294

Return: None

2295

"""

2295

"""

2296

self.shape_spc_Buffer = (self.dataOut.nChannels,

2296

self.shape_spc_Buffer = (self.dataOut.nChannels,

2297

self.processingHeaderObj.nHeights,

2297

self.processingHeaderObj.nHeights,

2298

self.processingHeaderObj.profilesPerBlock)

2298

self.processingHeaderObj.profilesPerBlock)

2299

2300

self.shape_cspc_Buffer = (self.dataOut.nPairs,

2300

self.shape_cspc_Buffer = (self.dataOut.nPairs,

2301

self.processingHeaderObj.nHeights,

2301

self.processingHeaderObj.nHeights,

2302

self.processingHeaderObj.profilesPerBlock)

2302

self.processingHeaderObj.profilesPerBlock)

2303

2304

self.shape_dc_Buffer = (self.dataOut.nChannels,

2304

self.shape_dc_Buffer = (self.dataOut.nChannels,

2305

self.processingHeaderObj.nHeights)

2305

self.processingHeaderObj.nHeights)

2306

2307

2308

def writeBlock(self):

2308

def writeBlock(self):

2309

"""

2309

"""

2310

Escribe el buffer en el file designado

2310

Escribe el buffer en el file designado

2311

2312

Affected:

2312

Affected:

2313

self.data_spc

2313

self.data_spc

2314

self.data_cspc

2314

self.data_cspc

2315

self.data_dc

2315

self.data_dc

2316

self.flagIsNewFile

2316

self.flagIsNewFile

2317

self.flagIsNewBlock

2317

self.flagIsNewBlock

2318

self.nTotalBlocks

2318

self.nTotalBlocks

2319

self.nWriteBlocks

2319

self.nWriteBlocks

2320

2321

Return: None

2321

Return: None

2322

"""

2322

"""

2323

2324

spc = numpy.transpose( self.data_spc, (0,2,1) )

2324

spc = numpy.transpose( self.data_spc, (0,2,1) )

2325

if not( self.processingHeaderObj.shif_fft ):

2325

if not( self.processingHeaderObj.shif_fft ):

2326

spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2326

spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2327

data = spc.reshape((-1))

2327

data = spc.reshape((-1))

2328

data = data.astype(self.dtype[0])

2328

data = data.astype(self.dtype[0])

2329

data.tofile(self.fp)

2329

data.tofile(self.fp)

2330

2331

if self.data_cspc != None:

2331

if self.data_cspc != None:

2332

data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )

2332

data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )

2333

cspc = numpy.transpose( self.data_cspc, (0,2,1) )

2333

cspc = numpy.transpose( self.data_cspc, (0,2,1) )

2334

if not( self.processingHeaderObj.shif_fft ):

2334

if not( self.processingHeaderObj.shif_fft ):

2335

cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2335

cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2336

data['real'] = cspc.real

2336

data['real'] = cspc.real

2337

data['imag'] = cspc.imag

2337

data['imag'] = cspc.imag

2338

data = data.reshape((-1))

2338

data = data.reshape((-1))

2339

data.tofile(self.fp)

2339

data.tofile(self.fp)

2340

2341

if self.data_dc != None:

2341

if self.data_dc != None:

2342

data = numpy.zeros( self.shape_dc_Buffer, self.dtype )

2342

data = numpy.zeros( self.shape_dc_Buffer, self.dtype )

2343

dc = self.data_dc

2343

dc = self.data_dc

2344

data['real'] = dc.real

2344

data['real'] = dc.real

2345

data['imag'] = dc.imag

2345

data['imag'] = dc.imag

2346

data = data.reshape((-1))

2346

data = data.reshape((-1))

2347

data.tofile(self.fp)

2347

data.tofile(self.fp)

2348

2349

self.data_spc.fill(0)

2349

self.data_spc.fill(0)

2350

self.data_dc.fill(0)

2350

self.data_dc.fill(0)

2351

if self.data_cspc != None:

2351

if self.data_cspc != None:

2352

self.data_cspc.fill(0)

2352

self.data_cspc.fill(0)

2353

2354

self.flagIsNewFile = 0

2354

self.flagIsNewFile = 0

2355

self.flagIsNewBlock = 1

2355

self.flagIsNewBlock = 1

2356

self.nTotalBlocks += 1

2356

self.nTotalBlocks += 1

2357

self.nWriteBlocks += 1

2357

self.nWriteBlocks += 1

2358

self.blockIndex += 1

2358

self.blockIndex += 1

2359

2360

2361

def putData(self):

2361

def putData(self):

2362

"""

2362

"""

2363

Setea un bloque de datos y luego los escribe en un file

2363

Setea un bloque de datos y luego los escribe en un file

2364

2365

Affected:

2365

Affected:

2366

self.data_spc

2366

self.data_spc

2367

self.data_cspc

2367

self.data_cspc

2368

self.data_dc

2368

self.data_dc

2369

2370

Return:

2370

Return:

2371

0 : Si no hay data o no hay mas files que puedan escribirse

2371

0 : Si no hay data o no hay mas files que puedan escribirse

2372

1 : Si se escribio la data de un bloque en un file

2372

1 : Si se escribio la data de un bloque en un file

2373

"""

2373

"""

2374

2375

if self.dataOut.flagNoData:

2375

if self.dataOut.flagNoData:

2376

return 0

2376

return 0

2377

2378

self.flagIsNewBlock = 0

2378

self.flagIsNewBlock = 0

2379

2380

if self.dataOut.flagTimeBlock:

2380

if self.dataOut.flagTimeBlock:

2381

self.data_spc.fill(0)

2381

self.data_spc.fill(0)

2382

self.data_cspc.fill(0)

2382

self.data_cspc.fill(0)

2383

self.data_dc.fill(0)

2383

self.data_dc.fill(0)

2384

self.setNextFile()

2384

self.setNextFile()

2385

2386

if self.flagIsNewFile == 0:

2386

if self.flagIsNewFile == 0:

2387

self.setBasicHeader()

2387

self.setBasicHeader()

2388

2389

self.data_spc = self.dataOut.data_spc.copy()

2389

self.data_spc = self.dataOut.data_spc.copy()

2390

self.data_cspc = self.dataOut.data_cspc.copy()

2390

self.data_cspc = self.dataOut.data_cspc.copy()

2391

self.data_dc = self.dataOut.data_dc.copy()

2391

self.data_dc = self.dataOut.data_dc.copy()

2392

2393

# #self.processingHeaderObj.dataBlocksPerFile)

2393

# #self.processingHeaderObj.dataBlocksPerFile)

2394

if self.hasAllDataInBuffer():

2394

if self.hasAllDataInBuffer():

2395

# self.setFirstHeader()

2395

# self.setFirstHeader()

2396

self.writeNextBlock()

2396

self.writeNextBlock()

2397

2398

return 1

2398

return 1

2399

2400

2401

def __getProcessFlags(self):

2401

def __getProcessFlags(self):

2402

2403

processFlags = 0

2403

processFlags = 0

2404

2405

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2405

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2406

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2406

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2407

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2407

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2408

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2408

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2409

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2409

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2410

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2410

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2411

2412

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2412

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2413

2414

2415

2416

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

2416

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

2417

PROCFLAG.DATATYPE_SHORT,

2417

PROCFLAG.DATATYPE_SHORT,

2418

PROCFLAG.DATATYPE_LONG,

2418

PROCFLAG.DATATYPE_LONG,

2419

PROCFLAG.DATATYPE_INT64,

2419

PROCFLAG.DATATYPE_INT64,

2420

PROCFLAG.DATATYPE_FLOAT,

2420

PROCFLAG.DATATYPE_FLOAT,

2421

PROCFLAG.DATATYPE_DOUBLE]

2421

PROCFLAG.DATATYPE_DOUBLE]

2422

2423

2424

for index in range(len(dtypeList)):

2424

for index in range(len(dtypeList)):

2425

if self.dataOut.dtype == dtypeList[index]:

2425

if self.dataOut.dtype == dtypeList[index]:

2426

dtypeValue = datatypeValueList[index]

2426

dtypeValue = datatypeValueList[index]

2427

break

2427

break

2428

2429

processFlags += dtypeValue

2429

processFlags += dtypeValue

2430

2431

if self.dataOut.flagDecodeData:

2431

if self.dataOut.flagDecodeData:

2432

processFlags += PROCFLAG.DECODE_DATA

2432

processFlags += PROCFLAG.DECODE_DATA

2433

2434

if self.dataOut.flagDeflipData:

2434

if self.dataOut.flagDeflipData:

2435

processFlags += PROCFLAG.DEFLIP_DATA

2435

processFlags += PROCFLAG.DEFLIP_DATA

2436

2437

if self.dataOut.code != None:

2437

if self.dataOut.code != None:

2438

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

2438

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

2439

2440

if self.dataOut.nIncohInt > 1:

2440

if self.dataOut.nIncohInt > 1:

2441

processFlags += PROCFLAG.INCOHERENT_INTEGRATION

2441

processFlags += PROCFLAG.INCOHERENT_INTEGRATION

2442

2443

if self.dataOut.data_dc != None:

2443

if self.dataOut.data_dc != None:

2444

processFlags += PROCFLAG.SAVE_CHANNELS_DC

2444

processFlags += PROCFLAG.SAVE_CHANNELS_DC

2445

2446

return processFlags

2446

return processFlags

2447

2448

2449

def __getBlockSize(self):

2449

def __getBlockSize(self):

2450

'''

2450

'''

2451

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra

2451

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra

2452

'''

2452

'''

2453

2454

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2454

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2455

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2455

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2456

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2456

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2457

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2457

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2458

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2458

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2459

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2459

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2460

2461

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2461

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2462

datatypeValueList = [1,2,4,8,4,8]

2462

datatypeValueList = [1,2,4,8,4,8]

2463

for index in range(len(dtypeList)):

2463

for index in range(len(dtypeList)):

2464

if self.dataOut.dtype == dtypeList[index]:

2464

if self.dataOut.dtype == dtypeList[index]:

2465

datatypeValue = datatypeValueList[index]

2465

datatypeValue = datatypeValueList[index]

2466

break

2466

break

2467

2468

2469

pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints

2469

pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints

2470

2471

pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)

2471

pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)

2472

blocksize = (pts2write_SelfSpectra*datatypeValue)

2472

blocksize = (pts2write_SelfSpectra*datatypeValue)

2473

2474

if self.dataOut.data_cspc != None:

2474

if self.dataOut.data_cspc != None:

2475

pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)

2475

pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)

2476

blocksize += (pts2write_CrossSpectra*datatypeValue*2)

2476

blocksize += (pts2write_CrossSpectra*datatypeValue*2)

2477

2478

if self.dataOut.data_dc != None:

2478

if self.dataOut.data_dc != None:

2479

pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)

2479

pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)

2480

blocksize += (pts2write_DCchannels*datatypeValue*2)

2480

blocksize += (pts2write_DCchannels*datatypeValue*2)

2481

2482

blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO

2482

blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO

2483

2484

return blocksize

2484

return blocksize

2485

2486

def setFirstHeader(self):

2486

def setFirstHeader(self):

2487

2488

"""

2488

"""

2489

Obtiene una copia del First Header

2489

Obtiene una copia del First Header

2490

2491

Affected:

2491

Affected:

2492

self.systemHeaderObj

2492

self.systemHeaderObj

2493

self.radarControllerHeaderObj

2493

self.radarControllerHeaderObj

2494

self.dtype

2494

self.dtype

2495

2496

Return:

2496

Return:

2497

None

2497

None

2498

"""

2498

"""

2499

2500

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

2500

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

2501

self.systemHeaderObj.nChannels = self.dataOut.nChannels

2501

self.systemHeaderObj.nChannels = self.dataOut.nChannels

2502

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

2502

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

2503

2504

self.setBasicHeader()

2504

self.setBasicHeader()

2505

2506

processingHeaderSize = 40 # bytes

2506

processingHeaderSize = 40 # bytes

2507

self.processingHeaderObj.dtype = 1 # Spectra

2507

self.processingHeaderObj.dtype = 1 # Spectra

2508

self.processingHeaderObj.blockSize = self.__getBlockSize()

2508

self.processingHeaderObj.blockSize = self.__getBlockSize()

2509

self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints

2509

self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints

2510

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

2510

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

2511

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

2511

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

2512

self.processingHeaderObj.processFlags = self.__getProcessFlags()

2512

self.processingHeaderObj.processFlags = self.__getProcessFlags()

2513

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval

2513

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval

2514

self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt

2514

self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt

2515

self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels

2515

self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels

2516

self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT

2516

self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT

2517

2518

if self.processingHeaderObj.totalSpectra > 0:

2518

if self.processingHeaderObj.totalSpectra > 0:

2519

channelList = []

2519

channelList = []

2520

for channel in range(self.dataOut.nChannels):

2520

for channel in range(self.dataOut.nChannels):

2521

channelList.append(channel)

2521

channelList.append(channel)

2522

channelList.append(channel)

2522

channelList.append(channel)

2523

2524

pairsList = []

2524

pairsList = []

2525

for pair in self.dataOut.pairsList:

2525

for pair in self.dataOut.pairsList:

2526

pairsList.append(pair[0])

2526

pairsList.append(pair[0])

2527

pairsList.append(pair[1])

2527

pairsList.append(pair[1])

2528

spectraComb = channelList + pairsList

2528

spectraComb = channelList + pairsList

2529

spectraComb = numpy.array(spectraComb,dtype="u1")

2529

spectraComb = numpy.array(spectraComb,dtype="u1")

2530

self.processingHeaderObj.spectraComb = spectraComb

2530

self.processingHeaderObj.spectraComb = spectraComb

2531

sizeOfSpcComb = len(spectraComb)

2531

sizeOfSpcComb = len(spectraComb)

2532

processingHeaderSize += sizeOfSpcComb

2532

processingHeaderSize += sizeOfSpcComb

2533

2534

# The processing header should not have information about code

2534

# The processing header should not have information about code

2535

# if self.dataOut.code != None:

2535

# if self.dataOut.code != None:

2536

# self.processingHeaderObj.code = self.dataOut.code

2536

# self.processingHeaderObj.code = self.dataOut.code

2537

# self.processingHeaderObj.nCode = self.dataOut.nCode

2537

# self.processingHeaderObj.nCode = self.dataOut.nCode

2538

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

2538

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

2539

# nCodeSize = 4 # bytes

2539

# nCodeSize = 4 # bytes

2540

# nBaudSize = 4 # bytes

2540

# nBaudSize = 4 # bytes

2541

# codeSize = 4 # bytes

2541

# codeSize = 4 # bytes

2542

# sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)

2542

# sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)

2543

# processingHeaderSize += sizeOfCode

2543

# processingHeaderSize += sizeOfCode

2544

2545

if self.processingHeaderObj.nWindows != 0:

2545

if self.processingHeaderObj.nWindows != 0:

2546

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

2546

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

2547

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2547

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2548

self.processingHeaderObj.nHeights = self.dataOut.nHeights

2548

self.processingHeaderObj.nHeights = self.dataOut.nHeights

2549

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

2549

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

2550

sizeOfFirstHeight = 4

2550

sizeOfFirstHeight = 4

2551

sizeOfdeltaHeight = 4

2551

sizeOfdeltaHeight = 4

2552

sizeOfnHeights = 4

2552

sizeOfnHeights = 4

2553

sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows

2553

sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows

2554

processingHeaderSize += sizeOfWindows

2554

processingHeaderSize += sizeOfWindows

2555

2556

self.processingHeaderObj.size = processingHeaderSize

2556

self.processingHeaderObj.size = processingHeaderSize

2557

2558

class SpectraHeisWriter(Operation):

2558

class SpectraHeisWriter(Operation):

2559

# set = None

2559

# set = None

2560

setFile = None

2560

setFile = None

2561

idblock = None

2561

idblock = None

2562

doypath = None

2562

doypath = None

2563

subfolder = None

2563

subfolder = None

2564

2565

def __init__(self):

2565

def __init__(self):

2566

self.wrObj = FITS()

2566

self.wrObj = FITS()

2567

# self.dataOut = dataOut

2567

# self.dataOut = dataOut

2568

self.nTotalBlocks=0

2568

self.nTotalBlocks=0

2569

# self.set = None

2569

# self.set = None

2570

self.setFile = None

2570

self.setFile = None

2571

self.idblock = 0

2571

self.idblock = 0

2572

self.wrpath = None

2572

self.wrpath = None

2573

self.doypath = None

2573

self.doypath = None

2574

self.subfolder = None

2574

self.subfolder = None

2575

self.isConfig = False

2575

self.isConfig = False

2576

2577

def isNumber(str):

2577

def isNumber(str):

2578

"""

2578

"""

2579

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

2579

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

2580

2581

Excepciones:

2581

Excepciones:

2582

Si un determinado string no puede ser convertido a numero

2582

Si un determinado string no puede ser convertido a numero

2583

Input:

2583

Input:

2584

str, string al cual se le analiza para determinar si convertible a un numero o no

2584

str, string al cual se le analiza para determinar si convertible a un numero o no

2585

2586

Return:

2586

Return:

2587

True : si el string es uno numerico

2587

True : si el string es uno numerico

2588

False : no es un string numerico

2588

False : no es un string numerico

2589

"""

2589

"""

2590

try:

2590

try:

2591

float( str )

2591

float( str )

2592

return True

2592

return True

2593

except:

2593

except:

2594

return False

2594

return False

2595

2596

def setup(self, dataOut, wrpath):

2596

def setup(self, dataOut, wrpath):

2597

2598

if not(os.path.exists(wrpath)):

2598

if not(os.path.exists(wrpath)):

2599

os.mkdir(wrpath)

2599

os.mkdir(wrpath)

2600

2601

self.wrpath = wrpath

2601

self.wrpath = wrpath

2602

# self.setFile = 0

2602

# self.setFile = 0

2603

self.dataOut = dataOut

2603

self.dataOut = dataOut

2604

2605

def putData(self):

2605

def putData(self):

2606

name= time.localtime( self.dataOut.utctime)

2606

name= time.localtime( self.dataOut.utctime)

2607

ext=".fits"

2607

ext=".fits"

2608

2609

if self.doypath == None:

2609

if self.doypath == None:

2610

self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))

2610

self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))

2611

self.doypath = os.path.join( self.wrpath, self.subfolder )

2611

self.doypath = os.path.join( self.wrpath, self.subfolder )

2612

os.mkdir(self.doypath)

2612

os.mkdir(self.doypath)

2613

2614

if self.setFile == None:

2614

if self.setFile == None:

2615

# self.set = self.dataOut.set

2615

# self.set = self.dataOut.set

2616

self.setFile = 0

2616

self.setFile = 0

2617

# if self.set != self.dataOut.set:

2617

# if self.set != self.dataOut.set:

2618

## self.set = self.dataOut.set

2618

## self.set = self.dataOut.set

2619

# self.setFile = 0

2619

# self.setFile = 0

2620

2621

#make the filename

2621

#make the filename

2622

file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)

2622

file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)

2623

2624

filename = os.path.join(self.wrpath,self.subfolder, file)

2624

filename = os.path.join(self.wrpath,self.subfolder, file)

2625

2626

idblock = numpy.array([self.idblock],dtype="int64")

2626

idblock = numpy.array([self.idblock],dtype="int64")

2627

header=self.wrObj.cFImage(idblock=idblock,

2627

header=self.wrObj.cFImage(idblock=idblock,

2628

year=time.gmtime(self.dataOut.utctime).tm_year,

2628

year=time.gmtime(self.dataOut.utctime).tm_year,

2629

month=time.gmtime(self.dataOut.utctime).tm_mon,

2629

month=time.gmtime(self.dataOut.utctime).tm_mon,

2630

day=time.gmtime(self.dataOut.utctime).tm_mday,

2630

day=time.gmtime(self.dataOut.utctime).tm_mday,

2631

hour=time.gmtime(self.dataOut.utctime).tm_hour,

2631

hour=time.gmtime(self.dataOut.utctime).tm_hour,

2632

minute=time.gmtime(self.dataOut.utctime).tm_min,

2632

minute=time.gmtime(self.dataOut.utctime).tm_min,

2633

second=time.gmtime(self.dataOut.utctime).tm_sec)

2633

second=time.gmtime(self.dataOut.utctime).tm_sec)

2634

2635

c=3E8

2635

c=3E8

2636

deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2636

deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2637

freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))

2637

freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))

2638

2639

colList = []

2639

colList = []

2640

2641

colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)

2641

colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)

2642

2643

colList.append(colFreq)

2643

colList.append(colFreq)

2644

2645

nchannel=self.dataOut.nChannels

2645

nchannel=self.dataOut.nChannels

2646

2647

for i in range(nchannel):

2647

for i in range(nchannel):

2648

col = self.wrObj.writeData(name="PCh"+str(i+1),

2648

col = self.wrObj.writeData(name="PCh"+str(i+1),

2649

format=str(self.dataOut.nFFTPoints)+'E',

2649

format=str(self.dataOut.nFFTPoints)+'E',

2650

data=10*numpy.log10(self.dataOut.data_spc[i,:]))

2650

data=10*numpy.log10(self.dataOut.data_spc[i,:]))

2651

2652

colList.append(col)

2652

colList.append(col)

2653

2654

data=self.wrObj.Ctable(colList=colList)

2654

data=self.wrObj.Ctable(colList=colList)

2655

2656

self.wrObj.CFile(header,data)

2656

self.wrObj.CFile(header,data)

2657

2658

self.wrObj.wFile(filename)

2658

self.wrObj.wFile(filename)

2659

2660

#update the setFile

2660

#update the setFile

2661

self.setFile += 1

2661

self.setFile += 1

2662

self.idblock += 1

2662

self.idblock += 1

2663

2664

return 1

2664

return 1

2665

2666

def run(self, dataOut, **kwargs):

2666

def run(self, dataOut, **kwargs):

2667

2668

if not(self.isConfig):

2668

if not(self.isConfig):

2669

2670

self.setup(dataOut, **kwargs)

2670

self.setup(dataOut, **kwargs)

2671

self.isConfig = True

2671

self.isConfig = True

2672

2673

self.putData()

2673

self.putData()

2674

2675

2676

class FITS:

2676

class FITS:

2677

name=None

2677

name=None

2678

format=None

2678

format=None

2679

array =None

2679

array =None

2680

data =None

2680

data =None

2681

thdulist=None

2681

thdulist=None

2682

prihdr=None

2682

prihdr=None

2683

hdu=None

2683

hdu=None

2684

2685

def __init__(self):

2685

def __init__(self):

2686

2687

pass

2687

pass

2688

2689

def setColF(self,name,format,array):

2689

def setColF(self,name,format,array):

2690

self.name=name

2690

self.name=name

2691

self.format=format

2691

self.format=format

2692

self.array=array

2692

self.array=array

2693

a1=numpy.array([self.array],dtype=numpy.float32)

2693

a1=numpy.array([self.array],dtype=numpy.float32)

2694

self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)

2694

self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)

2695

return self.col1

2695

return self.col1

2696

2697

# def setColP(self,name,format,data):

2697

# def setColP(self,name,format,data):

2698

# self.name=name

2698

# self.name=name

2699

# self.format=format

2699

# self.format=format

2700

# self.data=data

2700

# self.data=data

2701

# a2=numpy.array([self.data],dtype=numpy.float32)

2701

# a2=numpy.array([self.data],dtype=numpy.float32)

2702

# self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)

2702

# self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)

2703

# return self.col2

2703

# return self.col2

2704

2705

2706

def writeData(self,name,format,data):

2706

def writeData(self,name,format,data):

2707

self.name=name

2707

self.name=name

2708

self.format=format

2708

self.format=format

2709

self.data=data

2709

self.data=data

2710

a2=numpy.array([self.data],dtype=numpy.float32)

2710

a2=numpy.array([self.data],dtype=numpy.float32)

2711

self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)

2711

self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)

2712

return self.col2

2712

return self.col2

2713

2714

def cFImage(self,idblock,year,month,day,hour,minute,second):

2714

def cFImage(self,idblock,year,month,day,hour,minute,second):

2715

self.hdu= pyfits.PrimaryHDU(idblock)

2715

self.hdu= pyfits.PrimaryHDU(idblock)

2716

self.hdu.header.set("Year",year)

2716

self.hdu.header.set("Year",year)

2717

self.hdu.header.set("Month",month)

2717

self.hdu.header.set("Month",month)

2718

self.hdu.header.set("Day",day)

2718

self.hdu.header.set("Day",day)

2719

self.hdu.header.set("Hour",hour)

2719

self.hdu.header.set("Hour",hour)

2720

self.hdu.header.set("Minute",minute)

2720

self.hdu.header.set("Minute",minute)

2721

self.hdu.header.set("Second",second)

2721

self.hdu.header.set("Second",second)

2722

return self.hdu

2722

return self.hdu

2723

2724

2725

def Ctable(self,colList):

2725

def Ctable(self,colList):

2726

self.cols=pyfits.ColDefs(colList)

2726

self.cols=pyfits.ColDefs(colList)

2727

self.tbhdu = pyfits.new_table(self.cols)

2727

self.tbhdu = pyfits.new_table(self.cols)

2728

return self.tbhdu

2728

return self.tbhdu

2729

2730

2731

def CFile(self,hdu,tbhdu):

2731

def CFile(self,hdu,tbhdu):

2732

self.thdulist=pyfits.HDUList([hdu,tbhdu])

2732

self.thdulist=pyfits.HDUList([hdu,tbhdu])

2733

2734

def wFile(self,filename):

2734

def wFile(self,filename):

2735

if os.path.isfile(filename):

2735

if os.path.isfile(filename):

2736

os.remove(filename)

2736

os.remove(filename)

2737

self.thdulist.writeto(filename)

2737

self.thdulist.writeto(filename)

2738

2739

2740

class ParameterConf:

2741

ELEMENTNAME = 'Parameter'

2742

def __init__(self):

2743

self.name = ''

2744

self.value = ''

2745

2746

def readXml(self, parmElement):

2747

self.name = parmElement.get('name')

2748

self.value = parmElement.get('value')

2749

2750

def getElementName(self):

2751

return self.ELEMENTNAME

2752

2753

class Metadata:

2754

2755

def __init__(self, filename):

2756

self.parmConfObjList = []

2757

self.readXml(filename)

2758

2759

def readXml(self, filename):

2760

self.projectElement = None

2761

self.procUnitConfObjDict = {}

2762

self.projectElement = ElementTree().parse(filename)

2763

self.project = self.projectElement.tag

2764

2765

parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())

2766

2767

for parmElement in parmElementList:

2768

parmConfObj = ParameterConf()

2769

parmConfObj.readXml(parmElement)

2770

self.parmConfObjList.append(parmConfObj)

2771

2772

class FitsWriter(Operation):

2773

2774

def __init__(self):

2775

self.isConfig = False

2776

self.dataBlocksPerFile = None

2777

self.blockIndex = 0

2778

self.flagIsNewFile = 1

2779

self.fitsObj = None

2780

self.optchar = 'P'

2781

self.ext = '.fits'

2782

self.setFile = 0

2783

2784

def setFitsHeader(self, dataOut, metadatafile):

2785

2786

header_data = pyfits.PrimaryHDU()

2787

2788

metadata4fits = Metadata(metadatafile)

2789

for parameter in metadata4fits.parmConfObjList:

2790

parm_name = parameter.name

2791

parm_value = parameter.value

2792

2793

if parm_value == 'fromdatadatetime':

2794

value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())

2795

elif parm_value == 'fromdataheights':

2796

value = dataOut.nHeights

2797

elif parm_value == 'fromdatachannel':

2798

value = dataOut.nChannels

2799

elif parm_value == 'fromdatasamples':

2800

value = dataOut.nFFTPoints

2801

else:

2802

value = parm_value

2803

2804

header_data.header[parm_name] = value

2805

2806

header_data.header['NBLOCK'] = self.blockIndex

2807

2808

header_data.writeto(self.filename)

2809

2810

2811

def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):

2812

2813

self.path = path

2814

self.dataOut = dataOut

2815

self.metadatafile = metadatafile

2816

self.dataBlocksPerFile = dataBlocksPerFile

2817

2818

def open(self):

2819

self.fitsObj = pyfits.open(self.filename, mode='update')

2820

2821

2822

def addData(self, data):

2823

self.open()

2824

extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATA'])

2825

extension.header['UTCTIME'] = self.dataOut.utctime

2826

self.fitsObj.append(extension)

2827

self.blockIndex += 1

2828

self.fitsObj[0].header['NBLOCK'] = self.blockIndex

2829

2830

self.write()

2831

2832

def write(self):

2833

2834

self.fitsObj.flush(verbose=True)

2835

self.fitsObj.close()

2836

2837

2838

def setNextFile(self):

2839

2840

ext = self.ext

2841

path = self.path

2842

2843

timeTuple = time.localtime( self.dataOut.utctime)

2844

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

2845

2846

fullpath = os.path.join( path, subfolder )

2847

if not( os.path.exists(fullpath) ):

2848

os.mkdir(fullpath)

2849

self.setFile = -1 #inicializo mi contador de seteo

2850

else:

2851

filesList = os.listdir( fullpath )

2852

if len( filesList ) > 0:

2853

filesList = sorted( filesList, key=str.lower )

2854

filen = filesList[-1]

2855

2856

if isNumber( filen[8:11] ):

2857

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

2858

else:

2859

self.setFile = -1

2860

else:

2861

self.setFile = -1 #inicializo mi contador de seteo

2862

2863

setFile = self.setFile

2864

setFile += 1

2865

2866

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

2867

timeTuple.tm_year,

2868

timeTuple.tm_yday,

2869

setFile,

2870

ext )

2871

2872

filename = os.path.join( path, subfolder, file )

2873

2874

self.blockIndex = 0

2875

self.filename = filename

2876

self.setFile = setFile

2877

self.flagIsNewFile = 1

2878

2879

print 'Writing the file: %s'%self.filename

2880

2881

self.setFitsHeader(self.dataOut, self.metadatafile)

2882

2883

return 1

2884

2885

def writeBlock(self):

2886

self.addData(self.dataOut.data_spc)

2887

self.flagIsNewFile = 0

2888

2889

2890

def __setNewBlock(self):

2891

2892

if self.flagIsNewFile:

2893

return 1

2894

2895

if self.blockIndex < self.dataBlocksPerFile:

2896

return 1

2897

2898

if not( self.setNextFile() ):

2899

return 0

2900

2901

return 1

2902

2903

def writeNextBlock(self):

2904

if not( self.__setNewBlock() ):

2905

return 0

2906

self.writeBlock()

2907

return 1

2908

2909

def putData(self):

2910

if self.flagIsNewFile:

2911

self.setNextFile()

2912

self.writeNextBlock()

2913

2914

def run(self, dataOut, **kwargs):

2915

if not(self.isConfig):

2916

self.setup(dataOut, **kwargs)

2917

self.isConfig = True

2918

self.putData()

2919

2920

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages

             import os, sys
             path = os.path.split(os.getcwd())[0]
             sys.path.append(path)
             from controller import *
             desc = "Sun Experiment Test"
             filename = "sunexp.xml"
             controllerObj = Project()
             controllerObj.setup(id = '191', name='test01', description=desc)
             #/Users/dsuarez/Documents/RadarData/SunExperiment
             #/Volumes/data_e/PaseDelSol/Raw/100KHZ
             readUnitConfObj = controllerObj.addReadUnit(datatype='Voltage',
                                                         path='/Users/dsuarez/Documents/RadarData/SunExperiment',
                                                         startDate='2013/02/06',
                                                         endDate='2013/12/31',
                                                         startTime='00:30:00',
                                                         endTime='17:40:59',
                                                         online=0,
                                                         delay=3,
                                                         walk=1)
             procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId())
             procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraHeis', inputId=procUnitConfObj0.getId())
             opObj11 = procUnitConfObj1.addOperation(name='IncohInt4SpectraHeis', optype='other')
             opObj11.addParameter(name='timeInterval', value='5', format='float')
             opObj11 = procUnitConfObj1.addOperation(name='SpectraHeisScope', optype='other')
             opObj11.addParameter(name='idfigure', value='10', format='int')
             opObj11.addParameter(name='wintitle', value='SpectraHeisPlot', format='str')
             #opObj11.addParameter(name='ymin', value='125', format='int')
             #opObj11.addParameter(name='ymax', value='140', format='int')
             #opObj11.addParameter(name='channelList', value='0,1,2', format='intlist')
             #opObj11.addParameter(name='showprofile', value='1', format='int')
-            opObj11.addParameter(name='save', value='1', format='bool')
+            #opObj11.addParameter(name='save', value='1', format='bool')
-            opObj11.addParameter(name='figfile', value='spc-noise.png', format='str')
+            #opObj11.addParameter(name='figfile', value='spc-noise.png', format='str')
-            opObj11.addParameter(name='figpath', value='/Users/dsuarez/Pictures/sun_pics', format='str')
+            #opObj11.addParameter(name='figpath', value='/Users/dsuarez/Pictures/sun_pics', format='str')
-            opObj11.addParameter(name='ftp', value='1', format='int')
+            #opObj11.addParameter(name='ftp', value='1', format='int')
-            opObj11.addParameter(name='ftpratio', value='10', format='int')
+            #opObj11.addParameter(name='ftpratio', value='10', format='int')
             opObj11 = procUnitConfObj1.addOperation(name='RTIfromSpectraHeis', optype='other')
             opObj11.addParameter(name='idfigure', value='6', format='int')
             opObj11.addParameter(name='wintitle', value='RTIPLot', format='str')
-            #opObj11.addParameter(name='zmin', value='10', format='int')
+            opObj11.addParameter(name='xmin', value='11.5', format='float')
-            #opObj11.addParameter(name='zmax', value='40', format='int')
+            opObj11.addParameter(name='xmax', value='12.5', format='float')
             opObj11.addParameter(name='ymin', value='60', format='int')
             opObj11.addParameter(name='ymax', value='85', format='int')
             #opObj11.addParameter(name='channelList', value='0,1,2,3', format='intlist')
             #opObj11.addParameter(name='timerange', value='600', format='int')
             #opObj11.addParameter(name='showprofile', value='0', format='int')
-            opObj11.addParameter(name='save', value='1', format='bool')
+            #opObj11.addParameter(name='save', value='1', format='bool')
-            opObj11.addParameter(name='figfile', value='rti-noise.png', format='str')
+            #opObj11.addParameter(name='figfile', value='rti-noise.png', format='str')
-            opObj11.addParameter(name='figpath', value='/Users/dsuarez/Pictures/sun_pics', format='str')
+            #opObj11.addParameter(name='figpath', value='/Users/dsuarez/Pictures/sun_pics', format='str')
-            opObj11.addParameter(name='ftp', value='1', format='int')
+            #opObj11.addParameter(name='ftp', value='1', format='int')
-            opObj11.addParameter(name='ftpratio', value='10', format='int')
+            #opObj11.addParameter(name='ftpratio', value='10', format='int')
+            opObj11.addParameter(name='useLocalTime', value='1', format='bool')
+            opObj11.addParameter(name='timezone', value='300', format='int')
-            opObj11 = procUnitConfObj1.addOperation(name='SpectraHeisWriter', optype='other')
+            #opObj11 = procUnitConfObj1.addOperation(name='SpectraHeisWriter', optype='other')
-            opObj11.addParameter(name='wrpath', value='/Users/dsuarez/Remote', format='str')
+            #opObj11.addParameter(name='wrpath', value='/Users/dsuarez/Remote', format='str')
-            #opObj11.addParameter(name='blocksPerFile', value='200', format='int')
+            ##opObj11.addParameter(name='blocksPerFile', value='200', format='int')
+            opObj11 = procUnitConfObj1.addOperation(name='FitsWriter', optype='other')
+            opObj11.addParameter(name='path', value='/Users/dsuarez/Remote', format='str')
+            opObj11.addParameter(name='dataBlocksPerFile', value='10', format='int')
+            opObj11.addParameter(name='metadatafile', value='/Users/dsuarez/Desktop/metadata_fits.xml', format='str')
             print "Escribiendo el archivo XML"
             controllerObj.writeXml(filename)
             print "Leyendo el archivo XML"
             controllerObj.readXml(filename)
             controllerObj.createObjects()
             controllerObj.connectObjects()
             controllerObj.run()
   No newline at end of file