schain Commit - r352:c7267cf690c5 · Jicamarca Repository

             '''
             $Author: murco $
             $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
             '''
             import os, sys
             import glob
             import time
             import numpy
             import fnmatch
             import time, datetime
+            from xml.etree.ElementTree import Element, SubElement, ElementTree
             try:
                 import pyfits
             except:
                 print "pyfits module has not been imported, it should be installed to save files in fits format"
             from jrodata import *
             from jroheaderIO import *
             from jroprocessing import *
             LOCALTIME = True #-18000
             def isNumber(str):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float( str )
                     return True
                 except:
                     return False
             def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename,'rb')
                 except:
                     raise IOError, "The file %s can't be opened" %(filename)
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isFileinThisTime(filename, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename,'rb')
                 except:
                     raise IOError, "The file %s can't be opened" %(filename)
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = basicHeaderObj.datatime
                 thisTime = basicHeaderObj.datatime.time()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return None
                 if not ((startTime <= thisTime) and (endTime > thisTime)):
                     return None
                 return thisDatetime
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for file in fileList:
                     try:
                         year = int(file[1:5])
                         doy  = int(file[5:8])
                     except:
                         continue
                     if (os.path.splitext(file)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(file)
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def checkForRealPath(path, foldercounter, year, doy, set, ext):
                 """
                 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
                 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
                 el path exacto de un determinado file.
                 Example    :
                     nombre correcto del file es  .../.../D2009307/P2009307367.ext
                     Entonces la funcion prueba con las siguientes combinaciones
                         .../.../y2009307367.ext
                         .../.../Y2009307367.ext
                         .../.../x2009307/y2009307367.ext
                         .../.../x2009307/Y2009307367.ext
                         .../.../X2009307/y2009307367.ext
                         .../.../X2009307/Y2009307367.ext
                     siendo para este caso, la ultima combinacion de letras, identica al file buscado
                 Return:
                     Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
                     caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
                     para el filename
                 """
                 fullfilename = None
                 find_flag = False
                 filename = None
                 prefixDirList = [None,'d','D']
                 if ext.lower() == ".r": #voltage
                     prefixFileList = ['d','D']
                 elif ext.lower() == ".pdata": #spectra
                     prefixFileList = ['p','P']
                 else:
                     return None, filename
                 #barrido por las combinaciones posibles
                 for prefixDir in prefixDirList:
                     thispath = path
                     if prefixDir != None:
                         #formo el nombre del directorio xYYYYDDD (x=d o x=D)
                         if foldercounter == 0:
                             thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
                         else:
                             thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
                     for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
                         filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
                         fullfilename = os.path.join( thispath, filename ) #formo el path completo
                         if os.path.exists( fullfilename ): #verifico que exista
                             find_flag = True
                             break
                     if find_flag:
                         break
                 if not(find_flag):
                     return None, filename
                 return fullfilename, filename
             def isDoyFolder(folder):
                 try:
                     year = int(folder[1:5])
                 except:
                     return 0
                 try:
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
             class JRODataIO:
                 c = 3E8
                 isConfig = False
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
                 processingHeaderObj = ProcessingHeader()
                 online = 0
                 dtype = None
                 pathList = []
                 filenameList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagTimeBlock = 0
                 flagIsNewBlock = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
                 ippSeconds = None
                 fileSizeByHeader = None
                 fileIndex = None
                 profileIndex = None
                 blockIndex = None
                 nTotalBlocks = None
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
                 blocksize = None
                 def __init__(self):
                     raise ValueError, "Not implemented"
                 def run(self):
                     raise ValueError, "Not implemented"
                 def getOutput(self):
                     return self.dataOut
             class JRODataReader(JRODataIO, ProcessingUnit):
                 nReadBlocks = 0
                 delay  = 10   #number of seconds waiting a new file
                 nTries  = 3  #quantity tries
                 nFiles = 3   #number of files for searching
                 path = None
                 foldercounter = 0
                 flagNoMoreFiles = 0
                 datetimeList = []
                 __isFirstTimeOnline = 1
                 __printInfo = True
                 profileIndex = None
                 def __init__(self):
                     """
                     """
                     raise ValueError, "This method has not been implemented"
                 def createObjByDefault(self):
                     """
                     """
                     raise ValueError, "This method has not been implemented"
                 def getBlockDimension(self):
                     raise ValueError, "No implemented"
                 def __searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.r',
                                         walk=True):
                     pathList = []
                     if not walk:
                         pathList.append(path)
                     else:
                         dirList = []
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isDoyFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None
                         thisDate = startDate
                         while(thisDate <= endDate):
                             year = thisDate.timetuple().tm_year
                             doy = thisDate.timetuple().tm_yday
                             matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
                             if len(matchlist) == 0:
                                 thisDate += datetime.timedelta(1)
                                 continue
                             for match in matchlist:
                                 pathList.append(os.path.join(path,match,expLabel))
                             thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
                         return None, None
                     print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
                     filenameList = []
                     datetimeList = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             thisDatetime = isFileinThisTime(filename, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "Any file was found for the time range %s - %s" %(startTime, endTime)
                         return None, None
                     print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True):
                     """
                     Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
                     devuelve el archivo encontrado ademas de otros datos.
                     Input:
                         path           :    carpeta donde estan contenidos los files que contiene data
                         expLabel        :     Nombre del subexperimento (subfolder)
                         ext              :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
                         directory   :    eL directorio donde esta el file encontrado
                         filename    :    el ultimo file de una determinada carpeta
                         year        :    el anho
                         doy         :    el numero de dia del anho
                         set         :    el set del archivo
                     """
                     dirList = []
                     if not walk:
                         fullpath = path
                     else:
                         #Filtra solo los directorios
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isDoyFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None, None, None, None
                         dirList = sorted( dirList, key=str.lower )
                         doypath = dirList[-1]
                         foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
                         fullpath = os.path.join(path, doypath, expLabel)
                     print "%s folder was found: " %(fullpath )
                     filename = getlastFileFromPath(fullpath, ext)
                     if not(filename):
                         return None, None, None, None, None
                     print "%s file was found" %(filename)
                     if not(self.__verifyFile(os.path.join(fullpath, filename))):
                         return None, None, None, None, None
                     year = int( filename[1:5] )
                     doy  = int( filename[5:8] )
                     set  = int( filename[8:11] )
                     return fullpath, foldercounter, filename, year, doy, set
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print "No more Files"
                             return 0
                         filename = self.filenameList[idFile]
                         if not(self.__verifyFile(filename)):
                             continue
                         fileSize = os.path.getsize(filename)
                         fp = open(filename,'rb')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fp = fp
                     print "Setting the file: %s"%self.filename
                     return 1
                 def __setNextFileOnline(self):
                     """
                     Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
                     no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
                     siguientes.
                     Affected:
                         self.flagIsNewFile
                         self.filename
                         self.fileSize
                         self.fp
                         self.set
                         self.flagNoMoreFiles
                     Return:
 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
 : si el file fue abierto con exito y esta listo a ser leido
                     Excepciones:
                         Si un determinado file no puede ser abierto
                     """
                     nFiles = 0
                     fileOk_flag = False
                     firstTime_flag = True
                     self.set += 1
                     if self.set > 999:
                         self.set = 0
                         self.foldercounter += 1
                     #busca el 1er file disponible
                     fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                     if fullfilename:
                         if self.__verifyFile(fullfilename, False):
                             fileOk_flag = True
                     #si no encuentra un file entonces espera y vuelve a buscar
                     if not(fileOk_flag):
                         for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
                             if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
                                 tries = self.nTries
                             else:
                                 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
                             for nTries in range( tries ):
                                 if firstTime_flag:
                                     print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
                                     time.sleep( self.delay )
                                 else:
                                     print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
                                 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                                 if fullfilename:
                                     if self.__verifyFile(fullfilename):
                                         fileOk_flag = True
                                         break
                             if fileOk_flag:
                                 break
                             firstTime_flag = False
                             print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename
                             self.set += 1
                             if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
                                 self.set = 0
                                 self.doy += 1
                                 self.foldercounter = 0
                     if fileOk_flag:
                         self.fileSize = os.path.getsize( fullfilename )
                         self.filename = fullfilename
                         self.flagIsNewFile = 1
                         if self.fp != None: self.fp.close()
                         self.fp = open(fullfilename, 'rb')
                         self.flagNoMoreFiles = 0
                         print 'Setting the file: %s' % fullfilename
                     else:
                         self.fileSize = 0
                         self.filename = None
                         self.flagIsNewFile = 0
                         self.fp = None
                         self.flagNoMoreFiles = 1
                         print 'No more Files'
                     return fileOk_flag
                 def setNextFile(self):
                     if self.fp != None:
                         self.fp.close()
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         return 0
                     self.__readFirstHeader()
                     self.nReadBlocks = 0
                     return 1
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.__rdBasicHeader()
                             return 1
                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         time.sleep( self.delay )
                     return 0
                 def __jumpToLastBlock(self):
                     if not(self.__isFirstTimeOnline):
                         return
                     csize = self.fileSize - self.fp.tell()
                     #sata el primer bloque de datos
                     if csize > self.processingHeaderObj.blockSize:
                         self.fp.seek(self.fp.tell() + self.processingHeaderObj.blockSize)
                     else:
                         return
                     csize = self.fileSize - self.fp.tell()
                     neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     factor = int(csize/neededsize)
                     if factor > 0:
                         self.fp.seek(self.fp.tell() + factor*neededsize)
                     self.flagIsNewFile = 0
                     self.__isFirstTimeOnline = 0
                 def __setNewBlock(self):
                     if self.fp == None:
                         return 0
                     if self.online:
                         self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         return 1
                     self.lastUTTime = self.basicHeaderObj.utc
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.__rdBasicHeader()
                         return 1
                     if self.__waitNewBlock():
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
                     self.flagTimeBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagTimeBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def __rdProcessingHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.processingHeaderObj.read(fp)
                 def __rdRadarControllerHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.radarControllerHeaderObj.read(fp)
                 def __rdSystemHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.systemHeaderObj.read(fp)
                 def __rdBasicHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.basicHeaderObj.read(fp)
                 def __readFirstHeader(self):
                     self.__rdBasicHeader()
                     self.__rdSystemHeader()
                     self.__rdRadarControllerHeader()
                     self.__rdProcessingHeader()
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
                     else:
                         raise ValueError, 'Data type was not defined'
                     self.dtype = datatype_str
                     self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
             #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
             #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def __verifyFile(self, filename, msgFlag=True):
                     msg = None
                     try:
                         fp = open(filename, 'rb')
                         currentPosition = fp.tell()
                     except:
                         if msgFlag:
                             print "The file %s can't be opened" % (filename)
                         return False
                     neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                     if neededSize == 0:
                         basicHeaderObj = BasicHeader(LOCALTIME)
                         systemHeaderObj = SystemHeader()
                         radarControllerHeaderObj = RadarControllerHeader()
                         processingHeaderObj = ProcessingHeader()
                         try:
                             if not( basicHeaderObj.read(fp) ): raise IOError
                             if not( systemHeaderObj.read(fp) ): raise IOError
                             if not( radarControllerHeaderObj.read(fp) ): raise IOError
                             if not( processingHeaderObj.read(fp) ): raise IOError
                             data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                             neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
                         except:
                             if msgFlag:
                                 print "\tThe file %s is empty or it hasn't enough data" % filename
                             fp.close()
                             return False
                     else:
                         msg = "\tSkipping the file %s due to it hasn't enough data" %filename
                     fp.close()
                     fileSize = os.path.getsize(filename)
                     currentSize = fileSize - currentPosition
                     if currentSize < neededSize:
                         if msgFlag and (msg != None):
                             print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
                         return False
                     return True
                 def setup(self,
                             path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=0,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True):
                     if path == None:
                         raise ValueError, "The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if online:
                         print "Searching files in online mode..."
                         for nTries in range( self.nTries ):
                             fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk)
                             if fullpath:
                                 break
                             print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
                             time.sleep( self.delay )
                         if not(fullpath):
                             print "There 'isn't valied files in %s" % path
                             return None
                         self.year = year
                         self.doy  = doy
                         self.set  = set - 1
                         self.path = path
                         self.foldercounter = foldercounter
                     else:
                         print "Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
                             print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime())
                             sys.exit(-1)
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                     self.online = online
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "No files"
                         sys.exit(-1)
             #        self.updateDataHeader()
                     return self.dataOut
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds
                     self.dataOut.flagTimeBlock = self.flagTimeBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                 def getFirstHeader(self):
                     raise ValueError, "This method has not been implemented"
                 def getData():
                     raise ValueError, "This method has not been implemented"
                 def hasNotDataInBuffer():
                     raise ValueError, "This method has not been implemented"
                 def readBlock():
                     raise ValueError, "This method has not been implemented"
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print "Number of read blocks per file %04d" %self.nReadBlocks
                 def printTotalBlocks(self):
                     print "Number of read blocks %04d" %self.nTotalBlocks
                 def printNumberOfBlock(self):
                     if self.flagIsNewBlock:
                         print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
                 def printInfo(self):
                     if self.__printInfo == False:
                         return
                     self.basicHeaderObj.printInfo()
                     self.systemHeaderObj.printInfo()
                     self.radarControllerHeaderObj.printInfo()
                     self.processingHeaderObj.printInfo()
                     self.__printInfo = False
                 def run(self, **kwargs):
                     if not(self.isConfig):
             #            self.dataOut = dataOut
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             class JRODataWriter(JRODataIO, Operation):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 blockIndex = 0
                 path = None
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 def __init__(self, dataOut=None):
                     raise ValueError, "Not implemented"
                 def hasAllDataInBuffer(self):
                     raise ValueError, "Not implemented"
                 def setBlockDimension(self):
                     raise ValueError, "Not implemented"
                 def writeBlock(self):
                     raise ValueError, "No implemented"
                 def putData(self):
                     raise ValueError, "No implemented"
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize #bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond  = (self.dataOut.utctime - utc)* 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
                     raise ValueError, "No implemented"
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                     self.dtype = self.dataOut.dtype
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def setNextFile(self):
                     """
                     Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, file )
                     fp = open( filename,'wb' )
                     self.blockIndex = 0
                     #guardando atributos
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.setFirstHeader()
                     print 'Writing the file: %s'%self.filename
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=None, set=0, ext=None):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path      :    el path destino en el cual se escribiran los files a crear
                         format    :    formato en el cual sera salvado un file
                         set       :    el setebo del file
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     ext = ext.lower()
                     self.ext = ext
                     self.path = path
                     self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     if not(self.setNextFile()):
                         print "There isn't a next file"
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class VoltageReader(JRODataReader):
                 """
                 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
                 perfiles*alturas*canales) son almacenados en la variable "buffer".
                                          perfiles * alturas * canales
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = VoltageReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         #to get one profile
                         profile =  readerObj.getData()
                         #print the profile
                         print profile
                         #If you want to see all datablock
                         print readerObj.datablock
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 ext = ".r"
                 optchar = "D"
                 dataOut = None
                 def __init__(self):
                     """
                     Inicializador de la clase VoltageReader para la lectura de datos de voltage.
                     Input:
                         dataOut    :    Objeto de la clase Voltage. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Variables afectadas:
                         self.dataOut
                     Return:
                         None
                     """
                     self.isConfig = False
                     self.datablock = None
                     self.utc = 0
                     self.ext = ".r"
                     self.optchar = "D"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.online = 0
                     self.fp = None
                     self.idFile = None
                     self.dtype = None
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.filename = None
                     self.fileSize = None
                     self.firstHeaderSize = 0
                     self.basicHeaderSize = 24
                     self.pathList = []
                     self.filenameList = []
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
                     self.flagNoMoreFiles = 0
                     self.set = 0
                     self.path = None
                     self.profileIndex = 2**32-1
                     self.delay  = 3   #seconds
                     self.nTries  = 3  #quantity tries
                     self.nFiles = 3   #number of files for searching
                     self.nReadBlocks = 0
                     self.flagIsNewFile = 1
                     self.__isFirstTimeOnline = 1
                     self.ippSeconds = 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     self.nTotalBlocks = 0
                     self.blocksize = 0
                     self.dataOut = self.createObjByDefault()
                 def createObjByDefault(self):
                     dataObj = Voltage()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
                         return 1
                     return 0
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.blocksize
                     Return:
                         None
                     """
                     pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
                     self.blocksize = pts2read
                 def readBlock(self):
                     """
                     readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Inputs:
                         None
                     Return:
                         None
                     Affected:
                         self.profileIndex
                         self.datablock
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
                     junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
                     try:
                         junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
                     except:
                         print "The read block (%3d) has not enough data" %self.nReadBlocks
                         return 0
                     junk = numpy.transpose(junk, (2,0,1))
                     self.datablock = junk['real'] + junk['imag']*1j
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.dataOut.dtype = self.dtype
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
                     self.dataOut.ippSeconds = self.ippSeconds
                     self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.flagShiftFFT = False
                     if self.radarControllerHeaderObj.code != None:
                         self.dataOut.nCode = self.radarControllerHeaderObj.nCode
                         self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
                         self.dataOut.code = self.radarControllerHeaderObj.code
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip
                     self.dataOut.flagShiftFFT = False
                 def getData(self):
                     """
                     getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Ademas incrementa el contador del buffer en 1.
                     Return:
                         data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                      buffer. Si no hay mas archivos a leer retorna None.
                     Variables afectadas:
                         self.dataOut
                         self.profileIndex
                     Affected:
                         self.dataOut
                         self.profileIndex
                         self.flagTimeBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             return 0
                         self.getFirstHeader()
                     if self.datablock == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.dataOut.data = self.datablock[:,self.profileIndex,:]
                     self.dataOut.flagNoData = False
                     self.getBasicHeader()
                     self.profileIndex += 1
                     return self.dataOut.data
             class VoltageWriter(JRODataWriter):
                 """
                 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".r"
                 optchar = "D"
                 shapeBuffer = None
                 def __init__(self):
                     """
                     Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                     Return: None
                     """
                     self.nTotalBlocks = 0
                     self.profileIndex = 0
                     self.isConfig = False
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.nTotalBlocks = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
                         return 1
                     return 0
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
                                         self.processingHeaderObj.nHeights,
                                         self.systemHeaderObj.nChannels)
                     self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
                                                  self.processingHeaderObj.profilesPerBlock,
                                                  self.processingHeaderObj.nHeights),
                                                  dtype=numpy.dtype('complex64'))
                 def writeBlock(self):
                     """
                     Escribe el buffer en el file designado
                     Affected:
                         self.profileIndex
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.blockIndex
                     Return: None
                     """
                     data = numpy.zeros( self.shapeBuffer, self.dtype )
                     junk = numpy.transpose(self.datablock, (1,2,0))
                     data['real'] = junk.real
                     data['imag'] = junk.imag
                     data = data.reshape( (-1) )
                     data.tofile( self.fp )
                     self.datablock.fill(0)
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.flagIsNewBlock
                         self.profileIndex
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagTimeBlock:
                         self.datablock.fill(0)
                         self.profileIndex = 0
                         self.setNextFile()
                     if self.profileIndex == 0:
                         self.setBasicHeader()
                     self.datablock[:,self.profileIndex,:] = self.dataOut.data
                     self.profileIndex += 1
                     if self.hasAllDataInBuffer():
                         #if self.flagIsNewFile:
                         self.writeNextBlock()
             #            self.setFirstHeader()
                     return 1
                 def __getProcessFlags(self):
                     processFlags = 0
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList =  [PROCFLAG.DATATYPE_CHAR,
                                        PROCFLAG.DATATYPE_SHORT,
                                        PROCFLAG.DATATYPE_LONG,
                                        PROCFLAG.DATATYPE_INT64,
                                        PROCFLAG.DATATYPE_FLOAT,
                                        PROCFLAG.DATATYPE_DOUBLE]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             dtypeValue = datatypeValueList[index]
                             break
                     processFlags += dtypeValue
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code != None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nCohInt > 1:
                         processFlags += PROCFLAG.COHERENT_INTEGRATION
                     return processFlags
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
                     '''
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList = [1,2,4,8,4,8]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             datatypeValue = datatypeValueList[index]
                             break
                     blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.dataOut.nProfiles * datatypeValue * 2)
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.setBasicHeader()
                     processingHeaderSize = 40 # bytes
                     self.processingHeaderObj.dtype = 0 # Voltage
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.processFlags = self.__getProcessFlags()
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
                     self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
                     self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
                     if self.dataOut.code != None:
                         self.processingHeaderObj.code = self.dataOut.code
                         self.processingHeaderObj.nCode = self.dataOut.nCode
                         self.processingHeaderObj.nBaud = self.dataOut.nBaud
                         codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)
                         processingHeaderSize += codesize
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                         processingHeaderSize += 12
                     self.processingHeaderObj.size = processingHeaderSize
             class SpectraReader(JRODataReader):
                 """
                 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
                 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
                                     paresCanalesIguales    * alturas * perfiles  (Self Spectra)
                                     paresCanalesDiferentes * alturas * perfiles  (Cross Spectra)
                                     canales * alturas                            (DC Channels)
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = SpectraReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         readerObj.getData()
                         print readerObj.data_spc
                         print readerObj.data_cspc
                         print readerObj.data_dc
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 pts2read_SelfSpectra = 0
                 pts2read_CrossSpectra = 0
                 pts2read_DCchannels = 0
                 ext = ".pdata"
                 optchar = "P"
                 dataOut = None
                 nRdChannels = None
                 nRdPairs = None
                 rdPairList = []
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraReader para la lectura de datos de espectros.
                     Inputs:
                         dataOut    :    Objeto de la clase Spectra. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Affected:
                         self.dataOut
                     Return      : None
                     """
                     self.isConfig = False
                     self.pts2read_SelfSpectra = 0
                     self.pts2read_CrossSpectra = 0
                     self.pts2read_DCchannels = 0
                     self.datablock = None
                     self.utc = None
                     self.ext = ".pdata"
                     self.optchar = "P"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.online = 0
                     self.fp = None
                     self.idFile = None
                     self.dtype = None
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.filename = None
                     self.fileSize = None
                     self.firstHeaderSize = 0
                     self.basicHeaderSize = 24
                     self.pathList = []
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
                     self.flagNoMoreFiles = 0
                     self.set = 0
                     self.path = None
                     self.delay  = 60   #seconds
                     self.nTries = 3  #quantity tries
                     self.nFiles = 3   #number of files for searching
                     self.nReadBlocks = 0
                     self.flagIsNewFile = 1
                     self.__isFirstTimeOnline = 1
                     self.ippSeconds = 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     self.nTotalBlocks = 0
                     self.blocksize = 0
                     self.dataOut = self.createObjByDefault()
                     self.profileIndex = 1 #Always
                 def createObjByDefault(self):
                     dataObj = Spectra()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     return 1
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.nRdChannels
                         self.nRdPairs
                         self.pts2read_SelfSpectra
                         self.pts2read_CrossSpectra
                         self.pts2read_DCchannels
                         self.blocksize
                         self.dataOut.nChannels
                         self.dataOut.nPairs
                     Return:
                         None
                     """
                     self.nRdChannels = 0
                     self.nRdPairs = 0
                     self.rdPairList = []
                     for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
                         if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
                             self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
                         else:
                             self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
                             self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
                     pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
                     self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
                     self.blocksize = self.pts2read_SelfSpectra
                     if self.processingHeaderObj.flag_cspc:
                         self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
                         self.blocksize += self.pts2read_CrossSpectra
                     if self.processingHeaderObj.flag_dc:
                         self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
                         self.blocksize += self.pts2read_DCchannels
             #        self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
                 def readBlock(self):
                     """
                     Lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Return: None
                     Variables afectadas:
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
                     blockOk_flag = False
                     fpointer = self.fp.tell()
                     spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
                     spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
                         cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_dc:
                         dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
                         dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
                     if not(self.processingHeaderObj.shif_fft):
                         #desplaza a la derecha en el eje 2 determinadas posiciones
                         shift = int(self.processingHeaderObj.profilesPerBlock/2)
                         spc = numpy.roll( spc, shift , axis=2 )
                         if self.processingHeaderObj.flag_cspc:
                             #desplaza a la derecha en el eje 2 determinadas posiciones
                             cspc = numpy.roll( cspc, shift, axis=2 )
             #            self.processingHeaderObj.shif_fft = True
                     spc = numpy.transpose( spc, (0,2,1) )
                     self.data_spc = spc
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.transpose( cspc, (0,2,1) )
                         self.data_cspc = cspc['real'] + cspc['imag']*1j
                     else:
                         self.data_cspc = None
                     if self.processingHeaderObj.flag_dc:
                         self.data_dc = dc['real'] + dc['imag']*1j
                     else:
                         self.data_dc = None
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.dataOut.dtype = self.dtype
                     self.dataOut.nPairs = self.nRdPairs
                     self.dataOut.pairsList = self.rdPairList
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
                     self.dataOut.ippSeconds = self.ippSeconds
                     self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
                     self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip
                     if self.processingHeaderObj.code != None:
                         self.dataOut.nCode = self.processingHeaderObj.nCode
                         self.dataOut.nBaud = self.processingHeaderObj.nBaud
                         self.dataOut.code = self.processingHeaderObj.code
                         self.dataOut.flagDecodeData = True
                 def getData(self):
                     """
                     Copia el buffer de lectura a la clase "Spectra",
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Return:
 :    Si no hay mas archivos disponibles
 :    Si hizo una buena copia del buffer
                     Affected:
                         self.dataOut
                         self.flagTimeBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             self.dataOut.flagNoData = True
                             return 0
                     #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
                     if self.data_dc == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.getBasicHeader()
                     self.getFirstHeader()
                     self.dataOut.data_spc = self.data_spc
                     self.dataOut.data_cspc = self.data_cspc
                     self.dataOut.data_dc = self.data_dc
                     self.dataOut.flagNoData = False
                     return self.dataOut.data_spc
             class SpectraWriter(JRODataWriter):
                 """
                 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".pdata"
                 optchar = "P"
                 shape_spc_Buffer = None
                 shape_cspc_Buffer = None
                 shape_dc_Buffer = None
                 data_spc = None
                 data_cspc = None
                 data_dc = None
             #    dataOut = None
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj
                     Return: None
                     """
                     self.isConfig = False
                     self.nTotalBlocks = 0
                     self.data_spc = None
                     self.data_cspc = None
                     self.data_dc = None
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.nTotalBlocks = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.noMoreFiles = 0
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     return 1
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shape_spc_Buffer = (self.dataOut.nChannels,
                                              self.processingHeaderObj.nHeights,
                                              self.processingHeaderObj.profilesPerBlock)
                     self.shape_cspc_Buffer = (self.dataOut.nPairs,
                                               self.processingHeaderObj.nHeights,
                                               self.processingHeaderObj.profilesPerBlock)
                     self.shape_dc_Buffer = (self.dataOut.nChannels,
                                             self.processingHeaderObj.nHeights)
                 def writeBlock(self):
                     """
                     Escribe el buffer en el file designado
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.nWriteBlocks
                     Return: None
                     """
                     spc = numpy.transpose( self.data_spc, (0,2,1) )
                     if not( self.processingHeaderObj.shif_fft ):
                         spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                     data = spc.reshape((-1))
                     data = data.astype(self.dtype[0])
                     data.tofile(self.fp)
                     if self.data_cspc != None:
                         data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
                         cspc = numpy.transpose( self.data_cspc, (0,2,1) )
                         if not( self.processingHeaderObj.shif_fft ):
                             cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                         data['real'] = cspc.real
                         data['imag'] = cspc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     if self.data_dc != None:
                         data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
                         dc = self.data_dc
                         data['real'] = dc.real
                         data['imag'] = dc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     self.data_spc.fill(0)
                     self.data_dc.fill(0)
                     if self.data_cspc != None:
                         self.data_cspc.fill(0)
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nWriteBlocks += 1
                     self.blockIndex += 1
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagTimeBlock:
                         self.data_spc.fill(0)
                         self.data_cspc.fill(0)
                         self.data_dc.fill(0)
                         self.setNextFile()
                     if self.flagIsNewFile == 0:
                         self.setBasicHeader()
                     self.data_spc = self.dataOut.data_spc.copy()
                     self.data_cspc = self.dataOut.data_cspc.copy()
                     self.data_dc = self.dataOut.data_dc.copy()
                     # #self.processingHeaderObj.dataBlocksPerFile)
                     if self.hasAllDataInBuffer():
             #            self.setFirstHeader()
                         self.writeNextBlock()
                     return 1
                 def __getProcessFlags(self):
                     processFlags = 0
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList =  [PROCFLAG.DATATYPE_CHAR,
                                        PROCFLAG.DATATYPE_SHORT,
                                        PROCFLAG.DATATYPE_LONG,
                                        PROCFLAG.DATATYPE_INT64,
                                        PROCFLAG.DATATYPE_FLOAT,
                                        PROCFLAG.DATATYPE_DOUBLE]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             dtypeValue = datatypeValueList[index]
                             break
                     processFlags += dtypeValue
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code != None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nIncohInt > 1:
                         processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                     if self.dataOut.data_dc != None:
                         processFlags += PROCFLAG.SAVE_CHANNELS_DC
                     return processFlags
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
                     '''
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList = [1,2,4,8,4,8]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             datatypeValue = datatypeValueList[index]
                             break
                     pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
                     pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
                     blocksize = (pts2write_SelfSpectra*datatypeValue)
                     if self.dataOut.data_cspc != None:
                         pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
                         blocksize += (pts2write_CrossSpectra*datatypeValue*2)
                     if self.dataOut.data_dc != None:
                         pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
                         blocksize += (pts2write_DCchannels*datatypeValue*2)
                     blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.setBasicHeader()
                     processingHeaderSize = 40 # bytes
                     self.processingHeaderObj.dtype = 1 # Spectra
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.processFlags = self.__getProcessFlags()
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
                     self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
                     self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
                     self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
                     if self.processingHeaderObj.totalSpectra > 0:
                         channelList = []
                         for channel in range(self.dataOut.nChannels):
                             channelList.append(channel)
                             channelList.append(channel)
                         pairsList = []
                         for pair in self.dataOut.pairsList:
                             pairsList.append(pair[0])
                             pairsList.append(pair[1])
                         spectraComb = channelList + pairsList
                         spectraComb = numpy.array(spectraComb,dtype="u1")
                         self.processingHeaderObj.spectraComb = spectraComb
                         sizeOfSpcComb = len(spectraComb)
                         processingHeaderSize += sizeOfSpcComb
             #        The processing header should not have information about code
             #        if self.dataOut.code != None:
             #            self.processingHeaderObj.code = self.dataOut.code
             #            self.processingHeaderObj.nCode = self.dataOut.nCode
             #            self.processingHeaderObj.nBaud = self.dataOut.nBaud
             #            nCodeSize = 4 # bytes
             #            nBaudSize = 4 # bytes
             #            codeSize = 4 # bytes
             #            sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)
             #            processingHeaderSize += sizeOfCode
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                         sizeOfFirstHeight = 4
                         sizeOfdeltaHeight = 4
                         sizeOfnHeights = 4
                         sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows
                         processingHeaderSize += sizeOfWindows
                     self.processingHeaderObj.size = processingHeaderSize
             class SpectraHeisWriter(Operation):
             #    set = None
                 setFile = None
                 idblock = None
                 doypath = None
                 subfolder = None
                 def __init__(self):
                     self.wrObj = FITS()
             #        self.dataOut = dataOut
                     self.nTotalBlocks=0
             #        self.set = None
                     self.setFile = None
                     self.idblock = 0
                     self.wrpath = None
                     self.doypath = None
                     self.subfolder = None
                     self.isConfig = False
                 def isNumber(str):
                     """
                     Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                     Excepciones:
                     Si un determinado string no puede ser convertido a numero
                     Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                     Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                     """
                     try:
                         float( str )
                         return True
                     except:
                         return False
                 def setup(self, dataOut, wrpath):
                     if not(os.path.exists(wrpath)):
                         os.mkdir(wrpath)
                     self.wrpath = wrpath
             #        self.setFile = 0
                     self.dataOut = dataOut
                 def putData(self):
                     name= time.localtime( self.dataOut.utctime)
                     ext=".fits"
                     if self.doypath == None:
                        self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
                        self.doypath = os.path.join( self.wrpath, self.subfolder )
                        os.mkdir(self.doypath)
                     if self.setFile == None:
             #           self.set = self.dataOut.set
                        self.setFile = 0
             #        if self.set != self.dataOut.set:
             ##            self.set = self.dataOut.set
             #            self.setFile = 0
                     #make the filename
                     file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
                     filename = os.path.join(self.wrpath,self.subfolder, file)
                     idblock = numpy.array([self.idblock],dtype="int64")
                     header=self.wrObj.cFImage(idblock=idblock,
                                             year=time.gmtime(self.dataOut.utctime).tm_year,
                                             month=time.gmtime(self.dataOut.utctime).tm_mon,
                                             day=time.gmtime(self.dataOut.utctime).tm_mday,
                                             hour=time.gmtime(self.dataOut.utctime).tm_hour,
                                             minute=time.gmtime(self.dataOut.utctime).tm_min,
                                             second=time.gmtime(self.dataOut.utctime).tm_sec)
                     c=3E8
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
                     colList = []
                     colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
                     colList.append(colFreq)
                     nchannel=self.dataOut.nChannels
                     for i in range(nchannel):
                         col = self.wrObj.writeData(name="PCh"+str(i+1),
                                                      format=str(self.dataOut.nFFTPoints)+'E',
                                                       data=10*numpy.log10(self.dataOut.data_spc[i,:]))
                         colList.append(col)
                     data=self.wrObj.Ctable(colList=colList)
                     self.wrObj.CFile(header,data)
                     self.wrObj.wFile(filename)
                     #update the setFile
                     self.setFile += 1
                     self.idblock += 1
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class FITS:
                 name=None
                 format=None
                 array =None
                 data =None
                 thdulist=None
                 prihdr=None
                 hdu=None
                 def __init__(self):
                     pass
                 def setColF(self,name,format,array):
                     self.name=name
                     self.format=format
                     self.array=array
                     a1=numpy.array([self.array],dtype=numpy.float32)
                     self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
                     return self.col1
             #    def setColP(self,name,format,data):
             #        self.name=name
             #        self.format=format
             #        self.data=data
             #        a2=numpy.array([self.data],dtype=numpy.float32)
             #        self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
             #        return self.col2
                 def writeData(self,name,format,data):
                     self.name=name
                     self.format=format
                     self.data=data
                     a2=numpy.array([self.data],dtype=numpy.float32)
                     self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
                     return self.col2
                 def cFImage(self,idblock,year,month,day,hour,minute,second):
                     self.hdu= pyfits.PrimaryHDU(idblock)
                     self.hdu.header.set("Year",year)
                     self.hdu.header.set("Month",month)
                     self.hdu.header.set("Day",day)
                     self.hdu.header.set("Hour",hour)
                     self.hdu.header.set("Minute",minute)
                     self.hdu.header.set("Second",second)
                     return self.hdu
                 def Ctable(self,colList):
                     self.cols=pyfits.ColDefs(colList)
                     self.tbhdu = pyfits.new_table(self.cols)
                     return self.tbhdu
                 def CFile(self,hdu,tbhdu):
                     self.thdulist=pyfits.HDUList([hdu,tbhdu])
                 def wFile(self,filename):
                     if os.path.isfile(filename):
                         os.remove(filename)
                     self.thdulist.writeto(filename)
             class ParameterConf:
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.name = ''
                     self.value = ''
                 def readXml(self, parmElement):
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                 def getElementName(self):
                     return self.ELEMENTNAME
             class Metadata:
                 def __init__(self, filename):
                     self.parmConfObjList = []
                     self.readXml(filename)
                 def readXml(self, filename):
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     self.projectElement = ElementTree().parse(filename)
                     self.project = self.projectElement.tag
                     parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         self.parmConfObjList.append(parmConfObj)
             class FitsWriter(Operation):
                 def __init__(self):
                     self.isConfig = False
                     self.dataBlocksPerFile = None
                     self.blockIndex = 0
                     self.flagIsNewFile = 1
                     self.fitsObj = None
                     self.optchar = 'P'
                     self.ext = '.fits'
                     self.setFile = 0
                 def setFitsHeader(self, dataOut, metadatafile):
                     header_data = pyfits.PrimaryHDU()
                     metadata4fits = Metadata(metadatafile)
                     for parameter in metadata4fits.parmConfObjList:
                         parm_name = parameter.name
                         parm_value = parameter.value
                         if parm_value == 'fromdatadatetime':
                             value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
                         elif parm_value == 'fromdataheights':
                             value = dataOut.nHeights
                         elif parm_value == 'fromdatachannel':
                             value = dataOut.nChannels
                         elif parm_value == 'fromdatasamples':
                             value = dataOut.nFFTPoints
                         else:
                             value = parm_value
                         header_data.header[parm_name] = value
                     header_data.header['NBLOCK'] = self.blockIndex
                     header_data.writeto(self.filename)
                 def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):
                     self.path = path
                     self.dataOut = dataOut
                     self.metadatafile = metadatafile
                     self.dataBlocksPerFile = dataBlocksPerFile
                 def open(self):
                     self.fitsObj = pyfits.open(self.filename, mode='update')
                 def addData(self, data):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATA'])
                     extension.header['UTCTIME'] = self.dataOut.utctime
                     self.fitsObj.append(extension)
                     self.blockIndex += 1
                     self.fitsObj[0].header['NBLOCK'] = self.blockIndex
                     self.write()
                 def write(self):
                     self.fitsObj.flush(verbose=True)
                     self.fitsObj.close()
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, file )
                     self.blockIndex = 0
                     self.filename = filename
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     print 'Writing the file: %s'%self.filename
                     self.setFitsHeader(self.dataOut, self.metadatafile)
                     return 1
                 def writeBlock(self):
                     self.addData(self.dataOut.data_spc)
                     self.flagIsNewFile = 0
                 def __setNewBlock(self):
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.dataBlocksPerFile:
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def putData(self):
                     if self.flagIsNewFile:
                         self.setNextFile()
                     self.writeNextBlock()
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()

1

'''

1

'''

2

3

$Author: murco $

3

$Author: murco $

4

$Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $

4

$Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $

5

'''

5

'''

6

7

import os, sys

7

import os, sys

8

import glob

8

import glob

9

import time

9

import time

10

import numpy

10

import numpy

11

import fnmatch

11

import fnmatch

12

import time, datetime

12

import time, datetime

13

from xml.etree.ElementTree import Element, SubElement, ElementTree

13

try:

14

try:

14

import pyfits

15

import pyfits

15

except:

16

except:

16

print "pyfits module has not been imported, it should be installed to save files in fits format"

17

print "pyfits module has not been imported, it should be installed to save files in fits format"

17

18

from jrodata import *

19

from jrodata import *

19

from jroheaderIO import *

20

from jroheaderIO import *

20

from jroprocessing import *

21

from jroprocessing import *

21

22

LOCALTIME = True #-18000

23

LOCALTIME = True #-18000

23

24

def isNumber(str):

25

def isNumber(str):

25

"""

26

"""

26

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

27

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

27

28

Excepciones:

29

Excepciones:

29

Si un determinado string no puede ser convertido a numero

30

Si un determinado string no puede ser convertido a numero

30

Input:

31

Input:

31

str, string al cual se le analiza para determinar si convertible a un numero o no

32

str, string al cual se le analiza para determinar si convertible a un numero o no

32

33

Return:

34

Return:

34

True : si el string es uno numerico

35

True : si el string es uno numerico

35

False : no es un string numerico

36

False : no es un string numerico

36

"""

37

"""

37

try:

38

try:

38

float( str )

39

float( str )

39

return True

40

return True

40

except:

41

except:

41

return False

42

return False

42

43

def isThisFileinRange(filename, startUTSeconds, endUTSeconds):

44

def isThisFileinRange(filename, startUTSeconds, endUTSeconds):

44

"""

45

"""

45

Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.

46

Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.

46

47

Inputs:

48

Inputs:

48

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

49

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

49

50

startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en

51

startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en

51

segundos contados desde 01/01/1970.

52

segundos contados desde 01/01/1970.

52

endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en

53

endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en

53

segundos contados desde 01/01/1970.

54

segundos contados desde 01/01/1970.

54

55

Return:

56

Return:

56

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

57

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

57

fecha especificado, de lo contrario retorna False.

58

fecha especificado, de lo contrario retorna False.

58

59

Excepciones:

60

Excepciones:

60

Si el archivo no existe o no puede ser abierto

61

Si el archivo no existe o no puede ser abierto

61

Si la cabecera no puede ser leida.

62

Si la cabecera no puede ser leida.

62

63

"""

64

"""

64

basicHeaderObj = BasicHeader(LOCALTIME)

65

basicHeaderObj = BasicHeader(LOCALTIME)

65

66

try:

67

try:

67

fp = open(filename,'rb')

68

fp = open(filename,'rb')

68

except:

69

except:

69

raise IOError, "The file %s can't be opened" %(filename)

70

raise IOError, "The file %s can't be opened" %(filename)

70

71

sts = basicHeaderObj.read(fp)

72

sts = basicHeaderObj.read(fp)

72

fp.close()

73

fp.close()

73

74

if not(sts):

75

if not(sts):

75

print "Skipping the file %s because it has not a valid header" %(filename)

76

print "Skipping the file %s because it has not a valid header" %(filename)

76

return 0

77

return 0

77

78

if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):

79

if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):

79

return 0

80

return 0

80

81

return 1

82

return 1

82

83

def isFileinThisTime(filename, startTime, endTime):

84

def isFileinThisTime(filename, startTime, endTime):

84

"""

85

"""

85

Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.

86

Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.

86

87

Inputs:

88

Inputs:

88

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

89

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

89

90

startTime : tiempo inicial del rango seleccionado en formato datetime.time

91

startTime : tiempo inicial del rango seleccionado en formato datetime.time

91

92

endTime : tiempo final del rango seleccionado en formato datetime.time

93

endTime : tiempo final del rango seleccionado en formato datetime.time

93

94

Return:

95

Return:

95

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

96

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

96

fecha especificado, de lo contrario retorna False.

97

fecha especificado, de lo contrario retorna False.

97

98

Excepciones:

99

Excepciones:

99

Si el archivo no existe o no puede ser abierto

100

Si el archivo no existe o no puede ser abierto

100

Si la cabecera no puede ser leida.

101

Si la cabecera no puede ser leida.

101

102

"""

103

"""

103

104

105

try:

106

try:

106

fp = open(filename,'rb')

107

fp = open(filename,'rb')

107

except:

108

except:

108

raise IOError, "The file %s can't be opened" %(filename)

109

raise IOError, "The file %s can't be opened" %(filename)

109

110

basicHeaderObj = BasicHeader(LOCALTIME)

111

basicHeaderObj = BasicHeader(LOCALTIME)

111

sts = basicHeaderObj.read(fp)

112

sts = basicHeaderObj.read(fp)

112

fp.close()

113

fp.close()

113

114

thisDatetime = basicHeaderObj.datatime

115

thisDatetime = basicHeaderObj.datatime

115

thisTime = basicHeaderObj.datatime.time()

116

thisTime = basicHeaderObj.datatime.time()

116

117

if not(sts):

118

if not(sts):

118

print "Skipping the file %s because it has not a valid header" %(filename)

119

print "Skipping the file %s because it has not a valid header" %(filename)

119

return None

120

return None

120

121

if not ((startTime <= thisTime) and (endTime > thisTime)):

122

if not ((startTime <= thisTime) and (endTime > thisTime)):

122

return None

123

return None

123

124

return thisDatetime

125

return thisDatetime

125

126

def getlastFileFromPath(path, ext):

127

def getlastFileFromPath(path, ext):

127

"""

128

"""

128

Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"

129

Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"

129

al final de la depuracion devuelve el ultimo file de la lista que quedo.

130

al final de la depuracion devuelve el ultimo file de la lista que quedo.

130

131

Input:

132

Input:

132

fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta

133

fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta

133

ext : extension de los files contenidos en una carpeta

134

ext : extension de los files contenidos en una carpeta

134

135

Return:

136

Return:

136

El ultimo file de una determinada carpeta, no se considera el path.

137

El ultimo file de una determinada carpeta, no se considera el path.

137

"""

138

"""

138

validFilelist = []

139

validFilelist = []

139

fileList = os.listdir(path)

140

fileList = os.listdir(path)

140

141

# 0 1234 567 89A BCDE

142

# 0 1234 567 89A BCDE

142

# H YYYY DDD SSS .ext

143

# H YYYY DDD SSS .ext

143

144

for file in fileList:

145

for file in fileList:

145

try:

146

try:

146

year = int(file[1:5])

147

year = int(file[1:5])

147

doy = int(file[5:8])

148

doy = int(file[5:8])

148

149

150

except:

151

except:

151

continue

152

continue

152

153

if (os.path.splitext(file)[-1].lower() != ext.lower()):

154

if (os.path.splitext(file)[-1].lower() != ext.lower()):

154

continue

155

continue

155

156

validFilelist.append(file)

157

validFilelist.append(file)

157

158

if validFilelist:

159

if validFilelist:

159

validFilelist = sorted( validFilelist, key=str.lower )

160

validFilelist = sorted( validFilelist, key=str.lower )

160

return validFilelist[-1]

161

return validFilelist[-1]

161

162

return None

163

return None

163

164

def checkForRealPath(path, foldercounter, year, doy, set, ext):

165

def checkForRealPath(path, foldercounter, year, doy, set, ext):

165

"""

166

"""

166

Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,

167

Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,

167

Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar

168

Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar

168

el path exacto de un determinado file.

169

el path exacto de un determinado file.

169

170

Example :

171

Example :

171

nombre correcto del file es .../.../D2009307/P2009307367.ext

172

nombre correcto del file es .../.../D2009307/P2009307367.ext

172

173

Entonces la funcion prueba con las siguientes combinaciones

174

Entonces la funcion prueba con las siguientes combinaciones

174

.../.../y2009307367.ext

175

.../.../y2009307367.ext

175

.../.../Y2009307367.ext

176

.../.../Y2009307367.ext

176

.../.../x2009307/y2009307367.ext

177

.../.../x2009307/y2009307367.ext

177

.../.../x2009307/Y2009307367.ext

178

.../.../x2009307/Y2009307367.ext

178

.../.../X2009307/y2009307367.ext

179

.../.../X2009307/y2009307367.ext

179

.../.../X2009307/Y2009307367.ext

180

.../.../X2009307/Y2009307367.ext

180

siendo para este caso, la ultima combinacion de letras, identica al file buscado

181

siendo para este caso, la ultima combinacion de letras, identica al file buscado

181

182

Return:

183

Return:

183

Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file

184

Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file

184

caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas

185

caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas

185

para el filename

186

para el filename

186

"""

187

"""

187

fullfilename = None

188

fullfilename = None

188

find_flag = False

189

find_flag = False

189

filename = None

190

filename = None

190

191

prefixDirList = [None,'d','D']

192

prefixDirList = [None,'d','D']

192

if ext.lower() == ".r": #voltage

193

if ext.lower() == ".r": #voltage

193

prefixFileList = ['d','D']

194

prefixFileList = ['d','D']

194

elif ext.lower() == ".pdata": #spectra

195

elif ext.lower() == ".pdata": #spectra

195

prefixFileList = ['p','P']

196

prefixFileList = ['p','P']

196

else:

197

else:

197

return None, filename

198

return None, filename

198

199

#barrido por las combinaciones posibles

200

#barrido por las combinaciones posibles

200

for prefixDir in prefixDirList:

201

for prefixDir in prefixDirList:

201

thispath = path

202

thispath = path

202

if prefixDir != None:

203

if prefixDir != None:

203

#formo el nombre del directorio xYYYYDDD (x=d o x=D)

204

#formo el nombre del directorio xYYYYDDD (x=d o x=D)

204

if foldercounter == 0:

205

if foldercounter == 0:

205

thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))

206

thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))

206

else:

207

else:

207

thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))

208

thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))

208

for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"

209

for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"

209

filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext

210

filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext

210

fullfilename = os.path.join( thispath, filename ) #formo el path completo

211

fullfilename = os.path.join( thispath, filename ) #formo el path completo

211

212

if os.path.exists( fullfilename ): #verifico que exista

213

if os.path.exists( fullfilename ): #verifico que exista

213

find_flag = True

214

find_flag = True

214

break

215

break

215

if find_flag:

216

if find_flag:

216

break

217

break

217

218

if not(find_flag):

219

if not(find_flag):

219

return None, filename

220

return None, filename

220

221

return fullfilename, filename

222

return fullfilename, filename

222

223

def isDoyFolder(folder):

224

def isDoyFolder(folder):

224

try:

225

try:

225

year = int(folder[1:5])

226

year = int(folder[1:5])

226

except:

227

except:

227

return 0

228

return 0

228

229

try:

230

try:

230

doy = int(folder[5:8])

231

doy = int(folder[5:8])

231

except:

232

except:

232

return 0

233

return 0

233

234

return 1

235

return 1

235

236

class JRODataIO:

237

class JRODataIO:

237

238

c = 3E8

239

c = 3E8

239

240

isConfig = False

241

isConfig = False

241

242

basicHeaderObj = BasicHeader(LOCALTIME)

243

basicHeaderObj = BasicHeader(LOCALTIME)

243

244

systemHeaderObj = SystemHeader()

245

systemHeaderObj = SystemHeader()

245

246

radarControllerHeaderObj = RadarControllerHeader()

247

radarControllerHeaderObj = RadarControllerHeader()

247

248

processingHeaderObj = ProcessingHeader()

249

processingHeaderObj = ProcessingHeader()

249

250

online = 0

251

online = 0

251

252

dtype = None

253

dtype = None

253

254

pathList = []

255

pathList = []

255

256

filenameList = []

257

filenameList = []

257

258

filename = None

259

filename = None

259

260

ext = None

261

ext = None

261

262

flagIsNewFile = 1

263

flagIsNewFile = 1

263

264

flagTimeBlock = 0

265

flagTimeBlock = 0

265

266

flagIsNewBlock = 0

267

flagIsNewBlock = 0

267

268

fp = None

269

fp = None

269

270

firstHeaderSize = 0

271

firstHeaderSize = 0

271

272

basicHeaderSize = 24

273

basicHeaderSize = 24

273

274

versionFile = 1103

275

versionFile = 1103

275

276

fileSize = None

277

fileSize = None

277

278

ippSeconds = None

279

ippSeconds = None

279

280

fileSizeByHeader = None

281

fileSizeByHeader = None

281

282

fileIndex = None

283

fileIndex = None

283

284

profileIndex = None

285

profileIndex = None

285

286

blockIndex = None

287

blockIndex = None

287

288

nTotalBlocks = None

289

nTotalBlocks = None

289

290

maxTimeStep = 30

291

maxTimeStep = 30

291

292

lastUTTime = None

293

lastUTTime = None

293

294

datablock = None

295

datablock = None

295

296

dataOut = None

297

dataOut = None

297

298

blocksize = None

299

blocksize = None

299

300

def __init__(self):

301

def __init__(self):

301

302

raise ValueError, "Not implemented"

303

raise ValueError, "Not implemented"

303

304

def run(self):

305

def run(self):

305

306

raise ValueError, "Not implemented"

307

raise ValueError, "Not implemented"

307

308

def getOutput(self):

309

def getOutput(self):

309

310

return self.dataOut

311

return self.dataOut

311

312

class JRODataReader(JRODataIO, ProcessingUnit):

313

class JRODataReader(JRODataIO, ProcessingUnit):

313

314

nReadBlocks = 0

315

nReadBlocks = 0

315

316

delay = 10 #number of seconds waiting a new file

317

delay = 10 #number of seconds waiting a new file

317

318

nTries = 3 #quantity tries

319

nTries = 3 #quantity tries

319

320

nFiles = 3 #number of files for searching

321

nFiles = 3 #number of files for searching

321

322

path = None

323

path = None

323

324

foldercounter = 0

325

foldercounter = 0

325

326

flagNoMoreFiles = 0

327

flagNoMoreFiles = 0

327

328

datetimeList = []

329

datetimeList = []

329

330

__isFirstTimeOnline = 1

331

__isFirstTimeOnline = 1

331

332

__printInfo = True

333

__printInfo = True

333

334

profileIndex = None

335

profileIndex = None

335

336

def __init__(self):

337

def __init__(self):

337

338

"""

339

"""

339

340

"""

341

"""

341

342

raise ValueError, "This method has not been implemented"

343

raise ValueError, "This method has not been implemented"

343

344

345

def createObjByDefault(self):

346

def createObjByDefault(self):

346

"""

347

"""

347

348

"""

349

"""

349

raise ValueError, "This method has not been implemented"

350

raise ValueError, "This method has not been implemented"

350

351

def getBlockDimension(self):

352

def getBlockDimension(self):

352

353

raise ValueError, "No implemented"

354

raise ValueError, "No implemented"

354

355

def __searchFilesOffLine(self,

356

def __searchFilesOffLine(self,

356

path,

357

path,

357

startDate,

358

startDate,

358

endDate,

359

endDate,

359

startTime=datetime.time(0,0,0),

360

startTime=datetime.time(0,0,0),

360

endTime=datetime.time(23,59,59),

361

endTime=datetime.time(23,59,59),

361

set=None,

362

set=None,

362

expLabel='',

363

expLabel='',

363

ext='.r',

364

ext='.r',

364

walk=True):

365

walk=True):

365

366

pathList = []

367

pathList = []

367

368

if not walk:

369

if not walk:

369

pathList.append(path)

370

pathList.append(path)

370

371

else:

372

else:

372

dirList = []

373

dirList = []

373

for thisPath in os.listdir(path):

374

for thisPath in os.listdir(path):

374

if not os.path.isdir(os.path.join(path,thisPath)):

375

if not os.path.isdir(os.path.join(path,thisPath)):

375

continue

376

continue

376

if not isDoyFolder(thisPath):

377

if not isDoyFolder(thisPath):

377

continue

378

continue

378

379

dirList.append(thisPath)

380

dirList.append(thisPath)

380

381

if not(dirList):

382

if not(dirList):

382

return None, None

383

return None, None

383

384

thisDate = startDate

385

thisDate = startDate

385

386

while(thisDate <= endDate):

387

while(thisDate <= endDate):

387

year = thisDate.timetuple().tm_year

388

year = thisDate.timetuple().tm_year

388

doy = thisDate.timetuple().tm_yday

389

doy = thisDate.timetuple().tm_yday

389

390

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

391

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

391

if len(matchlist) == 0:

392

if len(matchlist) == 0:

392

thisDate += datetime.timedelta(1)

393

thisDate += datetime.timedelta(1)

393

continue

394

continue

394

for match in matchlist:

395

for match in matchlist:

395

pathList.append(os.path.join(path,match,expLabel))

396

pathList.append(os.path.join(path,match,expLabel))

396

397

thisDate += datetime.timedelta(1)

398

thisDate += datetime.timedelta(1)

398

399

if pathList == []:

400

if pathList == []:

400

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

401

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

401

return None, None

402

return None, None

402

403

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

404

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

404

405

filenameList = []

406

filenameList = []

406

datetimeList = []

407

datetimeList = []

407

408

for i in range(len(pathList)):

409

for i in range(len(pathList)):

409

410

thisPath = pathList[i]

411

thisPath = pathList[i]

411

412

fileList = glob.glob1(thisPath, "*%s" %ext)

413

fileList = glob.glob1(thisPath, "*%s" %ext)

413

fileList.sort()

414

fileList.sort()

414

415

for file in fileList:

416

for file in fileList:

416

417

filename = os.path.join(thisPath,file)

418

filename = os.path.join(thisPath,file)

418

thisDatetime = isFileinThisTime(filename, startTime, endTime)

419

thisDatetime = isFileinThisTime(filename, startTime, endTime)

419

420

if not(thisDatetime):

421

if not(thisDatetime):

421

continue

422

continue

422

423

filenameList.append(filename)

424

filenameList.append(filename)

424

datetimeList.append(thisDatetime)

425

datetimeList.append(thisDatetime)

425

426

if not(filenameList):

427

if not(filenameList):

427

print "Any file was found for the time range %s - %s" %(startTime, endTime)

428

print "Any file was found for the time range %s - %s" %(startTime, endTime)

428

return None, None

429

return None, None

429

430

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

431

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

431

print

432

print

432

433

for i in range(len(filenameList)):

434

for i in range(len(filenameList)):

434

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

435

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

435

436

self.filenameList = filenameList

437

self.filenameList = filenameList

437

self.datetimeList = datetimeList

438

self.datetimeList = datetimeList

438

439

return pathList, filenameList

440

return pathList, filenameList

440

441

def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True):

442

def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True):

442

443

"""

444

"""

444

Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y

445

Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y

445

devuelve el archivo encontrado ademas de otros datos.

446

devuelve el archivo encontrado ademas de otros datos.

446

447

Input:

448

Input:

448

path : carpeta donde estan contenidos los files que contiene data

449

path : carpeta donde estan contenidos los files que contiene data

449

450

expLabel : Nombre del subexperimento (subfolder)

451

expLabel : Nombre del subexperimento (subfolder)

451

452

ext : extension de los files

453

ext : extension de los files

453

454

walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)

455

walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)

455

456

Return:

457

Return:

457

directory : eL directorio donde esta el file encontrado

458

directory : eL directorio donde esta el file encontrado

458

filename : el ultimo file de una determinada carpeta

459

filename : el ultimo file de una determinada carpeta

459

year : el anho

460

year : el anho

460

doy : el numero de dia del anho

461

doy : el numero de dia del anho

461

set : el set del archivo

462

set : el set del archivo

462

463

464

"""

465

"""

465

dirList = []

466

dirList = []

466

467

if not walk:

468

if not walk:

468

fullpath = path

469

fullpath = path

469

470

else:

471

else:

471

#Filtra solo los directorios

472

#Filtra solo los directorios

472

for thisPath in os.listdir(path):

473

for thisPath in os.listdir(path):

473

if not os.path.isdir(os.path.join(path,thisPath)):

474

if not os.path.isdir(os.path.join(path,thisPath)):

474

continue

475

continue

475

if not isDoyFolder(thisPath):

476

if not isDoyFolder(thisPath):

476

continue

477

continue

477

478

dirList.append(thisPath)

479

dirList.append(thisPath)

479

480

if not(dirList):

481

if not(dirList):

481

return None, None, None, None, None

482

return None, None, None, None, None

482

483

dirList = sorted( dirList, key=str.lower )

484

dirList = sorted( dirList, key=str.lower )

484

485

doypath = dirList[-1]

486

doypath = dirList[-1]

486

foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0

487

foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0

487

fullpath = os.path.join(path, doypath, expLabel)

488

fullpath = os.path.join(path, doypath, expLabel)

488

489

490

print "%s folder was found: " %(fullpath )

491

print "%s folder was found: " %(fullpath )

491

492

filename = getlastFileFromPath(fullpath, ext)

493

filename = getlastFileFromPath(fullpath, ext)

493

494

if not(filename):

495

if not(filename):

495

return None, None, None, None, None

496

return None, None, None, None, None

496

497

print "%s file was found" %(filename)

498

print "%s file was found" %(filename)

498

499

if not(self.__verifyFile(os.path.join(fullpath, filename))):

500

if not(self.__verifyFile(os.path.join(fullpath, filename))):

500

return None, None, None, None, None

501

return None, None, None, None, None

501

502

year = int( filename[1:5] )

503

year = int( filename[1:5] )

503

doy = int( filename[5:8] )

504

doy = int( filename[5:8] )

504

set = int( filename[8:11] )

505

set = int( filename[8:11] )

505

506

return fullpath, foldercounter, filename, year, doy, set

507

return fullpath, foldercounter, filename, year, doy, set

507

508

def __setNextFileOffline(self):

509

def __setNextFileOffline(self):

509

510

idFile = self.fileIndex

511

idFile = self.fileIndex

511

512

while (True):

513

while (True):

513

idFile += 1

514

idFile += 1

514

if not(idFile < len(self.filenameList)):

515

if not(idFile < len(self.filenameList)):

515

self.flagNoMoreFiles = 1

516

self.flagNoMoreFiles = 1

516

print "No more Files"

517

print "No more Files"

517

return 0

518

return 0

518

519

filename = self.filenameList[idFile]

520

filename = self.filenameList[idFile]

520

521

if not(self.__verifyFile(filename)):

522

if not(self.__verifyFile(filename)):

522

continue

523

continue

523

524

fileSize = os.path.getsize(filename)

525

fileSize = os.path.getsize(filename)

525

fp = open(filename,'rb')

526

fp = open(filename,'rb')

526

break

527

break

527

528

self.flagIsNewFile = 1

529

self.flagIsNewFile = 1

529

self.fileIndex = idFile

530

self.fileIndex = idFile

530

self.filename = filename

531

self.filename = filename

531

self.fileSize = fileSize

532

self.fileSize = fileSize

532

self.fp = fp

533

self.fp = fp

533

534

print "Setting the file: %s"%self.filename

535

print "Setting the file: %s"%self.filename

535

536

return 1

537

return 1

537

538

def __setNextFileOnline(self):

539

def __setNextFileOnline(self):

539

"""

540

"""

540

Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si

541

Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si

541

no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files

542

no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files

542

siguientes.

543

siguientes.

543

544

Affected:

545

Affected:

545

self.flagIsNewFile

546

self.flagIsNewFile

546

self.filename

547

self.filename

547

self.fileSize

548

self.fileSize

548

self.fp

549

self.fp

549

self.set

550

self.set

550

self.flagNoMoreFiles

551

self.flagNoMoreFiles

551

552

Return:

553

Return:

553

0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado

554

0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado

554

1 : si el file fue abierto con exito y esta listo a ser leido

555

1 : si el file fue abierto con exito y esta listo a ser leido

555

556

Excepciones:

557

Excepciones:

557

Si un determinado file no puede ser abierto

558

Si un determinado file no puede ser abierto

558

"""

559

"""

559

nFiles = 0

560

nFiles = 0

560

fileOk_flag = False

561

fileOk_flag = False

561

firstTime_flag = True

562

firstTime_flag = True

562

563

self.set += 1

564

self.set += 1

564

565

if self.set > 999:

566

if self.set > 999:

566

self.set = 0

567

self.set = 0

567

self.foldercounter += 1

568

self.foldercounter += 1

568

569

#busca el 1er file disponible

570

#busca el 1er file disponible

570

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

571

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

571

if fullfilename:

572

if fullfilename:

572

if self.__verifyFile(fullfilename, False):

573

if self.__verifyFile(fullfilename, False):

573

fileOk_flag = True

574

fileOk_flag = True

574

575

#si no encuentra un file entonces espera y vuelve a buscar

576

#si no encuentra un file entonces espera y vuelve a buscar

576

if not(fileOk_flag):

577

if not(fileOk_flag):

577

for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles

578

for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles

578

579

if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces

580

if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces

580

tries = self.nTries

581

tries = self.nTries

581

else:

582

else:

582

tries = 1 #si no es la 1era vez entonces solo lo hace una vez

583

tries = 1 #si no es la 1era vez entonces solo lo hace una vez

583

584

for nTries in range( tries ):

585

for nTries in range( tries ):

585

if firstTime_flag:

586

if firstTime_flag:

586

print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )

587

print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )

587

time.sleep( self.delay )

588

time.sleep( self.delay )

588

else:

589

else:

589

print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)

590

print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)

590

591

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

592

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

592

if fullfilename:

593

if fullfilename:

593

if self.__verifyFile(fullfilename):

594

if self.__verifyFile(fullfilename):

594

fileOk_flag = True

595

fileOk_flag = True

595

break

596

break

596

597

if fileOk_flag:

598

if fileOk_flag:

598

break

599

break

599

600

firstTime_flag = False

601

firstTime_flag = False

601

602

print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename

603

print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename

603

self.set += 1

604

self.set += 1

604

605

if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta

606

if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta

606

self.set = 0

607

self.set = 0

607

self.doy += 1

608

self.doy += 1

608

self.foldercounter = 0

609

self.foldercounter = 0

609

610

if fileOk_flag:

611

if fileOk_flag:

611

self.fileSize = os.path.getsize( fullfilename )

612

self.fileSize = os.path.getsize( fullfilename )

612

self.filename = fullfilename

613

self.filename = fullfilename

613

self.flagIsNewFile = 1

614

self.flagIsNewFile = 1

614

if self.fp != None: self.fp.close()

615

if self.fp != None: self.fp.close()

615

self.fp = open(fullfilename, 'rb')

616

self.fp = open(fullfilename, 'rb')

616

self.flagNoMoreFiles = 0

617

self.flagNoMoreFiles = 0

617

print 'Setting the file: %s' % fullfilename

618

print 'Setting the file: %s' % fullfilename

618

else:

619

else:

619

self.fileSize = 0

620

self.fileSize = 0

620

self.filename = None

621

self.filename = None

621

self.flagIsNewFile = 0

622

self.flagIsNewFile = 0

622

self.fp = None

623

self.fp = None

623

self.flagNoMoreFiles = 1

624

self.flagNoMoreFiles = 1

624

print 'No more Files'

625

print 'No more Files'

625

626

return fileOk_flag

627

return fileOk_flag

627

628

629

def setNextFile(self):

630

def setNextFile(self):

630

if self.fp != None:

631

if self.fp != None:

631

self.fp.close()

632

self.fp.close()

632

633

if self.online:

634

if self.online:

634

newFile = self.__setNextFileOnline()

635

newFile = self.__setNextFileOnline()

635

else:

636

else:

636

newFile = self.__setNextFileOffline()

637

newFile = self.__setNextFileOffline()

637

638

if not(newFile):

639

if not(newFile):

639

return 0

640

return 0

640

641

self.__readFirstHeader()

642

self.__readFirstHeader()

642

self.nReadBlocks = 0

643

self.nReadBlocks = 0

643

return 1

644

return 1

644

645

def __waitNewBlock(self):

646

def __waitNewBlock(self):

646

"""

647

"""

647

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

648

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

648

649

Si el modo de lectura es OffLine siempre retorn 0

650

Si el modo de lectura es OffLine siempre retorn 0

650

"""

651

"""

651

if not self.online:

652

if not self.online:

652

return 0

653

return 0

653

654

if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):

655

if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):

655

return 0

656

return 0

656

657

currentPointer = self.fp.tell()

658

currentPointer = self.fp.tell()

658

659

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

660

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

660

661

for nTries in range( self.nTries ):

662

for nTries in range( self.nTries ):

662

663

self.fp.close()

664

self.fp.close()

664

self.fp = open( self.filename, 'rb' )

665

self.fp = open( self.filename, 'rb' )

665

self.fp.seek( currentPointer )

666

self.fp.seek( currentPointer )

666

667

self.fileSize = os.path.getsize( self.filename )

668

self.fileSize = os.path.getsize( self.filename )

668

currentSize = self.fileSize - currentPointer

669

currentSize = self.fileSize - currentPointer

669

670

if ( currentSize >= neededSize ):

671

if ( currentSize >= neededSize ):

671

self.__rdBasicHeader()

672

self.__rdBasicHeader()

672

return 1

673

return 1

673

674

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

675

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

675

time.sleep( self.delay )

676

time.sleep( self.delay )

676

677

678

return 0

679

return 0

679

680

def __jumpToLastBlock(self):

681

def __jumpToLastBlock(self):

681

682

if not(self.__isFirstTimeOnline):

683

if not(self.__isFirstTimeOnline):

683

return

684

return

684

685

csize = self.fileSize - self.fp.tell()

686

csize = self.fileSize - self.fp.tell()

686

687

#sata el primer bloque de datos

688

#sata el primer bloque de datos

688

if csize > self.processingHeaderObj.blockSize:

689

if csize > self.processingHeaderObj.blockSize:

689

self.fp.seek(self.fp.tell() + self.processingHeaderObj.blockSize)

690

self.fp.seek(self.fp.tell() + self.processingHeaderObj.blockSize)

690

else:

691

else:

691

return

692

return

692

693

csize = self.fileSize - self.fp.tell()

694

csize = self.fileSize - self.fp.tell()

694

neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

695

neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

695

factor = int(csize/neededsize)

696

factor = int(csize/neededsize)

696

if factor > 0:

697

if factor > 0:

697

self.fp.seek(self.fp.tell() + factor*neededsize)

698

self.fp.seek(self.fp.tell() + factor*neededsize)

698

699

self.flagIsNewFile = 0

700

self.flagIsNewFile = 0

700

self.__isFirstTimeOnline = 0

701

self.__isFirstTimeOnline = 0

701

702

703

def __setNewBlock(self):

704

def __setNewBlock(self):

704

705

if self.fp == None:

706

if self.fp == None:

706

return 0

707

return 0

707

708

if self.online:

709

if self.online:

709

self.__jumpToLastBlock()

710

self.__jumpToLastBlock()

710

711

if self.flagIsNewFile:

712

if self.flagIsNewFile:

712

return 1

713

return 1

713

714

self.lastUTTime = self.basicHeaderObj.utc

715

self.lastUTTime = self.basicHeaderObj.utc

715

currentSize = self.fileSize - self.fp.tell()

716

currentSize = self.fileSize - self.fp.tell()

716

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

717

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

717

718

if (currentSize >= neededSize):

719

if (currentSize >= neededSize):

719

self.__rdBasicHeader()

720

self.__rdBasicHeader()

720

return 1

721

return 1

721

722

if self.__waitNewBlock():

723

if self.__waitNewBlock():

723

return 1

724

return 1

724

725

if not(self.setNextFile()):

726

if not(self.setNextFile()):

726

return 0

727

return 0

727

728

deltaTime = self.basicHeaderObj.utc - self.lastUTTime #

729

deltaTime = self.basicHeaderObj.utc - self.lastUTTime #

729

730

self.flagTimeBlock = 0

731

self.flagTimeBlock = 0

731

732

if deltaTime > self.maxTimeStep:

733

if deltaTime > self.maxTimeStep:

733

self.flagTimeBlock = 1

734

self.flagTimeBlock = 1

734

735

return 1

736

return 1

736

737

738

def readNextBlock(self):

739

def readNextBlock(self):

739

if not(self.__setNewBlock()):

740

if not(self.__setNewBlock()):

740

return 0

741

return 0

741

742

if not(self.readBlock()):

743

if not(self.readBlock()):

743

return 0

744

return 0

744

745

return 1

746

return 1

746

747

def __rdProcessingHeader(self, fp=None):

748

def __rdProcessingHeader(self, fp=None):

748

if fp == None:

749

if fp == None:

749

fp = self.fp

750

fp = self.fp

750

751

self.processingHeaderObj.read(fp)

752

self.processingHeaderObj.read(fp)

752

753

def __rdRadarControllerHeader(self, fp=None):

754

def __rdRadarControllerHeader(self, fp=None):

754

if fp == None:

755

if fp == None:

755

fp = self.fp

756

fp = self.fp

756

757

self.radarControllerHeaderObj.read(fp)

758

self.radarControllerHeaderObj.read(fp)

758

759

def __rdSystemHeader(self, fp=None):

760

def __rdSystemHeader(self, fp=None):

760

if fp == None:

761

if fp == None:

761

fp = self.fp

762

fp = self.fp

762

763

self.systemHeaderObj.read(fp)

764

self.systemHeaderObj.read(fp)

764

765

def __rdBasicHeader(self, fp=None):

766

def __rdBasicHeader(self, fp=None):

766

if fp == None:

767

if fp == None:

767

fp = self.fp

768

fp = self.fp

768

769

self.basicHeaderObj.read(fp)

770

self.basicHeaderObj.read(fp)

770

771

772

def __readFirstHeader(self):

773

def __readFirstHeader(self):

773

self.__rdBasicHeader()

774

self.__rdBasicHeader()

774

self.__rdSystemHeader()

775

self.__rdSystemHeader()

775

self.__rdRadarControllerHeader()

776

self.__rdRadarControllerHeader()

776

self.__rdProcessingHeader()

777

self.__rdProcessingHeader()

777

778

self.firstHeaderSize = self.basicHeaderObj.size

779

self.firstHeaderSize = self.basicHeaderObj.size

779

780

datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

781

datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

781

if datatype == 0:

782

if datatype == 0:

782

datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])

783

datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])

783

elif datatype == 1:

784

elif datatype == 1:

784

datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])

785

datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])

785

elif datatype == 2:

786

elif datatype == 2:

786

datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])

787

datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])

787

elif datatype == 3:

788

elif datatype == 3:

788

datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])

789

datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])

789

elif datatype == 4:

790

elif datatype == 4:

790

datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])

791

datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])

791

elif datatype == 5:

792

elif datatype == 5:

792

datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])

793

datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])

793

else:

794

else:

794

raise ValueError, 'Data type was not defined'

795

raise ValueError, 'Data type was not defined'

795

796

self.dtype = datatype_str

797

self.dtype = datatype_str

797

self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c

798

self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c

798

self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)

799

self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)

799

# self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)

800

# self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)

800

# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)

801

# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)

801

self.getBlockDimension()

802

self.getBlockDimension()

802

803

804

def __verifyFile(self, filename, msgFlag=True):

805

def __verifyFile(self, filename, msgFlag=True):

805

msg = None

806

msg = None

806

try:

807

try:

807

fp = open(filename, 'rb')

808

fp = open(filename, 'rb')

808

currentPosition = fp.tell()

809

currentPosition = fp.tell()

809

except:

810

except:

810

if msgFlag:

811

if msgFlag:

811

print "The file %s can't be opened" % (filename)

812

print "The file %s can't be opened" % (filename)

812

return False

813

return False

813

814

neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize

815

neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize

815

816

if neededSize == 0:

817

if neededSize == 0:

817

basicHeaderObj = BasicHeader(LOCALTIME)

818

basicHeaderObj = BasicHeader(LOCALTIME)

818

systemHeaderObj = SystemHeader()

819

systemHeaderObj = SystemHeader()

819

radarControllerHeaderObj = RadarControllerHeader()

820

radarControllerHeaderObj = RadarControllerHeader()

820

processingHeaderObj = ProcessingHeader()

821

processingHeaderObj = ProcessingHeader()

821

822

try:

823

try:

823

if not( basicHeaderObj.read(fp) ): raise IOError

824

if not( basicHeaderObj.read(fp) ): raise IOError

824

if not( systemHeaderObj.read(fp) ): raise IOError

825

if not( systemHeaderObj.read(fp) ): raise IOError

825

if not( radarControllerHeaderObj.read(fp) ): raise IOError

826

if not( radarControllerHeaderObj.read(fp) ): raise IOError

826

if not( processingHeaderObj.read(fp) ): raise IOError

827

if not( processingHeaderObj.read(fp) ): raise IOError

827

data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

828

data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

828

829

neededSize = processingHeaderObj.blockSize + basicHeaderObj.size

830

neededSize = processingHeaderObj.blockSize + basicHeaderObj.size

830

831

except:

832

except:

832

if msgFlag:

833

if msgFlag:

833

print "\tThe file %s is empty or it hasn't enough data" % filename

834

print "\tThe file %s is empty or it hasn't enough data" % filename

834

835

fp.close()

836

fp.close()

836

return False

837

return False

837

else:

838

else:

838

msg = "\tSkipping the file %s due to it hasn't enough data" %filename

839

msg = "\tSkipping the file %s due to it hasn't enough data" %filename

839

840

fp.close()

841

fp.close()

841

fileSize = os.path.getsize(filename)

842

fileSize = os.path.getsize(filename)

842

currentSize = fileSize - currentPosition

843

currentSize = fileSize - currentPosition

843

if currentSize < neededSize:

844

if currentSize < neededSize:

844

if msgFlag and (msg != None):

845

if msgFlag and (msg != None):

845

print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename

846

print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename

846

return False

847

return False

847

848

return True

849

return True

849

850

def setup(self,

851

def setup(self,

851

path=None,

852

path=None,

852

startDate=None,

853

startDate=None,

853

endDate=None,

854

endDate=None,

854

startTime=datetime.time(0,0,0),

855

startTime=datetime.time(0,0,0),

855

endTime=datetime.time(23,59,59),

856

endTime=datetime.time(23,59,59),

856

set=0,

857

set=0,

857

expLabel = "",

858

expLabel = "",

858

ext = None,

859

ext = None,

859

online = False,

860

online = False,

860

delay = 60,

861

delay = 60,

861

walk = True):

862

walk = True):

862

863

if path == None:

864

if path == None:

864

raise ValueError, "The path is not valid"

865

raise ValueError, "The path is not valid"

865

866

if ext == None:

867

if ext == None:

867

ext = self.ext

868

ext = self.ext

868

869

if online:

870

if online:

870

print "Searching files in online mode..."

871

print "Searching files in online mode..."

871

872

for nTries in range( self.nTries ):

873

for nTries in range( self.nTries ):

873

fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk)

874

fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk)

874

875

if fullpath:

876

if fullpath:

876

break

877

break

877

878

print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)

879

print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)

879

time.sleep( self.delay )

880

time.sleep( self.delay )

880

881

if not(fullpath):

882

if not(fullpath):

882

print "There 'isn't valied files in %s" % path

883

print "There 'isn't valied files in %s" % path

883

return None

884

return None

884

885

self.year = year

886

self.year = year

886

self.doy = doy

887

self.doy = doy

887

self.set = set - 1

888

self.set = set - 1

888

self.path = path

889

self.path = path

889

self.foldercounter = foldercounter

890

self.foldercounter = foldercounter

890

891

else:

892

else:

892

print "Searching files in offline mode ..."

893

print "Searching files in offline mode ..."

893

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

894

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

894

startTime=startTime, endTime=endTime,

895

startTime=startTime, endTime=endTime,

895

set=set, expLabel=expLabel, ext=ext,

896

set=set, expLabel=expLabel, ext=ext,

896

walk=walk)

897

walk=walk)

897

898

if not(pathList):

899

if not(pathList):

899

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

900

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

900

datetime.datetime.combine(startDate,startTime).ctime(),

901

datetime.datetime.combine(startDate,startTime).ctime(),

901

datetime.datetime.combine(endDate,endTime).ctime())

902

datetime.datetime.combine(endDate,endTime).ctime())

902

903

sys.exit(-1)

904

sys.exit(-1)

904

905

906

self.fileIndex = -1

907

self.fileIndex = -1

907

self.pathList = pathList

908

self.pathList = pathList

908

self.filenameList = filenameList

909

self.filenameList = filenameList

909

910

self.online = online

911

self.online = online

911

self.delay = delay

912

self.delay = delay

912

ext = ext.lower()

913

ext = ext.lower()

913

self.ext = ext

914

self.ext = ext

914

915

if not(self.setNextFile()):

916

if not(self.setNextFile()):

916

if (startDate!=None) and (endDate!=None):

917

if (startDate!=None) and (endDate!=None):

917

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

918

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

918

elif startDate != None:

919

elif startDate != None:

919

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

920

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

920

else:

921

else:

921

print "No files"

922

print "No files"

922

923

sys.exit(-1)

924

sys.exit(-1)

924

925

# self.updateDataHeader()

926

# self.updateDataHeader()

926

927

return self.dataOut

928

return self.dataOut

928

929

def getBasicHeader(self):

930

def getBasicHeader(self):

930

931

self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds

932

self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds

932

933

self.dataOut.flagTimeBlock = self.flagTimeBlock

934

self.dataOut.flagTimeBlock = self.flagTimeBlock

934

935

self.dataOut.timeZone = self.basicHeaderObj.timeZone

936

self.dataOut.timeZone = self.basicHeaderObj.timeZone

936

937

self.dataOut.dstFlag = self.basicHeaderObj.dstFlag

938

self.dataOut.dstFlag = self.basicHeaderObj.dstFlag

938

939

self.dataOut.errorCount = self.basicHeaderObj.errorCount

940

self.dataOut.errorCount = self.basicHeaderObj.errorCount

940

941

self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime

942

self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime

942

943

def getFirstHeader(self):

944

def getFirstHeader(self):

944

945

raise ValueError, "This method has not been implemented"

946

raise ValueError, "This method has not been implemented"

946

947

def getData():

948

def getData():

948

949

raise ValueError, "This method has not been implemented"

950

raise ValueError, "This method has not been implemented"

950

951

def hasNotDataInBuffer():

952

def hasNotDataInBuffer():

952

953

raise ValueError, "This method has not been implemented"

954

raise ValueError, "This method has not been implemented"

954

955

def readBlock():

956

def readBlock():

956

957

raise ValueError, "This method has not been implemented"

958

raise ValueError, "This method has not been implemented"

958

959

def isEndProcess(self):

960

def isEndProcess(self):

960

961

return self.flagNoMoreFiles

962

return self.flagNoMoreFiles

962

963

def printReadBlocks(self):

964

def printReadBlocks(self):

964

965

print "Number of read blocks per file %04d" %self.nReadBlocks

966

print "Number of read blocks per file %04d" %self.nReadBlocks

966

967

def printTotalBlocks(self):

968

def printTotalBlocks(self):

968

969

print "Number of read blocks %04d" %self.nTotalBlocks

970

print "Number of read blocks %04d" %self.nTotalBlocks

970

971

def printNumberOfBlock(self):

972

def printNumberOfBlock(self):

972

973

if self.flagIsNewBlock:

974

if self.flagIsNewBlock:

974

print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())

975

print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())

975

976

def printInfo(self):

977

def printInfo(self):

977

978

if self.__printInfo == False:

979

if self.__printInfo == False:

979

return

980

return

980

981

self.basicHeaderObj.printInfo()

982

self.basicHeaderObj.printInfo()

982

self.systemHeaderObj.printInfo()

983

self.systemHeaderObj.printInfo()

983

self.radarControllerHeaderObj.printInfo()

984

self.radarControllerHeaderObj.printInfo()

984

self.processingHeaderObj.printInfo()

985

self.processingHeaderObj.printInfo()

985

986

self.__printInfo = False

987

self.__printInfo = False

987

988

989

def run(self, **kwargs):

990

def run(self, **kwargs):

990

991

if not(self.isConfig):

992

if not(self.isConfig):

992

993

# self.dataOut = dataOut

994

# self.dataOut = dataOut

994

self.setup(**kwargs)

995

self.setup(**kwargs)

995

self.isConfig = True

996

self.isConfig = True

996

997

self.getData()

998

self.getData()

998

999

class JRODataWriter(JRODataIO, Operation):

1000

class JRODataWriter(JRODataIO, Operation):

1000

1001

"""

1002

"""

1002

Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura

1003

Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura

1003

de los datos siempre se realiza por bloques.

1004

de los datos siempre se realiza por bloques.

1004

"""

1005

"""

1005

1006

blockIndex = 0

1007

blockIndex = 0

1007

1008

path = None

1009

path = None

1009

1010

setFile = None

1011

setFile = None

1011

1012

profilesPerBlock = None

1013

profilesPerBlock = None

1013

1014

blocksPerFile = None

1015

blocksPerFile = None

1015

1016

nWriteBlocks = 0

1017

nWriteBlocks = 0

1017

1018

def __init__(self, dataOut=None):

1019

def __init__(self, dataOut=None):

1019

raise ValueError, "Not implemented"

1020

raise ValueError, "Not implemented"

1020

1021

1022

def hasAllDataInBuffer(self):

1023

def hasAllDataInBuffer(self):

1023

raise ValueError, "Not implemented"

1024

raise ValueError, "Not implemented"

1024

1025

1026

def setBlockDimension(self):

1027

def setBlockDimension(self):

1027

raise ValueError, "Not implemented"

1028

raise ValueError, "Not implemented"

1028

1029

1030

def writeBlock(self):

1031

def writeBlock(self):

1031

raise ValueError, "No implemented"

1032

raise ValueError, "No implemented"

1032

1033

1034

def putData(self):

1035

def putData(self):

1035

raise ValueError, "No implemented"

1036

raise ValueError, "No implemented"

1036

1037

1038

def setBasicHeader(self):

1039

def setBasicHeader(self):

1039

1040

self.basicHeaderObj.size = self.basicHeaderSize #bytes

1041

self.basicHeaderObj.size = self.basicHeaderSize #bytes

1041

self.basicHeaderObj.version = self.versionFile

1042

self.basicHeaderObj.version = self.versionFile

1042

self.basicHeaderObj.dataBlock = self.nTotalBlocks

1043

self.basicHeaderObj.dataBlock = self.nTotalBlocks

1043

1044

utc = numpy.floor(self.dataOut.utctime)

1045

utc = numpy.floor(self.dataOut.utctime)

1045

milisecond = (self.dataOut.utctime - utc)* 1000.0

1046

milisecond = (self.dataOut.utctime - utc)* 1000.0

1046

1047

self.basicHeaderObj.utc = utc

1048

self.basicHeaderObj.utc = utc

1048

self.basicHeaderObj.miliSecond = milisecond

1049

self.basicHeaderObj.miliSecond = milisecond

1049

self.basicHeaderObj.timeZone = self.dataOut.timeZone

1050

self.basicHeaderObj.timeZone = self.dataOut.timeZone

1050

self.basicHeaderObj.dstFlag = self.dataOut.dstFlag

1051

self.basicHeaderObj.dstFlag = self.dataOut.dstFlag

1051

self.basicHeaderObj.errorCount = self.dataOut.errorCount

1052

self.basicHeaderObj.errorCount = self.dataOut.errorCount

1052

1053

def setFirstHeader(self):

1054

def setFirstHeader(self):

1054

"""

1055

"""

1055

Obtiene una copia del First Header

1056

Obtiene una copia del First Header

1056

1057

Affected:

1058

Affected:

1058

1059

self.basicHeaderObj

1060

self.basicHeaderObj

1060

self.systemHeaderObj

1061

self.systemHeaderObj

1061

self.radarControllerHeaderObj

1062

self.radarControllerHeaderObj

1062

self.processingHeaderObj self.

1063

self.processingHeaderObj self.

1063

1064

Return:

1065

Return:

1065

None

1066

None

1066

"""

1067

"""

1067

1068

raise ValueError, "No implemented"

1069

raise ValueError, "No implemented"

1069

1070

def __writeFirstHeader(self):

1071

def __writeFirstHeader(self):

1071

"""

1072

"""

1072

Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)

1073

Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)

1073

1074

Affected:

1075

Affected:

1075

__dataType

1076

__dataType

1076

1077

Return:

1078

Return:

1078

None

1079

None

1079

"""

1080

"""

1080

1081

# CALCULAR PARAMETROS

1082

# CALCULAR PARAMETROS

1082

1083

sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size

1084

sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size

1084

self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader

1085

self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader

1085

1086

self.basicHeaderObj.write(self.fp)

1087

self.basicHeaderObj.write(self.fp)

1087

self.systemHeaderObj.write(self.fp)

1088

self.systemHeaderObj.write(self.fp)

1088

self.radarControllerHeaderObj.write(self.fp)

1089

self.radarControllerHeaderObj.write(self.fp)

1089

self.processingHeaderObj.write(self.fp)

1090

self.processingHeaderObj.write(self.fp)

1090

1091

self.dtype = self.dataOut.dtype

1092

self.dtype = self.dataOut.dtype

1092

1093

def __setNewBlock(self):

1094

def __setNewBlock(self):

1094

"""

1095

"""

1095

Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header

1096

Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header

1096

1097

Return:

1098

Return:

1098

0 : si no pudo escribir nada

1099

0 : si no pudo escribir nada

1099

1 : Si escribio el Basic el First Header

1100

1 : Si escribio el Basic el First Header

1100

"""

1101

"""

1101

if self.fp == None:

1102

if self.fp == None:

1102

self.setNextFile()

1103

self.setNextFile()

1103

1104

if self.flagIsNewFile:

1105

if self.flagIsNewFile:

1105

return 1

1106

return 1

1106

1107

if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:

1108

if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:

1108

self.basicHeaderObj.write(self.fp)

1109

self.basicHeaderObj.write(self.fp)

1109

return 1

1110

return 1

1110

1111

if not( self.setNextFile() ):

1112

if not( self.setNextFile() ):

1112

return 0

1113

return 0

1113

1114

return 1

1115

return 1

1115

1116

1117

def writeNextBlock(self):

1118

def writeNextBlock(self):

1118

"""

1119

"""

1119

Selecciona el bloque siguiente de datos y los escribe en un file

1120

Selecciona el bloque siguiente de datos y los escribe en un file

1120

1121

Return:

1122

Return:

1122

0 : Si no hizo pudo escribir el bloque de datos

1123

0 : Si no hizo pudo escribir el bloque de datos

1123

1 : Si no pudo escribir el bloque de datos

1124

1 : Si no pudo escribir el bloque de datos

1124

"""

1125

"""

1125

if not( self.__setNewBlock() ):

1126

if not( self.__setNewBlock() ):

1126

return 0

1127

return 0

1127

1128

self.writeBlock()

1129

self.writeBlock()

1129

1130

return 1

1131

return 1

1131

1132

def setNextFile(self):

1133

def setNextFile(self):

1133

"""

1134

"""

1134

Determina el siguiente file que sera escrito

1135

Determina el siguiente file que sera escrito

1135

1136

Affected:

1137

Affected:

1137

self.filename

1138

self.filename

1138

self.subfolder

1139

self.subfolder

1139

self.fp

1140

self.fp

1140

self.setFile

1141

self.setFile

1141

self.flagIsNewFile

1142

self.flagIsNewFile

1142

1143

Return:

1144

Return:

1144

0 : Si el archivo no puede ser escrito

1145

0 : Si el archivo no puede ser escrito

1145

1 : Si el archivo esta listo para ser escrito

1146

1 : Si el archivo esta listo para ser escrito

1146

"""

1147

"""

1147

ext = self.ext

1148

ext = self.ext

1148

path = self.path

1149

path = self.path

1149

1150

if self.fp != None:

1151

if self.fp != None:

1151

self.fp.close()

1152

self.fp.close()

1152

1153

timeTuple = time.localtime( self.dataOut.utctime)

1154

timeTuple = time.localtime( self.dataOut.utctime)

1154

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

1155

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

1155

1156

fullpath = os.path.join( path, subfolder )

1157

fullpath = os.path.join( path, subfolder )

1157

if not( os.path.exists(fullpath) ):

1158

if not( os.path.exists(fullpath) ):

1158

os.mkdir(fullpath)

1159

os.mkdir(fullpath)

1159

self.setFile = -1 #inicializo mi contador de seteo

1160

self.setFile = -1 #inicializo mi contador de seteo

1160

else:

1161

else:

1161

filesList = os.listdir( fullpath )

1162

filesList = os.listdir( fullpath )

1162

if len( filesList ) > 0:

1163

if len( filesList ) > 0:

1163

filesList = sorted( filesList, key=str.lower )

1164

filesList = sorted( filesList, key=str.lower )

1164

filen = filesList[-1]

1165

filen = filesList[-1]

1165

# el filename debera tener el siguiente formato

1166

# el filename debera tener el siguiente formato

1166

# 0 1234 567 89A BCDE (hex)

1167

# 0 1234 567 89A BCDE (hex)

1167

# x YYYY DDD SSS .ext

1168

# x YYYY DDD SSS .ext

1168

if isNumber( filen[8:11] ):

1169

if isNumber( filen[8:11] ):

1169

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

1170

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

1170

else:

1171

else:

1171

self.setFile = -1

1172

self.setFile = -1

1172

else:

1173

else:

1173

self.setFile = -1 #inicializo mi contador de seteo

1174

self.setFile = -1 #inicializo mi contador de seteo

1174

1175

setFile = self.setFile

1176

setFile = self.setFile

1176

setFile += 1

1177

setFile += 1

1177

1178

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

1179

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

1179

timeTuple.tm_year,

1180

timeTuple.tm_year,

1180

timeTuple.tm_yday,

1181

timeTuple.tm_yday,

1181

setFile,

1182

setFile,

1182

ext )

1183

ext )

1183

1184

filename = os.path.join( path, subfolder, file )

1185

filename = os.path.join( path, subfolder, file )

1185

1186

fp = open( filename,'wb' )

1187

fp = open( filename,'wb' )

1187

1188

self.blockIndex = 0

1189

self.blockIndex = 0

1189

1190

#guardando atributos

1191

#guardando atributos

1191

self.filename = filename

1192

self.filename = filename

1192

self.subfolder = subfolder

1193

self.subfolder = subfolder

1193

self.fp = fp

1194

self.fp = fp

1194

self.setFile = setFile

1195

self.setFile = setFile

1195

self.flagIsNewFile = 1

1196

self.flagIsNewFile = 1

1196

1197

self.setFirstHeader()

1198

self.setFirstHeader()

1198

1199

print 'Writing the file: %s'%self.filename

1200

print 'Writing the file: %s'%self.filename

1200

1201

self.__writeFirstHeader()

1202

self.__writeFirstHeader()

1202

1203

return 1

1204

return 1

1204

1205

def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=None, set=0, ext=None):

1206

def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=None, set=0, ext=None):

1206

"""

1207

"""

1207

Setea el tipo de formato en la cual sera guardada la data y escribe el First Header

1208

Setea el tipo de formato en la cual sera guardada la data y escribe el First Header

1208

1209

Inputs:

1210

Inputs:

1210

path : el path destino en el cual se escribiran los files a crear

1211

path : el path destino en el cual se escribiran los files a crear

1211

format : formato en el cual sera salvado un file

1212

format : formato en el cual sera salvado un file

1212

set : el setebo del file

1213

set : el setebo del file

1213

1214

Return:

1215

Return:

1215

0 : Si no realizo un buen seteo

1216

0 : Si no realizo un buen seteo

1216

1 : Si realizo un buen seteo

1217

1 : Si realizo un buen seteo

1217

"""

1218

"""

1218

1219

if ext == None:

1220

if ext == None:

1220

ext = self.ext

1221

ext = self.ext

1221

1222

ext = ext.lower()

1223

ext = ext.lower()

1223

1224

self.ext = ext

1225

self.ext = ext

1225

1226

self.path = path

1227

self.path = path

1227

1228

self.setFile = set - 1

1229

self.setFile = set - 1

1229

1230

self.blocksPerFile = blocksPerFile

1231

self.blocksPerFile = blocksPerFile

1231

1232

self.profilesPerBlock = profilesPerBlock

1233

self.profilesPerBlock = profilesPerBlock

1233

1234

self.dataOut = dataOut

1235

self.dataOut = dataOut

1235

1236

if not(self.setNextFile()):

1237

if not(self.setNextFile()):

1237

print "There isn't a next file"

1238

print "There isn't a next file"

1238

return 0

1239

return 0

1239

1240

self.setBlockDimension()

1241

self.setBlockDimension()

1241

1242

return 1

1243

return 1

1243

1244

def run(self, dataOut, **kwargs):

1245

def run(self, dataOut, **kwargs):

1245

1246

if not(self.isConfig):

1247

if not(self.isConfig):

1247

1248

self.setup(dataOut, **kwargs)

1249

self.setup(dataOut, **kwargs)

1249

self.isConfig = True

1250

self.isConfig = True

1250

1251

self.putData()

1252

self.putData()

1252

1253

class VoltageReader(JRODataReader):

1254

class VoltageReader(JRODataReader):

1254

"""

1255

"""

1255

Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura

1256

Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura

1256

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:

1257

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:

1257

perfiles*alturas*canales) son almacenados en la variable "buffer".

1258

perfiles*alturas*canales) son almacenados en la variable "buffer".

1258

1259

perfiles * alturas * canales

1260

perfiles * alturas * canales

1260

1261

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1262

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1262

RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la

1263

RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la

1263

cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de

1264

cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de

1264

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1265

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1265

1266

Example:

1267

Example:

1267

1268

dpath = "/home/myuser/data"

1269

dpath = "/home/myuser/data"

1269

1270

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1271

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1271

1272

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1273

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1273

1274

readerObj = VoltageReader()

1275

readerObj = VoltageReader()

1275

1276

readerObj.setup(dpath, startTime, endTime)

1277

readerObj.setup(dpath, startTime, endTime)

1277

1278

while(True):

1279

while(True):

1279

1280

#to get one profile

1281

#to get one profile

1281

profile = readerObj.getData()

1282

profile = readerObj.getData()

1282

1283

#print the profile

1284

#print the profile

1284

print profile

1285

print profile

1285

1286

#If you want to see all datablock

1287

#If you want to see all datablock

1287

print readerObj.datablock

1288

print readerObj.datablock

1288

1289

if readerObj.flagNoMoreFiles:

1290

if readerObj.flagNoMoreFiles:

1290

break

1291

break

1291

1292

"""

1293

"""

1293

1294

ext = ".r"

1295

ext = ".r"

1295

1296

optchar = "D"

1297

optchar = "D"

1297

dataOut = None

1298

dataOut = None

1298

1299

1300

def __init__(self):

1301

def __init__(self):

1301

"""

1302

"""

1302

Inicializador de la clase VoltageReader para la lectura de datos de voltage.

1303

Inicializador de la clase VoltageReader para la lectura de datos de voltage.

1303

1304

Input:

1305

Input:

1305

dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para

1306

dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para

1306

almacenar un perfil de datos cada vez que se haga un requerimiento

1307

almacenar un perfil de datos cada vez que se haga un requerimiento

1307

(getData). El perfil sera obtenido a partir del buffer de datos,

1308

(getData). El perfil sera obtenido a partir del buffer de datos,

1308

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1309

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1309

bloque de datos.

1310

bloque de datos.

1310

Si este parametro no es pasado se creara uno internamente.

1311

Si este parametro no es pasado se creara uno internamente.

1311

1312

Variables afectadas:

1313

Variables afectadas:

1313

self.dataOut

1314

self.dataOut

1314

1315

Return:

1316

Return:

1316

None

1317

None

1317

"""

1318

"""

1318

1319

self.isConfig = False

1320

self.isConfig = False

1320

1321

self.datablock = None

1322

self.datablock = None

1322

1323

self.utc = 0

1324

self.utc = 0

1324

1325

self.ext = ".r"

1326

self.ext = ".r"

1326

1327

self.optchar = "D"

1328

self.optchar = "D"

1328

1329

self.basicHeaderObj = BasicHeader(LOCALTIME)

1330

self.basicHeaderObj = BasicHeader(LOCALTIME)

1330

1331

self.systemHeaderObj = SystemHeader()

1332

self.systemHeaderObj = SystemHeader()

1332

1333

self.radarControllerHeaderObj = RadarControllerHeader()

1334

self.radarControllerHeaderObj = RadarControllerHeader()

1334

1335

self.processingHeaderObj = ProcessingHeader()

1336

self.processingHeaderObj = ProcessingHeader()

1336

1337

self.online = 0

1338

self.online = 0

1338

1339

self.fp = None

1340

self.fp = None

1340

1341

self.idFile = None

1342

self.idFile = None

1342

1343

self.dtype = None

1344

self.dtype = None

1344

1345

self.fileSizeByHeader = None

1346

self.fileSizeByHeader = None

1346

1347

self.filenameList = []

1348

self.filenameList = []

1348

1349

self.filename = None

1350

self.filename = None

1350

1351

self.fileSize = None

1352

self.fileSize = None

1352

1353

self.firstHeaderSize = 0

1354

self.firstHeaderSize = 0

1354

1355

self.basicHeaderSize = 24

1356

self.basicHeaderSize = 24

1356

1357

self.pathList = []

1358

self.pathList = []

1358

1359

self.filenameList = []

1360

self.filenameList = []

1360

1361

self.lastUTTime = 0

1362

self.lastUTTime = 0

1362

1363

self.maxTimeStep = 30

1364

self.maxTimeStep = 30

1364

1365

self.flagNoMoreFiles = 0

1366

self.flagNoMoreFiles = 0

1366

1367

self.set = 0

1368

self.set = 0

1368

1369

self.path = None

1370

self.path = None

1370

1371

self.profileIndex = 2**32-1

1372

self.profileIndex = 2**32-1

1372

1373

self.delay = 3 #seconds

1374

self.delay = 3 #seconds

1374

1375

self.nTries = 3 #quantity tries

1376

self.nTries = 3 #quantity tries

1376

1377

self.nFiles = 3 #number of files for searching

1378

self.nFiles = 3 #number of files for searching

1378

1379

self.nReadBlocks = 0

1380

self.nReadBlocks = 0

1380

1381

self.flagIsNewFile = 1

1382

self.flagIsNewFile = 1

1382

1383

self.__isFirstTimeOnline = 1

1384

self.__isFirstTimeOnline = 1

1384

1385

self.ippSeconds = 0

1386

self.ippSeconds = 0

1386

1387

self.flagTimeBlock = 0

1388

self.flagTimeBlock = 0

1388

1389

self.flagIsNewBlock = 0

1390

self.flagIsNewBlock = 0

1390

1391

self.nTotalBlocks = 0

1392

self.nTotalBlocks = 0

1392

1393

self.blocksize = 0

1394

self.blocksize = 0

1394

1395

self.dataOut = self.createObjByDefault()

1396

self.dataOut = self.createObjByDefault()

1396

1397

def createObjByDefault(self):

1398

def createObjByDefault(self):

1398

1399

dataObj = Voltage()

1400

dataObj = Voltage()

1400

1401

return dataObj

1402

return dataObj

1402

1403

def __hasNotDataInBuffer(self):

1404

def __hasNotDataInBuffer(self):

1404

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1405

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1405

return 1

1406

return 1

1406

return 0

1407

return 0

1407

1408

1409

def getBlockDimension(self):

1410

def getBlockDimension(self):

1410

"""

1411

"""

1411

Obtiene la cantidad de puntos a leer por cada bloque de datos

1412

Obtiene la cantidad de puntos a leer por cada bloque de datos

1412

1413

Affected:

1414

Affected:

1414

self.blocksize

1415

self.blocksize

1415

1416

Return:

1417

Return:

1417

None

1418

None

1418

"""

1419

"""

1419

pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels

1420

pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels

1420

self.blocksize = pts2read

1421

self.blocksize = pts2read

1421

1422

1423

def readBlock(self):

1424

def readBlock(self):

1424

"""

1425

"""

1425

readBlock lee el bloque de datos desde la posicion actual del puntero del archivo

1426

readBlock lee el bloque de datos desde la posicion actual del puntero del archivo

1426

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

1427

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

1427

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

1428

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

1428

es seteado a 0

1429

es seteado a 0

1429

1430

Inputs:

1431

Inputs:

1431

None

1432

None

1432

1433

Return:

1434

Return:

1434

None

1435

None

1435

1436

Affected:

1437

Affected:

1437

self.profileIndex

1438

self.profileIndex

1438

self.datablock

1439

self.datablock

1439

self.flagIsNewFile

1440

self.flagIsNewFile

1440

self.flagIsNewBlock

1441

self.flagIsNewBlock

1441

self.nTotalBlocks

1442

self.nTotalBlocks

1442

1443

Exceptions:

1444

Exceptions:

1444

Si un bloque leido no es un bloque valido

1445

Si un bloque leido no es un bloque valido

1445

"""

1446

"""

1446

1447

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1448

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1448

1449

try:

1450

try:

1450

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1451

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1451

except:

1452

except:

1452

print "The read block (%3d) has not enough data" %self.nReadBlocks

1453

print "The read block (%3d) has not enough data" %self.nReadBlocks

1453

return 0

1454

return 0

1454

1455

junk = numpy.transpose(junk, (2,0,1))

1456

junk = numpy.transpose(junk, (2,0,1))

1456

self.datablock = junk['real'] + junk['imag']*1j

1457

self.datablock = junk['real'] + junk['imag']*1j

1457

1458

self.profileIndex = 0

1459

self.profileIndex = 0

1459

1460

self.flagIsNewFile = 0

1461

self.flagIsNewFile = 0

1461

self.flagIsNewBlock = 1

1462

self.flagIsNewBlock = 1

1462

1463

self.nTotalBlocks += 1

1464

self.nTotalBlocks += 1

1464

self.nReadBlocks += 1

1465

self.nReadBlocks += 1

1465

1466

return 1

1467

return 1

1467

1468

def getFirstHeader(self):

1469

def getFirstHeader(self):

1469

1470

self.dataOut.dtype = self.dtype

1471

self.dataOut.dtype = self.dtype

1471

1472

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

1473

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

1473

1474

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

1475

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

1475

1476

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

1477

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

1477

1478

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

1479

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

1479

1480

self.dataOut.ippSeconds = self.ippSeconds

1481

self.dataOut.ippSeconds = self.ippSeconds

1481

1482

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt

1483

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt

1483

1484

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

1485

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

1485

1486

self.dataOut.flagShiftFFT = False

1487

self.dataOut.flagShiftFFT = False

1487

1488

if self.radarControllerHeaderObj.code != None:

1489

if self.radarControllerHeaderObj.code != None:

1489

1490

self.dataOut.nCode = self.radarControllerHeaderObj.nCode

1491

self.dataOut.nCode = self.radarControllerHeaderObj.nCode

1491

1492

self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud

1493

self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud

1493

1494

self.dataOut.code = self.radarControllerHeaderObj.code

1495

self.dataOut.code = self.radarControllerHeaderObj.code

1495

1496

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

1497

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

1497

1498

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

1499

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

1499

1500

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

1501

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

1501

1502

self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip

1503

self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip

1503

1504

self.dataOut.flagShiftFFT = False

1505

self.dataOut.flagShiftFFT = False

1505

1506

def getData(self):

1507

def getData(self):

1507

"""

1508

"""

1508

getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"

1509

getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"

1509

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

1510

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

1510

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

1511

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

1511

1512

Ademas incrementa el contador del buffer en 1.

1513

Ademas incrementa el contador del buffer en 1.

1513

1514

Return:

1515

Return:

1515

data : retorna un perfil de voltages (alturas * canales) copiados desde el

1516

data : retorna un perfil de voltages (alturas * canales) copiados desde el

1516

buffer. Si no hay mas archivos a leer retorna None.

1517

buffer. Si no hay mas archivos a leer retorna None.

1517

1518

Variables afectadas:

1519

Variables afectadas:

1519

self.dataOut

1520

self.dataOut

1520

self.profileIndex

1521

self.profileIndex

1521

1522

Affected:

1523

Affected:

1523

self.dataOut

1524

self.dataOut

1524

self.profileIndex

1525

self.profileIndex

1525

self.flagTimeBlock

1526

self.flagTimeBlock

1526

self.flagIsNewBlock

1527

self.flagIsNewBlock

1527

"""

1528

"""

1528

1529

if self.flagNoMoreFiles:

1530

if self.flagNoMoreFiles:

1530

self.dataOut.flagNoData = True

1531

self.dataOut.flagNoData = True

1531

print 'Process finished'

1532

print 'Process finished'

1532

return 0

1533

return 0

1533

1534

self.flagTimeBlock = 0

1535

self.flagTimeBlock = 0

1535

self.flagIsNewBlock = 0

1536

self.flagIsNewBlock = 0

1536

1537

if self.__hasNotDataInBuffer():

1538

if self.__hasNotDataInBuffer():

1538

1539

if not( self.readNextBlock() ):

1540

if not( self.readNextBlock() ):

1540

return 0

1541

return 0

1541

1542

self.getFirstHeader()

1543

self.getFirstHeader()

1543

1544

if self.datablock == None:

1545

if self.datablock == None:

1545

self.dataOut.flagNoData = True

1546

self.dataOut.flagNoData = True

1546

return 0

1547

return 0

1547

1548

self.dataOut.data = self.datablock[:,self.profileIndex,:]

1549

self.dataOut.data = self.datablock[:,self.profileIndex,:]

1549

1550

self.dataOut.flagNoData = False

1551

self.dataOut.flagNoData = False

1551

1552

self.getBasicHeader()

1553

self.getBasicHeader()

1553

1554

self.profileIndex += 1

1555

self.profileIndex += 1

1555

1556

return self.dataOut.data

1557

return self.dataOut.data

1557

1558

1559

class VoltageWriter(JRODataWriter):

1560

class VoltageWriter(JRODataWriter):

1560

"""

1561

"""

1561

Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura

1562

Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura

1562

de los datos siempre se realiza por bloques.

1563

de los datos siempre se realiza por bloques.

1563

"""

1564

"""

1564

1565

ext = ".r"

1566

ext = ".r"

1566

1567

optchar = "D"

1568

optchar = "D"

1568

1569

shapeBuffer = None

1570

shapeBuffer = None

1570

1571

1572

def __init__(self):

1573

def __init__(self):

1573

"""

1574

"""

1574

Inicializador de la clase VoltageWriter para la escritura de datos de espectros.

1575

Inicializador de la clase VoltageWriter para la escritura de datos de espectros.

1575

1576

Affected:

1577

Affected:

1577

self.dataOut

1578

self.dataOut

1578

1579

Return: None

1580

Return: None

1580

"""

1581

"""

1581

1582

self.nTotalBlocks = 0

1583

self.nTotalBlocks = 0

1583

1584

self.profileIndex = 0

1585

self.profileIndex = 0

1585

1586

self.isConfig = False

1587

self.isConfig = False

1587

1588

self.fp = None

1589

self.fp = None

1589

1590

self.flagIsNewFile = 1

1591

self.flagIsNewFile = 1

1591

1592

self.nTotalBlocks = 0

1593

self.nTotalBlocks = 0

1593

1594

self.flagIsNewBlock = 0

1595

self.flagIsNewBlock = 0

1595

1596

self.setFile = None

1597

self.setFile = None

1597

1598

self.dtype = None

1599

self.dtype = None

1599

1600

self.path = None

1601

self.path = None

1601

1602

self.filename = None

1603

self.filename = None

1603

1604

self.basicHeaderObj = BasicHeader(LOCALTIME)

1605

self.basicHeaderObj = BasicHeader(LOCALTIME)

1605

1606

self.systemHeaderObj = SystemHeader()

1607

self.systemHeaderObj = SystemHeader()

1607

1608

self.radarControllerHeaderObj = RadarControllerHeader()

1609

self.radarControllerHeaderObj = RadarControllerHeader()

1609

1610

self.processingHeaderObj = ProcessingHeader()

1611

self.processingHeaderObj = ProcessingHeader()

1611

1612

def hasAllDataInBuffer(self):

1613

def hasAllDataInBuffer(self):

1613

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1614

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1614

return 1

1615

return 1

1615

return 0

1616

return 0

1616

1617

1618

def setBlockDimension(self):

1619

def setBlockDimension(self):

1619

"""

1620

"""

1620

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

1621

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

1621

1622

Affected:

1623

Affected:

1623

self.shape_spc_Buffer

1624

self.shape_spc_Buffer

1624

self.shape_cspc_Buffer

1625

self.shape_cspc_Buffer

1625

self.shape_dc_Buffer

1626

self.shape_dc_Buffer

1626

1627

Return: None

1628

Return: None

1628

"""

1629

"""

1629

self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,

1630

self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,

1630

self.processingHeaderObj.nHeights,

1631

self.processingHeaderObj.nHeights,

1631

self.systemHeaderObj.nChannels)

1632

self.systemHeaderObj.nChannels)

1632

1633

self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,

1634

self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,

1634

self.processingHeaderObj.profilesPerBlock,

1635

self.processingHeaderObj.profilesPerBlock,

1635

self.processingHeaderObj.nHeights),

1636

self.processingHeaderObj.nHeights),

1636

dtype=numpy.dtype('complex64'))

1637

dtype=numpy.dtype('complex64'))

1637

1638

1639

def writeBlock(self):

1640

def writeBlock(self):

1640

"""

1641

"""

1641

Escribe el buffer en el file designado

1642

Escribe el buffer en el file designado

1642

1643

Affected:

1644

Affected:

1644

self.profileIndex

1645

self.profileIndex

1645

self.flagIsNewFile

1646

self.flagIsNewFile

1646

self.flagIsNewBlock

1647

self.flagIsNewBlock

1647

self.nTotalBlocks

1648

self.nTotalBlocks

1648

self.blockIndex

1649

self.blockIndex

1649

1650

Return: None

1651

Return: None

1651

"""

1652

"""

1652

data = numpy.zeros( self.shapeBuffer, self.dtype )

1653

data = numpy.zeros( self.shapeBuffer, self.dtype )

1653

1654

junk = numpy.transpose(self.datablock, (1,2,0))

1655

junk = numpy.transpose(self.datablock, (1,2,0))

1655

1656

data['real'] = junk.real

1657

data['real'] = junk.real

1657

data['imag'] = junk.imag

1658

data['imag'] = junk.imag

1658

1659

data = data.reshape( (-1) )

1660

data = data.reshape( (-1) )

1660

1661

data.tofile( self.fp )

1662

data.tofile( self.fp )

1662

1663

self.datablock.fill(0)

1664

self.datablock.fill(0)

1664

1665

self.profileIndex = 0

1666

self.profileIndex = 0

1666

self.flagIsNewFile = 0

1667

self.flagIsNewFile = 0

1667

self.flagIsNewBlock = 1

1668

self.flagIsNewBlock = 1

1668

1669

self.blockIndex += 1

1670

self.blockIndex += 1

1670

self.nTotalBlocks += 1

1671

self.nTotalBlocks += 1

1671

1672

def putData(self):

1673

def putData(self):

1673

"""

1674

"""

1674

Setea un bloque de datos y luego los escribe en un file

1675

Setea un bloque de datos y luego los escribe en un file

1675

1676

Affected:

1677

Affected:

1677

self.flagIsNewBlock

1678

self.flagIsNewBlock

1678

self.profileIndex

1679

self.profileIndex

1679

1680

Return:

1681

Return:

1681

0 : Si no hay data o no hay mas files que puedan escribirse

1682

0 : Si no hay data o no hay mas files que puedan escribirse

1682

1 : Si se escribio la data de un bloque en un file

1683

1 : Si se escribio la data de un bloque en un file

1683

"""

1684

"""

1684

if self.dataOut.flagNoData:

1685

if self.dataOut.flagNoData:

1685

return 0

1686

return 0

1686

1687

self.flagIsNewBlock = 0

1688

self.flagIsNewBlock = 0

1688

1689

if self.dataOut.flagTimeBlock:

1690

if self.dataOut.flagTimeBlock:

1690

1691

self.datablock.fill(0)

1692

self.datablock.fill(0)

1692

self.profileIndex = 0

1693

self.profileIndex = 0

1693

self.setNextFile()

1694

self.setNextFile()

1694

1695

if self.profileIndex == 0:

1696

if self.profileIndex == 0:

1696

self.setBasicHeader()

1697

self.setBasicHeader()

1697

1698

self.datablock[:,self.profileIndex,:] = self.dataOut.data

1699

self.datablock[:,self.profileIndex,:] = self.dataOut.data

1699

1700

self.profileIndex += 1

1701

self.profileIndex += 1

1701

1702

if self.hasAllDataInBuffer():

1703

if self.hasAllDataInBuffer():

1703

#if self.flagIsNewFile:

1704

#if self.flagIsNewFile:

1704

self.writeNextBlock()

1705

self.writeNextBlock()

1705

# self.setFirstHeader()

1706

# self.setFirstHeader()

1706

1707

return 1

1708

return 1

1708

1709

def __getProcessFlags(self):

1710

def __getProcessFlags(self):

1710

1711

processFlags = 0

1712

processFlags = 0

1712

1713

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1714

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1714

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1715

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1715

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1716

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1716

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1717

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1717

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1718

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1718

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1719

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1719

1720

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1721

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1721

1722

1723

1724

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

1725

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

1725

PROCFLAG.DATATYPE_SHORT,

1726

PROCFLAG.DATATYPE_SHORT,

1726

PROCFLAG.DATATYPE_LONG,

1727

PROCFLAG.DATATYPE_LONG,

1727

PROCFLAG.DATATYPE_INT64,

1728

PROCFLAG.DATATYPE_INT64,

1728

PROCFLAG.DATATYPE_FLOAT,

1729

PROCFLAG.DATATYPE_FLOAT,

1729

PROCFLAG.DATATYPE_DOUBLE]

1730

PROCFLAG.DATATYPE_DOUBLE]

1730

1731

1732

for index in range(len(dtypeList)):

1733

for index in range(len(dtypeList)):

1733

if self.dataOut.dtype == dtypeList[index]:

1734

if self.dataOut.dtype == dtypeList[index]:

1734

dtypeValue = datatypeValueList[index]

1735

dtypeValue = datatypeValueList[index]

1735

break

1736

break

1736

1737

processFlags += dtypeValue

1738

processFlags += dtypeValue

1738

1739

if self.dataOut.flagDecodeData:

1740

if self.dataOut.flagDecodeData:

1740

processFlags += PROCFLAG.DECODE_DATA

1741

processFlags += PROCFLAG.DECODE_DATA

1741

1742

if self.dataOut.flagDeflipData:

1743

if self.dataOut.flagDeflipData:

1743

processFlags += PROCFLAG.DEFLIP_DATA

1744

processFlags += PROCFLAG.DEFLIP_DATA

1744

1745

if self.dataOut.code != None:

1746

if self.dataOut.code != None:

1746

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

1747

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

1747

1748

if self.dataOut.nCohInt > 1:

1749

if self.dataOut.nCohInt > 1:

1749

processFlags += PROCFLAG.COHERENT_INTEGRATION

1750

processFlags += PROCFLAG.COHERENT_INTEGRATION

1750

1751

return processFlags

1752

return processFlags

1752

1753

1754

def __getBlockSize(self):

1755

def __getBlockSize(self):

1755

'''

1756

'''

1756

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage

1757

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage

1757

'''

1758

'''

1758

1759

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1760

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1760

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1761

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1761

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1762

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1762

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1763

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1763

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1764

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1764

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1765

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1765

1766

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1767

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1767

datatypeValueList = [1,2,4,8,4,8]

1768

datatypeValueList = [1,2,4,8,4,8]

1768

for index in range(len(dtypeList)):

1769

for index in range(len(dtypeList)):

1769

if self.dataOut.dtype == dtypeList[index]:

1770

if self.dataOut.dtype == dtypeList[index]:

1770

datatypeValue = datatypeValueList[index]

1771

datatypeValue = datatypeValueList[index]

1771

break

1772

break

1772

1773

blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.dataOut.nProfiles * datatypeValue * 2)

1774

blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.dataOut.nProfiles * datatypeValue * 2)

1774

1775

return blocksize

1776

return blocksize

1776

1777

def setFirstHeader(self):

1778

def setFirstHeader(self):

1778

1779

"""

1780

"""

1780

Obtiene una copia del First Header

1781

Obtiene una copia del First Header

1781

1782

Affected:

1783

Affected:

1783

self.systemHeaderObj

1784

self.systemHeaderObj

1784

self.radarControllerHeaderObj

1785

self.radarControllerHeaderObj

1785

self.dtype

1786

self.dtype

1786

1787

Return:

1788

Return:

1788

None

1789

None

1789

"""

1790

"""

1790

1791

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

1792

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

1792

self.systemHeaderObj.nChannels = self.dataOut.nChannels

1793

self.systemHeaderObj.nChannels = self.dataOut.nChannels

1793

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

1794

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

1794

1795

self.setBasicHeader()

1796

self.setBasicHeader()

1796

1797

processingHeaderSize = 40 # bytes

1798

processingHeaderSize = 40 # bytes

1798

self.processingHeaderObj.dtype = 0 # Voltage

1799

self.processingHeaderObj.dtype = 0 # Voltage

1799

self.processingHeaderObj.blockSize = self.__getBlockSize()

1800

self.processingHeaderObj.blockSize = self.__getBlockSize()

1800

self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock

1801

self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock

1801

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

1802

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

1802

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

1803

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

1803

self.processingHeaderObj.processFlags = self.__getProcessFlags()

1804

self.processingHeaderObj.processFlags = self.__getProcessFlags()

1804

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt

1805

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt

1805

self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage

1806

self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage

1806

self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage

1807

self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage

1807

1808

if self.dataOut.code != None:

1809

if self.dataOut.code != None:

1809

self.processingHeaderObj.code = self.dataOut.code

1810

self.processingHeaderObj.code = self.dataOut.code

1810

self.processingHeaderObj.nCode = self.dataOut.nCode

1811

self.processingHeaderObj.nCode = self.dataOut.nCode

1811

self.processingHeaderObj.nBaud = self.dataOut.nBaud

1812

self.processingHeaderObj.nBaud = self.dataOut.nBaud

1812

codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)

1813

codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)

1813

processingHeaderSize += codesize

1814

processingHeaderSize += codesize

1814

1815

if self.processingHeaderObj.nWindows != 0:

1816

if self.processingHeaderObj.nWindows != 0:

1816

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

1817

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

1817

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

1818

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

1818

self.processingHeaderObj.nHeights = self.dataOut.nHeights

1819

self.processingHeaderObj.nHeights = self.dataOut.nHeights

1819

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

1820

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

1820

processingHeaderSize += 12

1821

processingHeaderSize += 12

1821

1822

self.processingHeaderObj.size = processingHeaderSize

1823

self.processingHeaderObj.size = processingHeaderSize

1823

1824

class SpectraReader(JRODataReader):

1825

class SpectraReader(JRODataReader):

1825

"""

1826

"""

1826

Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura

1827

Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura

1827

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)

1828

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)

1828

son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.

1829

son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.

1829

1830

paresCanalesIguales * alturas * perfiles (Self Spectra)

1831

paresCanalesIguales * alturas * perfiles (Self Spectra)

1831

paresCanalesDiferentes * alturas * perfiles (Cross Spectra)

1832

paresCanalesDiferentes * alturas * perfiles (Cross Spectra)

1832

canales * alturas (DC Channels)

1833

canales * alturas (DC Channels)

1833

1834

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1835

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1835

RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la

1836

RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la

1836

cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de

1837

cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de

1837

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1838

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1838

1839

Example:

1840

Example:

1840

dpath = "/home/myuser/data"

1841

dpath = "/home/myuser/data"

1841

1842

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1843

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1843

1844

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1845

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1845

1846

readerObj = SpectraReader()

1847

readerObj = SpectraReader()

1847

1848

readerObj.setup(dpath, startTime, endTime)

1849

readerObj.setup(dpath, startTime, endTime)

1849

1850

while(True):

1851

while(True):

1851

1852

readerObj.getData()

1853

readerObj.getData()

1853

1854

print readerObj.data_spc

1855

print readerObj.data_spc

1855

1856

print readerObj.data_cspc

1857

print readerObj.data_cspc

1857

1858

print readerObj.data_dc

1859

print readerObj.data_dc

1859

1860

if readerObj.flagNoMoreFiles:

1861

if readerObj.flagNoMoreFiles:

1861

break

1862

break

1862

1863

"""

1864

"""

1864

1865

pts2read_SelfSpectra = 0

1866

pts2read_SelfSpectra = 0

1866

1867

pts2read_CrossSpectra = 0

1868

pts2read_CrossSpectra = 0

1868

1869

pts2read_DCchannels = 0

1870

pts2read_DCchannels = 0

1870

1871

ext = ".pdata"

1872

ext = ".pdata"

1872

1873

optchar = "P"

1874

optchar = "P"

1874

1875

dataOut = None

1876

dataOut = None

1876

1877

nRdChannels = None

1878

nRdChannels = None

1878

1879

nRdPairs = None

1880

nRdPairs = None

1880

1881

rdPairList = []

1882

rdPairList = []

1882

1883

def __init__(self):

1884

def __init__(self):

1884

"""

1885

"""

1885

Inicializador de la clase SpectraReader para la lectura de datos de espectros.

1886

Inicializador de la clase SpectraReader para la lectura de datos de espectros.

1886

1887

Inputs:

1888

Inputs:

1888

dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para

1889

dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para

1889

almacenar un perfil de datos cada vez que se haga un requerimiento

1890

almacenar un perfil de datos cada vez que se haga un requerimiento

1890

(getData). El perfil sera obtenido a partir del buffer de datos,

1891

(getData). El perfil sera obtenido a partir del buffer de datos,

1891

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1892

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1892

bloque de datos.

1893

bloque de datos.

1893

Si este parametro no es pasado se creara uno internamente.

1894

Si este parametro no es pasado se creara uno internamente.

1894

1895

Affected:

1896

Affected:

1896

self.dataOut

1897

self.dataOut

1897

1898

Return : None

1899

Return : None

1899

"""

1900

"""

1900

1901

self.isConfig = False

1902

self.isConfig = False

1902

1903

self.pts2read_SelfSpectra = 0

1904

self.pts2read_SelfSpectra = 0

1904

1905

self.pts2read_CrossSpectra = 0

1906

self.pts2read_CrossSpectra = 0

1906

1907

self.pts2read_DCchannels = 0

1908

self.pts2read_DCchannels = 0

1908

1909

self.datablock = None

1910

self.datablock = None

1910

1911

self.utc = None

1912

self.utc = None

1912

1913

self.ext = ".pdata"

1914

self.ext = ".pdata"

1914

1915

self.optchar = "P"

1916

self.optchar = "P"

1916

1917

self.basicHeaderObj = BasicHeader(LOCALTIME)

1918

self.basicHeaderObj = BasicHeader(LOCALTIME)

1918

1919

self.systemHeaderObj = SystemHeader()

1920

self.systemHeaderObj = SystemHeader()

1920

1921

self.radarControllerHeaderObj = RadarControllerHeader()

1922

self.radarControllerHeaderObj = RadarControllerHeader()

1922

1923

self.processingHeaderObj = ProcessingHeader()

1924

self.processingHeaderObj = ProcessingHeader()

1924

1925

self.online = 0

1926

self.online = 0

1926

1927

self.fp = None

1928

self.fp = None

1928

1929

self.idFile = None

1930

self.idFile = None

1930

1931

self.dtype = None

1932

self.dtype = None

1932

1933

self.fileSizeByHeader = None

1934

self.fileSizeByHeader = None

1934

1935

self.filenameList = []

1936

self.filenameList = []

1936

1937

self.filename = None

1938

self.filename = None

1938

1939

self.fileSize = None

1940

self.fileSize = None

1940

1941

self.firstHeaderSize = 0

1942

self.firstHeaderSize = 0

1942

1943

self.basicHeaderSize = 24

1944

self.basicHeaderSize = 24

1944

1945

self.pathList = []

1946

self.pathList = []

1946

1947

self.lastUTTime = 0

1948

self.lastUTTime = 0

1948

1949

self.maxTimeStep = 30

1950

self.maxTimeStep = 30

1950

1951

self.flagNoMoreFiles = 0

1952

self.flagNoMoreFiles = 0

1952

1953

self.set = 0

1954

self.set = 0

1954

1955

self.path = None

1956

self.path = None

1956

1957

self.delay = 60 #seconds

1958

self.delay = 60 #seconds

1958

1959

self.nTries = 3 #quantity tries

1960

self.nTries = 3 #quantity tries

1960

1961

self.nFiles = 3 #number of files for searching

1962

self.nFiles = 3 #number of files for searching

1962

1963

self.nReadBlocks = 0

1964

self.nReadBlocks = 0

1964

1965

self.flagIsNewFile = 1

1966

self.flagIsNewFile = 1

1966

1967

self.__isFirstTimeOnline = 1

1968

self.__isFirstTimeOnline = 1

1968

1969

self.ippSeconds = 0

1970

self.ippSeconds = 0

1970

1971

self.flagTimeBlock = 0

1972

self.flagTimeBlock = 0

1972

1973

self.flagIsNewBlock = 0

1974

self.flagIsNewBlock = 0

1974

1975

self.nTotalBlocks = 0

1976

self.nTotalBlocks = 0

1976

1977

self.blocksize = 0

1978

self.blocksize = 0

1978

1979

self.dataOut = self.createObjByDefault()

1980

self.dataOut = self.createObjByDefault()

1980

1981

self.profileIndex = 1 #Always

1982

self.profileIndex = 1 #Always

1982

1983

1984

def createObjByDefault(self):

1985

def createObjByDefault(self):

1985

1986

dataObj = Spectra()

1987

dataObj = Spectra()

1987

1988

return dataObj

1989

return dataObj

1989

1990

def __hasNotDataInBuffer(self):

1991

def __hasNotDataInBuffer(self):

1991

return 1

1992

return 1

1992

1993

1994

def getBlockDimension(self):

1995

def getBlockDimension(self):

1995

"""

1996

"""

1996

Obtiene la cantidad de puntos a leer por cada bloque de datos

1997

Obtiene la cantidad de puntos a leer por cada bloque de datos

1997

1998

Affected:

1999

Affected:

1999

self.nRdChannels

2000

self.nRdChannels

2000

self.nRdPairs

2001

self.nRdPairs

2001

self.pts2read_SelfSpectra

2002

self.pts2read_SelfSpectra

2002

self.pts2read_CrossSpectra

2003

self.pts2read_CrossSpectra

2003

self.pts2read_DCchannels

2004

self.pts2read_DCchannels

2004

self.blocksize

2005

self.blocksize

2005

self.dataOut.nChannels

2006

self.dataOut.nChannels

2006

self.dataOut.nPairs

2007

self.dataOut.nPairs

2007

2008

Return:

2009

Return:

2009

None

2010

None

2010

"""

2011

"""

2011

self.nRdChannels = 0

2012

self.nRdChannels = 0

2012

self.nRdPairs = 0

2013

self.nRdPairs = 0

2013

self.rdPairList = []

2014

self.rdPairList = []

2014

2015

for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):

2016

for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):

2016

if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:

2017

if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:

2017

self.nRdChannels = self.nRdChannels + 1 #par de canales iguales

2018

self.nRdChannels = self.nRdChannels + 1 #par de canales iguales

2018

else:

2019

else:

2019

self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes

2020

self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes

2020

self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))

2021

self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))

2021

2022

pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock

2023

pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock

2023

2024

self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)

2025

self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)

2025

self.blocksize = self.pts2read_SelfSpectra

2026

self.blocksize = self.pts2read_SelfSpectra

2026

2027

if self.processingHeaderObj.flag_cspc:

2028

if self.processingHeaderObj.flag_cspc:

2028

self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)

2029

self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)

2029

self.blocksize += self.pts2read_CrossSpectra

2030

self.blocksize += self.pts2read_CrossSpectra

2030

2031

if self.processingHeaderObj.flag_dc:

2032

if self.processingHeaderObj.flag_dc:

2032

self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)

2033

self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)

2033

self.blocksize += self.pts2read_DCchannels

2034

self.blocksize += self.pts2read_DCchannels

2034

2035

# self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels

2036

# self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels

2036

2037

2038

def readBlock(self):

2039

def readBlock(self):

2039

"""

2040

"""

2040

Lee el bloque de datos desde la posicion actual del puntero del archivo

2041

Lee el bloque de datos desde la posicion actual del puntero del archivo

2041

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

2042

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

2042

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

2043

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

2043

es seteado a 0

2044

es seteado a 0

2044

2045

Return: None

2046

Return: None

2046

2047

Variables afectadas:

2048

Variables afectadas:

2048

2049

self.flagIsNewFile

2050

self.flagIsNewFile

2050

self.flagIsNewBlock

2051

self.flagIsNewBlock

2051

self.nTotalBlocks

2052

self.nTotalBlocks

2052

self.data_spc

2053

self.data_spc

2053

self.data_cspc

2054

self.data_cspc

2054

self.data_dc

2055

self.data_dc

2055

2056

Exceptions:

2057

Exceptions:

2057

Si un bloque leido no es un bloque valido

2058

Si un bloque leido no es un bloque valido

2058

"""

2059

"""

2059

blockOk_flag = False

2060

blockOk_flag = False

2060

fpointer = self.fp.tell()

2061

fpointer = self.fp.tell()

2061

2062

spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )

2063

spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )

2063

spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2064

spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2064

2065

if self.processingHeaderObj.flag_cspc:

2066

if self.processingHeaderObj.flag_cspc:

2066

cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )

2067

cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )

2067

cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2068

cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2068

2069

if self.processingHeaderObj.flag_dc:

2070

if self.processingHeaderObj.flag_dc:

2070

dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )

2071

dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )

2071

dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D

2072

dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D

2072

2073

2074

if not(self.processingHeaderObj.shif_fft):

2075

if not(self.processingHeaderObj.shif_fft):

2075

#desplaza a la derecha en el eje 2 determinadas posiciones

2076

#desplaza a la derecha en el eje 2 determinadas posiciones

2076

shift = int(self.processingHeaderObj.profilesPerBlock/2)

2077

shift = int(self.processingHeaderObj.profilesPerBlock/2)

2077

spc = numpy.roll( spc, shift , axis=2 )

2078

spc = numpy.roll( spc, shift , axis=2 )

2078

2079

if self.processingHeaderObj.flag_cspc:

2080

if self.processingHeaderObj.flag_cspc:

2080

#desplaza a la derecha en el eje 2 determinadas posiciones

2081

#desplaza a la derecha en el eje 2 determinadas posiciones

2081

cspc = numpy.roll( cspc, shift, axis=2 )

2082

cspc = numpy.roll( cspc, shift, axis=2 )

2082

2083

# self.processingHeaderObj.shif_fft = True

2084

# self.processingHeaderObj.shif_fft = True

2084

2085

spc = numpy.transpose( spc, (0,2,1) )

2086

spc = numpy.transpose( spc, (0,2,1) )

2086

self.data_spc = spc

2087

self.data_spc = spc

2087

2088

if self.processingHeaderObj.flag_cspc:

2089

if self.processingHeaderObj.flag_cspc:

2089

cspc = numpy.transpose( cspc, (0,2,1) )

2090

cspc = numpy.transpose( cspc, (0,2,1) )

2090

self.data_cspc = cspc['real'] + cspc['imag']*1j

2091

self.data_cspc = cspc['real'] + cspc['imag']*1j

2091

else:

2092

else:

2092

self.data_cspc = None

2093

self.data_cspc = None

2093

2094

if self.processingHeaderObj.flag_dc:

2095

if self.processingHeaderObj.flag_dc:

2095

self.data_dc = dc['real'] + dc['imag']*1j

2096

self.data_dc = dc['real'] + dc['imag']*1j

2096

else:

2097

else:

2097

self.data_dc = None

2098

self.data_dc = None

2098

2099

self.flagIsNewFile = 0

2100

self.flagIsNewFile = 0

2100

self.flagIsNewBlock = 1

2101

self.flagIsNewBlock = 1

2101

2102

self.nTotalBlocks += 1

2103

self.nTotalBlocks += 1

2103

self.nReadBlocks += 1

2104

self.nReadBlocks += 1

2104

2105

return 1

2106

return 1

2106

2107

def getFirstHeader(self):

2108

def getFirstHeader(self):

2108

2109

self.dataOut.dtype = self.dtype

2110

self.dataOut.dtype = self.dtype

2110

2111

self.dataOut.nPairs = self.nRdPairs

2112

self.dataOut.nPairs = self.nRdPairs

2112

2113

self.dataOut.pairsList = self.rdPairList

2114

self.dataOut.pairsList = self.rdPairList

2114

2115

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

2116

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

2116

2117

self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock

2118

self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock

2118

2119

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

2120

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

2120

2121

self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt

2122

self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt

2122

2123

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

2124

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

2124

2125

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

2126

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

2126

2127

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

2128

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

2128

2129

self.dataOut.ippSeconds = self.ippSeconds

2130

self.dataOut.ippSeconds = self.ippSeconds

2130

2131

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints

2132

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints

2132

2133

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

2134

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

2134

2135

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

2136

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

2136

2137

self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft

2138

self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft

2138

2139

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

2140

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

2140

2141

self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip

2142

self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip

2142

2143

if self.processingHeaderObj.code != None:

2144

if self.processingHeaderObj.code != None:

2144

2145

self.dataOut.nCode = self.processingHeaderObj.nCode

2146

self.dataOut.nCode = self.processingHeaderObj.nCode

2146

2147

self.dataOut.nBaud = self.processingHeaderObj.nBaud

2148

self.dataOut.nBaud = self.processingHeaderObj.nBaud

2148

2149

self.dataOut.code = self.processingHeaderObj.code

2150

self.dataOut.code = self.processingHeaderObj.code

2150

2151

self.dataOut.flagDecodeData = True

2152

self.dataOut.flagDecodeData = True

2152

2153

def getData(self):

2154

def getData(self):

2154

"""

2155

"""

2155

Copia el buffer de lectura a la clase "Spectra",

2156

Copia el buffer de lectura a la clase "Spectra",

2156

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

2157

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

2157

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

2158

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

2158

2159

Return:

2160

Return:

2160

0 : Si no hay mas archivos disponibles

2161

0 : Si no hay mas archivos disponibles

2161

1 : Si hizo una buena copia del buffer

2162

1 : Si hizo una buena copia del buffer

2162

2163

Affected:

2164

Affected:

2164

self.dataOut

2165

self.dataOut

2165

2166

self.flagTimeBlock

2167

self.flagTimeBlock

2167

self.flagIsNewBlock

2168

self.flagIsNewBlock

2168

"""

2169

"""

2169

2170

if self.flagNoMoreFiles:

2171

if self.flagNoMoreFiles:

2171

self.dataOut.flagNoData = True

2172

self.dataOut.flagNoData = True

2172

print 'Process finished'

2173

print 'Process finished'

2173

return 0

2174

return 0

2174

2175

self.flagTimeBlock = 0

2176

self.flagTimeBlock = 0

2176

self.flagIsNewBlock = 0

2177

self.flagIsNewBlock = 0

2177

2178

if self.__hasNotDataInBuffer():

2179

if self.__hasNotDataInBuffer():

2179

2180

if not( self.readNextBlock() ):

2181

if not( self.readNextBlock() ):

2181

self.dataOut.flagNoData = True

2182

self.dataOut.flagNoData = True

2182

return 0

2183

return 0

2183

2184

#data es un numpy array de 3 dmensiones (perfiles, alturas y canales)

2185

#data es un numpy array de 3 dmensiones (perfiles, alturas y canales)

2185

2186

if self.data_dc == None:

2187

if self.data_dc == None:

2187

self.dataOut.flagNoData = True

2188

self.dataOut.flagNoData = True

2188

return 0

2189

return 0

2189

2190

self.getBasicHeader()

2191

self.getBasicHeader()

2191

2192

self.getFirstHeader()

2193

self.getFirstHeader()

2193

2194

self.dataOut.data_spc = self.data_spc

2195

self.dataOut.data_spc = self.data_spc

2195

2196

self.dataOut.data_cspc = self.data_cspc

2197

self.dataOut.data_cspc = self.data_cspc

2197

2198

self.dataOut.data_dc = self.data_dc

2199

self.dataOut.data_dc = self.data_dc

2199

2200

self.dataOut.flagNoData = False

2201

self.dataOut.flagNoData = False

2201

2202

return self.dataOut.data_spc

2203

return self.dataOut.data_spc

2203

2204

2205

class SpectraWriter(JRODataWriter):

2206

class SpectraWriter(JRODataWriter):

2206

2207

"""

2208

"""

2208

Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura

2209

Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura

2209

de los datos siempre se realiza por bloques.

2210

de los datos siempre se realiza por bloques.

2210

"""

2211

"""

2211

2212

ext = ".pdata"

2213

ext = ".pdata"

2213

2214

optchar = "P"

2215

optchar = "P"

2215

2216

shape_spc_Buffer = None

2217

shape_spc_Buffer = None

2217

2218

shape_cspc_Buffer = None

2219

shape_cspc_Buffer = None

2219

2220

shape_dc_Buffer = None

2221

shape_dc_Buffer = None

2221

2222

data_spc = None

2223

data_spc = None

2223

2224

data_cspc = None

2225

data_cspc = None

2225

2226

data_dc = None

2227

data_dc = None

2227

2228

# dataOut = None

2229

# dataOut = None

2229

2230

def __init__(self):

2231

def __init__(self):

2231

"""

2232

"""

2232

Inicializador de la clase SpectraWriter para la escritura de datos de espectros.

2233

Inicializador de la clase SpectraWriter para la escritura de datos de espectros.

2233

2234

Affected:

2235

Affected:

2235

self.dataOut

2236

self.dataOut

2236

self.basicHeaderObj

2237

self.basicHeaderObj

2237

self.systemHeaderObj

2238

self.systemHeaderObj

2238

self.radarControllerHeaderObj

2239

self.radarControllerHeaderObj

2239

self.processingHeaderObj

2240

self.processingHeaderObj

2240

2241

Return: None

2242

Return: None

2242

"""

2243

"""

2243

2244

self.isConfig = False

2245

self.isConfig = False

2245

2246

self.nTotalBlocks = 0

2247

self.nTotalBlocks = 0

2247

2248

self.data_spc = None

2249

self.data_spc = None

2249

2250

self.data_cspc = None

2251

self.data_cspc = None

2251

2252

self.data_dc = None

2253

self.data_dc = None

2253

2254

self.fp = None

2255

self.fp = None

2255

2256

self.flagIsNewFile = 1

2257

self.flagIsNewFile = 1

2257

2258

self.nTotalBlocks = 0

2259

self.nTotalBlocks = 0

2259

2260

self.flagIsNewBlock = 0

2261

self.flagIsNewBlock = 0

2261

2262

self.setFile = None

2263

self.setFile = None

2263

2264

self.dtype = None

2265

self.dtype = None

2265

2266

self.path = None

2267

self.path = None

2267

2268

self.noMoreFiles = 0

2269

self.noMoreFiles = 0

2269

2270

self.filename = None

2271

self.filename = None

2271

2272

self.basicHeaderObj = BasicHeader(LOCALTIME)

2273

self.basicHeaderObj = BasicHeader(LOCALTIME)

2273

2274

self.systemHeaderObj = SystemHeader()

2275

self.systemHeaderObj = SystemHeader()

2275

2276

self.radarControllerHeaderObj = RadarControllerHeader()

2277

self.radarControllerHeaderObj = RadarControllerHeader()

2277

2278

self.processingHeaderObj = ProcessingHeader()

2279

self.processingHeaderObj = ProcessingHeader()

2279

2280

2281

def hasAllDataInBuffer(self):

2282

def hasAllDataInBuffer(self):

2282

return 1

2283

return 1

2283

2284

2285

def setBlockDimension(self):

2286

def setBlockDimension(self):

2286

"""

2287

"""

2287

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

2288

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

2288

2289

Affected:

2290

Affected:

2290

self.shape_spc_Buffer

2291

self.shape_spc_Buffer

2291

self.shape_cspc_Buffer

2292

self.shape_cspc_Buffer

2292

self.shape_dc_Buffer

2293

self.shape_dc_Buffer

2293

2294

Return: None

2295

Return: None

2295

"""

2296

"""

2296

self.shape_spc_Buffer = (self.dataOut.nChannels,

2297

self.shape_spc_Buffer = (self.dataOut.nChannels,

2297

self.processingHeaderObj.nHeights,

2298

self.processingHeaderObj.nHeights,

2298

self.processingHeaderObj.profilesPerBlock)

2299

self.processingHeaderObj.profilesPerBlock)

2299

2300

self.shape_cspc_Buffer = (self.dataOut.nPairs,

2301

self.shape_cspc_Buffer = (self.dataOut.nPairs,

2301

self.processingHeaderObj.nHeights,

2302

self.processingHeaderObj.nHeights,

2302

self.processingHeaderObj.profilesPerBlock)

2303

self.processingHeaderObj.profilesPerBlock)

2303

2304

self.shape_dc_Buffer = (self.dataOut.nChannels,

2305

self.shape_dc_Buffer = (self.dataOut.nChannels,

2305

self.processingHeaderObj.nHeights)

2306

self.processingHeaderObj.nHeights)

2306

2307

2308

def writeBlock(self):

2309

def writeBlock(self):

2309

"""

2310

"""

2310

Escribe el buffer en el file designado

2311

Escribe el buffer en el file designado

2311

2312

Affected:

2313

Affected:

2313

self.data_spc

2314

self.data_spc

2314

self.data_cspc

2315

self.data_cspc

2315

self.data_dc

2316

self.data_dc

2316

self.flagIsNewFile

2317

self.flagIsNewFile

2317

self.flagIsNewBlock

2318

self.flagIsNewBlock

2318

self.nTotalBlocks

2319

self.nTotalBlocks

2319

self.nWriteBlocks

2320

self.nWriteBlocks

2320

2321

Return: None

2322

Return: None

2322

"""

2323

"""

2323

2324

spc = numpy.transpose( self.data_spc, (0,2,1) )

2325

spc = numpy.transpose( self.data_spc, (0,2,1) )

2325

if not( self.processingHeaderObj.shif_fft ):

2326

if not( self.processingHeaderObj.shif_fft ):

2326

spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2327

spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2327

data = spc.reshape((-1))

2328

data = spc.reshape((-1))

2328

data = data.astype(self.dtype[0])

2329

data = data.astype(self.dtype[0])

2329

data.tofile(self.fp)

2330

data.tofile(self.fp)

2330

2331

if self.data_cspc != None:

2332

if self.data_cspc != None:

2332

data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )

2333

data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )

2333

cspc = numpy.transpose( self.data_cspc, (0,2,1) )

2334

cspc = numpy.transpose( self.data_cspc, (0,2,1) )

2334

if not( self.processingHeaderObj.shif_fft ):

2335

if not( self.processingHeaderObj.shif_fft ):

2335

cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2336

cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2336

data['real'] = cspc.real

2337

data['real'] = cspc.real

2337

data['imag'] = cspc.imag

2338

data['imag'] = cspc.imag

2338

data = data.reshape((-1))

2339

data = data.reshape((-1))

2339

data.tofile(self.fp)

2340

data.tofile(self.fp)

2340

2341

if self.data_dc != None:

2342

if self.data_dc != None:

2342

data = numpy.zeros( self.shape_dc_Buffer, self.dtype )

2343

data = numpy.zeros( self.shape_dc_Buffer, self.dtype )

2343

dc = self.data_dc

2344

dc = self.data_dc

2344

data['real'] = dc.real

2345

data['real'] = dc.real

2345

data['imag'] = dc.imag

2346

data['imag'] = dc.imag

2346

data = data.reshape((-1))

2347

data = data.reshape((-1))

2347

data.tofile(self.fp)

2348

data.tofile(self.fp)

2348

2349

self.data_spc.fill(0)

2350

self.data_spc.fill(0)

2350

self.data_dc.fill(0)

2351

self.data_dc.fill(0)

2351

if self.data_cspc != None:

2352

if self.data_cspc != None:

2352

self.data_cspc.fill(0)

2353

self.data_cspc.fill(0)

2353

2354

self.flagIsNewFile = 0

2355

self.flagIsNewFile = 0

2355

self.flagIsNewBlock = 1

2356

self.flagIsNewBlock = 1

2356

self.nTotalBlocks += 1

2357

self.nTotalBlocks += 1

2357

self.nWriteBlocks += 1

2358

self.nWriteBlocks += 1

2358

self.blockIndex += 1

2359

self.blockIndex += 1

2359

2360

2361

def putData(self):

2362

def putData(self):

2362

"""

2363

"""

2363

Setea un bloque de datos y luego los escribe en un file

2364

Setea un bloque de datos y luego los escribe en un file

2364

2365

Affected:

2366

Affected:

2366

self.data_spc

2367

self.data_spc

2367

self.data_cspc

2368

self.data_cspc

2368

self.data_dc

2369

self.data_dc

2369

2370

Return:

2371

Return:

2371

0 : Si no hay data o no hay mas files que puedan escribirse

2372

0 : Si no hay data o no hay mas files que puedan escribirse

2372

1 : Si se escribio la data de un bloque en un file

2373

1 : Si se escribio la data de un bloque en un file

2373

"""

2374

"""

2374

2375

if self.dataOut.flagNoData:

2376

if self.dataOut.flagNoData:

2376

return 0

2377

return 0

2377

2378

self.flagIsNewBlock = 0

2379

self.flagIsNewBlock = 0

2379

2380

if self.dataOut.flagTimeBlock:

2381

if self.dataOut.flagTimeBlock:

2381

self.data_spc.fill(0)

2382

self.data_spc.fill(0)

2382

self.data_cspc.fill(0)

2383

self.data_cspc.fill(0)

2383

self.data_dc.fill(0)

2384

self.data_dc.fill(0)

2384

self.setNextFile()

2385

self.setNextFile()

2385

2386

if self.flagIsNewFile == 0:

2387

if self.flagIsNewFile == 0:

2387

self.setBasicHeader()

2388

self.setBasicHeader()

2388

2389

self.data_spc = self.dataOut.data_spc.copy()

2390

self.data_spc = self.dataOut.data_spc.copy()

2390

self.data_cspc = self.dataOut.data_cspc.copy()

2391

self.data_cspc = self.dataOut.data_cspc.copy()

2391

self.data_dc = self.dataOut.data_dc.copy()

2392

self.data_dc = self.dataOut.data_dc.copy()

2392

2393

# #self.processingHeaderObj.dataBlocksPerFile)

2394

# #self.processingHeaderObj.dataBlocksPerFile)

2394

if self.hasAllDataInBuffer():

2395

if self.hasAllDataInBuffer():

2395

# self.setFirstHeader()

2396

# self.setFirstHeader()

2396

self.writeNextBlock()

2397

self.writeNextBlock()

2397

2398

return 1

2399

return 1

2399

2400

2401

def __getProcessFlags(self):

2402

def __getProcessFlags(self):

2402

2403

processFlags = 0

2404

processFlags = 0

2404

2405

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2406

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2406

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2407

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2407

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2408

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2408

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2409

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2409

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2410

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2410

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2411

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2411

2412

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2413

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2413

2414

2415

2416

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

2417

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

2417

PROCFLAG.DATATYPE_SHORT,

2418

PROCFLAG.DATATYPE_SHORT,

2418

PROCFLAG.DATATYPE_LONG,

2419

PROCFLAG.DATATYPE_LONG,

2419

PROCFLAG.DATATYPE_INT64,

2420

PROCFLAG.DATATYPE_INT64,

2420

PROCFLAG.DATATYPE_FLOAT,

2421

PROCFLAG.DATATYPE_FLOAT,

2421

PROCFLAG.DATATYPE_DOUBLE]

2422

PROCFLAG.DATATYPE_DOUBLE]

2422

2423

2424

for index in range(len(dtypeList)):

2425

for index in range(len(dtypeList)):

2425

if self.dataOut.dtype == dtypeList[index]:

2426

if self.dataOut.dtype == dtypeList[index]:

2426

dtypeValue = datatypeValueList[index]

2427

dtypeValue = datatypeValueList[index]

2427

break

2428

break

2428

2429

processFlags += dtypeValue

2430

processFlags += dtypeValue

2430

2431

if self.dataOut.flagDecodeData:

2432

if self.dataOut.flagDecodeData:

2432

processFlags += PROCFLAG.DECODE_DATA

2433

processFlags += PROCFLAG.DECODE_DATA

2433

2434

if self.dataOut.flagDeflipData:

2435

if self.dataOut.flagDeflipData:

2435

processFlags += PROCFLAG.DEFLIP_DATA

2436

processFlags += PROCFLAG.DEFLIP_DATA

2436

2437

if self.dataOut.code != None:

2438

if self.dataOut.code != None:

2438

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

2439

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

2439

2440

if self.dataOut.nIncohInt > 1:

2441

if self.dataOut.nIncohInt > 1:

2441

processFlags += PROCFLAG.INCOHERENT_INTEGRATION

2442

processFlags += PROCFLAG.INCOHERENT_INTEGRATION

2442

2443

if self.dataOut.data_dc != None:

2444

if self.dataOut.data_dc != None:

2444

processFlags += PROCFLAG.SAVE_CHANNELS_DC

2445

processFlags += PROCFLAG.SAVE_CHANNELS_DC

2445

2446

return processFlags

2447

return processFlags

2447

2448

2449

def __getBlockSize(self):

2450

def __getBlockSize(self):

2450

'''

2451

'''

2451

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra

2452

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra

2452

'''

2453

'''

2453

2454

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2455

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2455

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2456

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2456

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2457

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2457

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2458

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2458

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2459

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2459

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2460

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2460

2461

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2462

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2462

datatypeValueList = [1,2,4,8,4,8]

2463

datatypeValueList = [1,2,4,8,4,8]

2463

for index in range(len(dtypeList)):

2464

for index in range(len(dtypeList)):

2464

if self.dataOut.dtype == dtypeList[index]:

2465

if self.dataOut.dtype == dtypeList[index]:

2465

datatypeValue = datatypeValueList[index]

2466

datatypeValue = datatypeValueList[index]

2466

break

2467

break

2467

2468

2469

pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints

2470

pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints

2470

2471

pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)

2472

pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)

2472

blocksize = (pts2write_SelfSpectra*datatypeValue)

2473

blocksize = (pts2write_SelfSpectra*datatypeValue)

2473

2474

if self.dataOut.data_cspc != None:

2475

if self.dataOut.data_cspc != None:

2475

pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)

2476

pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)

2476

blocksize += (pts2write_CrossSpectra*datatypeValue*2)

2477

blocksize += (pts2write_CrossSpectra*datatypeValue*2)

2477

2478

if self.dataOut.data_dc != None:

2479

if self.dataOut.data_dc != None:

2479

pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)

2480

pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)

2480

blocksize += (pts2write_DCchannels*datatypeValue*2)

2481

blocksize += (pts2write_DCchannels*datatypeValue*2)

2481

2482

blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO

2483

blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO

2483

2484

return blocksize

2485

return blocksize

2485

2486

def setFirstHeader(self):

2487

def setFirstHeader(self):

2487

2488

"""

2489

"""

2489

Obtiene una copia del First Header

2490

Obtiene una copia del First Header

2490

2491

Affected:

2492

Affected:

2492

self.systemHeaderObj

2493

self.systemHeaderObj

2493

self.radarControllerHeaderObj

2494

self.radarControllerHeaderObj

2494

self.dtype

2495

self.dtype

2495

2496

Return:

2497

Return:

2497

None

2498

None

2498

"""

2499

"""

2499

2500

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

2501

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

2501

self.systemHeaderObj.nChannels = self.dataOut.nChannels

2502

self.systemHeaderObj.nChannels = self.dataOut.nChannels

2502

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

2503

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

2503

2504

self.setBasicHeader()

2505

self.setBasicHeader()

2505

2506

processingHeaderSize = 40 # bytes

2507

processingHeaderSize = 40 # bytes

2507

self.processingHeaderObj.dtype = 1 # Spectra

2508

self.processingHeaderObj.dtype = 1 # Spectra

2508

self.processingHeaderObj.blockSize = self.__getBlockSize()

2509

self.processingHeaderObj.blockSize = self.__getBlockSize()

2509

self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints

2510

self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints

2510

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

2511

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

2511

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

2512

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

2512

self.processingHeaderObj.processFlags = self.__getProcessFlags()

2513

self.processingHeaderObj.processFlags = self.__getProcessFlags()

2513

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval

2514

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval

2514

self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt

2515

self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt

2515

self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels

2516

self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels

2516

self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT

2517

self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT

2517

2518

if self.processingHeaderObj.totalSpectra > 0:

2519

if self.processingHeaderObj.totalSpectra > 0:

2519

channelList = []

2520

channelList = []

2520

for channel in range(self.dataOut.nChannels):

2521

for channel in range(self.dataOut.nChannels):

2521

channelList.append(channel)

2522

channelList.append(channel)

2522

channelList.append(channel)

2523

channelList.append(channel)

2523

2524

pairsList = []

2525

pairsList = []

2525

for pair in self.dataOut.pairsList:

2526

for pair in self.dataOut.pairsList:

2526

pairsList.append(pair[0])

2527

pairsList.append(pair[0])

2527

pairsList.append(pair[1])

2528

pairsList.append(pair[1])

2528

spectraComb = channelList + pairsList

2529

spectraComb = channelList + pairsList

2529

spectraComb = numpy.array(spectraComb,dtype="u1")

2530

spectraComb = numpy.array(spectraComb,dtype="u1")

2530

self.processingHeaderObj.spectraComb = spectraComb

2531

self.processingHeaderObj.spectraComb = spectraComb

2531

sizeOfSpcComb = len(spectraComb)

2532

sizeOfSpcComb = len(spectraComb)

2532

processingHeaderSize += sizeOfSpcComb

2533

processingHeaderSize += sizeOfSpcComb

2533

2534

# The processing header should not have information about code

2535

# The processing header should not have information about code

2535

# if self.dataOut.code != None:

2536

# if self.dataOut.code != None:

2536

# self.processingHeaderObj.code = self.dataOut.code

2537

# self.processingHeaderObj.code = self.dataOut.code

2537

# self.processingHeaderObj.nCode = self.dataOut.nCode

2538

# self.processingHeaderObj.nCode = self.dataOut.nCode

2538

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

2539

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

2539

# nCodeSize = 4 # bytes

2540

# nCodeSize = 4 # bytes

2540

# nBaudSize = 4 # bytes

2541

# nBaudSize = 4 # bytes

2541

# codeSize = 4 # bytes

2542

# codeSize = 4 # bytes

2542

# sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)

2543

# sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)

2543

# processingHeaderSize += sizeOfCode

2544

# processingHeaderSize += sizeOfCode

2544

2545

if self.processingHeaderObj.nWindows != 0:

2546

if self.processingHeaderObj.nWindows != 0:

2546

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

2547

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

2547

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2548

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2548

self.processingHeaderObj.nHeights = self.dataOut.nHeights

2549

self.processingHeaderObj.nHeights = self.dataOut.nHeights

2549

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

2550

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

2550

sizeOfFirstHeight = 4

2551

sizeOfFirstHeight = 4

2551

sizeOfdeltaHeight = 4

2552

sizeOfdeltaHeight = 4

2552

sizeOfnHeights = 4

2553

sizeOfnHeights = 4

2553

sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows

2554

sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows

2554

processingHeaderSize += sizeOfWindows

2555

processingHeaderSize += sizeOfWindows

2555

2556

self.processingHeaderObj.size = processingHeaderSize

2557

self.processingHeaderObj.size = processingHeaderSize

2557

2558

class SpectraHeisWriter(Operation):

2559

class SpectraHeisWriter(Operation):

2559

# set = None

2560

# set = None

2560

setFile = None

2561

setFile = None

2561

idblock = None

2562

idblock = None

2562

doypath = None

2563

doypath = None

2563

subfolder = None

2564

subfolder = None

2564

2565

def __init__(self):

2566

def __init__(self):

2566

self.wrObj = FITS()

2567

self.wrObj = FITS()

2567

# self.dataOut = dataOut

2568

# self.dataOut = dataOut

2568

self.nTotalBlocks=0

2569

self.nTotalBlocks=0

2569

# self.set = None

2570

# self.set = None

2570

self.setFile = None

2571

self.setFile = None

2571

self.idblock = 0

2572

self.idblock = 0

2572

self.wrpath = None

2573

self.wrpath = None

2573

self.doypath = None

2574

self.doypath = None

2574

self.subfolder = None

2575

self.subfolder = None

2575

self.isConfig = False

2576

self.isConfig = False

2576

2577

def isNumber(str):

2578

def isNumber(str):

2578

"""

2579

"""

2579

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

2580

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

2580

2581

Excepciones:

2582

Excepciones:

2582

Si un determinado string no puede ser convertido a numero

2583

Si un determinado string no puede ser convertido a numero

2583

Input:

2584

Input:

2584

str, string al cual se le analiza para determinar si convertible a un numero o no

2585

str, string al cual se le analiza para determinar si convertible a un numero o no

2585

2586

Return:

2587

Return:

2587

True : si el string es uno numerico

2588

True : si el string es uno numerico

2588

False : no es un string numerico

2589

False : no es un string numerico

2589

"""

2590

"""

2590

try:

2591

try:

2591

float( str )

2592

float( str )

2592

return True

2593

return True

2593

except:

2594

except:

2594

return False

2595

return False

2595

2596

def setup(self, dataOut, wrpath):

2597

def setup(self, dataOut, wrpath):

2597

2598

if not(os.path.exists(wrpath)):

2599

if not(os.path.exists(wrpath)):

2599

os.mkdir(wrpath)

2600

os.mkdir(wrpath)

2600

2601

self.wrpath = wrpath

2602

self.wrpath = wrpath

2602

# self.setFile = 0

2603

# self.setFile = 0

2603

self.dataOut = dataOut

2604

self.dataOut = dataOut

2604

2605

def putData(self):

2606

def putData(self):

2606

name= time.localtime( self.dataOut.utctime)

2607

name= time.localtime( self.dataOut.utctime)

2607

ext=".fits"

2608

ext=".fits"

2608

2609

if self.doypath == None:

2610

if self.doypath == None:

2610

self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))

2611

self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))

2611

self.doypath = os.path.join( self.wrpath, self.subfolder )

2612

self.doypath = os.path.join( self.wrpath, self.subfolder )

2612

os.mkdir(self.doypath)

2613

os.mkdir(self.doypath)

2613

2614

if self.setFile == None:

2615

if self.setFile == None:

2615

# self.set = self.dataOut.set

2616

# self.set = self.dataOut.set

2616

self.setFile = 0

2617

self.setFile = 0

2617

# if self.set != self.dataOut.set:

2618

# if self.set != self.dataOut.set:

2618

## self.set = self.dataOut.set

2619

## self.set = self.dataOut.set

2619

# self.setFile = 0

2620

# self.setFile = 0

2620

2621

#make the filename

2622

#make the filename

2622

file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)

2623

file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)

2623

2624

filename = os.path.join(self.wrpath,self.subfolder, file)

2625

filename = os.path.join(self.wrpath,self.subfolder, file)

2625

2626

idblock = numpy.array([self.idblock],dtype="int64")

2627

idblock = numpy.array([self.idblock],dtype="int64")

2627

header=self.wrObj.cFImage(idblock=idblock,

2628

header=self.wrObj.cFImage(idblock=idblock,

2628

year=time.gmtime(self.dataOut.utctime).tm_year,

2629

year=time.gmtime(self.dataOut.utctime).tm_year,

2629

month=time.gmtime(self.dataOut.utctime).tm_mon,

2630

month=time.gmtime(self.dataOut.utctime).tm_mon,

2630

day=time.gmtime(self.dataOut.utctime).tm_mday,

2631

day=time.gmtime(self.dataOut.utctime).tm_mday,

2631

hour=time.gmtime(self.dataOut.utctime).tm_hour,

2632

hour=time.gmtime(self.dataOut.utctime).tm_hour,

2632

minute=time.gmtime(self.dataOut.utctime).tm_min,

2633

minute=time.gmtime(self.dataOut.utctime).tm_min,

2633

second=time.gmtime(self.dataOut.utctime).tm_sec)

2634

second=time.gmtime(self.dataOut.utctime).tm_sec)

2634

2635

c=3E8

2636

c=3E8

2636

deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2637

deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2637

freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))

2638

freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))

2638

2639

colList = []

2640

colList = []

2640

2641

colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)

2642

colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)

2642

2643

colList.append(colFreq)

2644

colList.append(colFreq)

2644

2645

nchannel=self.dataOut.nChannels

2646

nchannel=self.dataOut.nChannels

2646

2647

for i in range(nchannel):

2648

for i in range(nchannel):

2648

col = self.wrObj.writeData(name="PCh"+str(i+1),

2649

col = self.wrObj.writeData(name="PCh"+str(i+1),

2649

format=str(self.dataOut.nFFTPoints)+'E',

2650

format=str(self.dataOut.nFFTPoints)+'E',

2650

data=10*numpy.log10(self.dataOut.data_spc[i,:]))

2651

data=10*numpy.log10(self.dataOut.data_spc[i,:]))

2651

2652

colList.append(col)

2653

colList.append(col)

2653

2654

data=self.wrObj.Ctable(colList=colList)

2655

data=self.wrObj.Ctable(colList=colList)

2655

2656

self.wrObj.CFile(header,data)

2657

self.wrObj.CFile(header,data)

2657

2658

self.wrObj.wFile(filename)

2659

self.wrObj.wFile(filename)

2659

2660

#update the setFile

2661

#update the setFile

2661

self.setFile += 1

2662

self.setFile += 1

2662

self.idblock += 1

2663

self.idblock += 1

2663

2664

return 1

2665

return 1

2665

2666

def run(self, dataOut, **kwargs):

2667

def run(self, dataOut, **kwargs):

2667

2668

if not(self.isConfig):

2669

if not(self.isConfig):

2669

2670

self.setup(dataOut, **kwargs)

2671

self.setup(dataOut, **kwargs)

2671

self.isConfig = True

2672

self.isConfig = True

2672

2673

self.putData()

2674

self.putData()

2674

2675

2676

class FITS:

2677

class FITS:

2677

name=None

2678

name=None

2678

format=None

2679

format=None

2679

array =None

2680

array =None

2680

data =None

2681

data =None

2681

thdulist=None

2682

thdulist=None

2682

prihdr=None

2683

prihdr=None

2683

hdu=None

2684

hdu=None

2684

2685

def __init__(self):

2686

def __init__(self):

2686

2687

pass

2688

pass

2688

2689

def setColF(self,name,format,array):

2690

def setColF(self,name,format,array):

2690

self.name=name

2691

self.name=name

2691

self.format=format

2692

self.format=format

2692

self.array=array

2693

self.array=array

2693

a1=numpy.array([self.array],dtype=numpy.float32)

2694

a1=numpy.array([self.array],dtype=numpy.float32)

2694

self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)

2695

self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)

2695

return self.col1

2696

return self.col1

2696

2697

# def setColP(self,name,format,data):

2698

# def setColP(self,name,format,data):

2698

# self.name=name

2699

# self.name=name

2699

# self.format=format

2700

# self.format=format

2700

# self.data=data

2701

# self.data=data

2701

# a2=numpy.array([self.data],dtype=numpy.float32)

2702

# a2=numpy.array([self.data],dtype=numpy.float32)

2702

# self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)

2703

# self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)

2703

# return self.col2

2704

# return self.col2

2704

2705

2706

def writeData(self,name,format,data):

2707

def writeData(self,name,format,data):

2707

self.name=name

2708

self.name=name

2708

self.format=format

2709

self.format=format

2709

self.data=data

2710

self.data=data

2710

a2=numpy.array([self.data],dtype=numpy.float32)

2711

a2=numpy.array([self.data],dtype=numpy.float32)

2711

self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)

2712

self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)

2712

return self.col2

2713

return self.col2

2713

2714

def cFImage(self,idblock,year,month,day,hour,minute,second):

2715

def cFImage(self,idblock,year,month,day,hour,minute,second):

2715

self.hdu= pyfits.PrimaryHDU(idblock)

2716

self.hdu= pyfits.PrimaryHDU(idblock)

2716

self.hdu.header.set("Year",year)

2717

self.hdu.header.set("Year",year)

2717

self.hdu.header.set("Month",month)

2718

self.hdu.header.set("Month",month)

2718

self.hdu.header.set("Day",day)

2719

self.hdu.header.set("Day",day)

2719

self.hdu.header.set("Hour",hour)

2720

self.hdu.header.set("Hour",hour)

2720

self.hdu.header.set("Minute",minute)

2721

self.hdu.header.set("Minute",minute)

2721

self.hdu.header.set("Second",second)

2722

self.hdu.header.set("Second",second)

2722

return self.hdu

2723

return self.hdu

2723

2724

2725

def Ctable(self,colList):

2726

def Ctable(self,colList):

2726

self.cols=pyfits.ColDefs(colList)

2727

self.cols=pyfits.ColDefs(colList)

2727

self.tbhdu = pyfits.new_table(self.cols)

2728

self.tbhdu = pyfits.new_table(self.cols)

2728

return self.tbhdu

2729

return self.tbhdu

2729

2730

2731

def CFile(self,hdu,tbhdu):

2732

def CFile(self,hdu,tbhdu):

2732

self.thdulist=pyfits.HDUList([hdu,tbhdu])

2733

self.thdulist=pyfits.HDUList([hdu,tbhdu])

2733

2734

def wFile(self,filename):

2735

def wFile(self,filename):

2735

if os.path.isfile(filename):

2736

if os.path.isfile(filename):

2736

os.remove(filename)

2737

os.remove(filename)

2737

self.thdulist.writeto(filename)

2738

self.thdulist.writeto(filename)

2738

2739

2740

class ParameterConf:

2741

class ParameterConf:

2741

ELEMENTNAME = 'Parameter'

2742

ELEMENTNAME = 'Parameter'

2742

def __init__(self):

2743

def __init__(self):

2743

self.name = ''

2744

self.name = ''

2744

self.value = ''

2745

self.value = ''

2745

2746

def readXml(self, parmElement):

2747

def readXml(self, parmElement):

2747

self.name = parmElement.get('name')

2748

self.name = parmElement.get('name')

2748

self.value = parmElement.get('value')

2749

self.value = parmElement.get('value')

2749

2750

def getElementName(self):

2751

def getElementName(self):

2751

return self.ELEMENTNAME

2752

return self.ELEMENTNAME

2752

2753

class Metadata:

2754

class Metadata:

2754

2755

def __init__(self, filename):

2756

def __init__(self, filename):

2756

self.parmConfObjList = []

2757

self.parmConfObjList = []

2757

self.readXml(filename)

2758

self.readXml(filename)

2758

2759

def readXml(self, filename):

2760

def readXml(self, filename):

2760

self.projectElement = None

2761

self.projectElement = None

2761

self.procUnitConfObjDict = {}

2762

self.procUnitConfObjDict = {}

2762

self.projectElement = ElementTree().parse(filename)

2763

self.projectElement = ElementTree().parse(filename)

2763

self.project = self.projectElement.tag

2764

self.project = self.projectElement.tag

2764

2765

parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())

2766

parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())

2766

2767

for parmElement in parmElementList:

2768

for parmElement in parmElementList:

2768

parmConfObj = ParameterConf()

2769

parmConfObj = ParameterConf()

2769

parmConfObj.readXml(parmElement)

2770

parmConfObj.readXml(parmElement)

2770

self.parmConfObjList.append(parmConfObj)

2771

self.parmConfObjList.append(parmConfObj)

2771

2772

class FitsWriter(Operation):

2773

class FitsWriter(Operation):

2773

2774

def __init__(self):

2775

def __init__(self):

2775

self.isConfig = False

2776

self.isConfig = False

2776

self.dataBlocksPerFile = None

2777

self.dataBlocksPerFile = None

2777

self.blockIndex = 0

2778

self.blockIndex = 0

2778

self.flagIsNewFile = 1

2779

self.flagIsNewFile = 1

2779

self.fitsObj = None

2780

self.fitsObj = None

2780

self.optchar = 'P'

2781

self.optchar = 'P'

2781

self.ext = '.fits'

2782

self.ext = '.fits'

2782

self.setFile = 0

2783

self.setFile = 0

2783

2784

def setFitsHeader(self, dataOut, metadatafile):

2785

def setFitsHeader(self, dataOut, metadatafile):

2785

2786

header_data = pyfits.PrimaryHDU()

2787

header_data = pyfits.PrimaryHDU()

2787

2788

metadata4fits = Metadata(metadatafile)

2789

metadata4fits = Metadata(metadatafile)

2789

for parameter in metadata4fits.parmConfObjList:

2790

for parameter in metadata4fits.parmConfObjList:

2790

parm_name = parameter.name

2791

parm_name = parameter.name

2791

parm_value = parameter.value

2792

parm_value = parameter.value

2792

2793

if parm_value == 'fromdatadatetime':

2794

if parm_value == 'fromdatadatetime':

2794

value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())

2795

value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())

2795

elif parm_value == 'fromdataheights':

2796

elif parm_value == 'fromdataheights':

2796

value = dataOut.nHeights

2797

value = dataOut.nHeights

2797

elif parm_value == 'fromdatachannel':

2798

elif parm_value == 'fromdatachannel':

2798

value = dataOut.nChannels

2799

value = dataOut.nChannels

2799

elif parm_value == 'fromdatasamples':

2800

elif parm_value == 'fromdatasamples':

2800

value = dataOut.nFFTPoints

2801

value = dataOut.nFFTPoints

2801

else:

2802

else:

2802

value = parm_value

2803

value = parm_value

2803

2804

header_data.header[parm_name] = value

2805

header_data.header[parm_name] = value

2805

2806

header_data.header['NBLOCK'] = self.blockIndex

2807

header_data.header['NBLOCK'] = self.blockIndex

2807

2808

header_data.writeto(self.filename)

2809

header_data.writeto(self.filename)

2809

2810

2811

def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):

2812

def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):

2812

2813

self.path = path

2814

self.path = path

2814

self.dataOut = dataOut

2815

self.dataOut = dataOut

2815

self.metadatafile = metadatafile

2816

self.metadatafile = metadatafile

2816

self.dataBlocksPerFile = dataBlocksPerFile

2817

self.dataBlocksPerFile = dataBlocksPerFile

2817

2818

def open(self):

2819

def open(self):

2819

self.fitsObj = pyfits.open(self.filename, mode='update')

2820

self.fitsObj = pyfits.open(self.filename, mode='update')

2820

2821

2822

def addData(self, data):

2823

def addData(self, data):

2823

self.open()

2824

self.open()

2824

extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATA'])

2825

extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATA'])

2825

extension.header['UTCTIME'] = self.dataOut.utctime

2826

extension.header['UTCTIME'] = self.dataOut.utctime

2826

self.fitsObj.append(extension)

2827

self.fitsObj.append(extension)

2827

self.blockIndex += 1

2828

self.blockIndex += 1

2828

self.fitsObj[0].header['NBLOCK'] = self.blockIndex

2829

self.fitsObj[0].header['NBLOCK'] = self.blockIndex

2829

2830

self.write()

2831

self.write()

2831

2832

def write(self):

2833

def write(self):

2833

2834

self.fitsObj.flush(verbose=True)

2835

self.fitsObj.flush(verbose=True)

2835

self.fitsObj.close()

2836

self.fitsObj.close()

2836

2837

2838

def setNextFile(self):

2839

def setNextFile(self):

2839

2840

ext = self.ext

2841

ext = self.ext

2841

path = self.path

2842

path = self.path

2842

2843

timeTuple = time.localtime( self.dataOut.utctime)

2844

timeTuple = time.localtime( self.dataOut.utctime)

2844

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

2845

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

2845

2846

fullpath = os.path.join( path, subfolder )

2847

fullpath = os.path.join( path, subfolder )

2847

if not( os.path.exists(fullpath) ):

2848

if not( os.path.exists(fullpath) ):

2848

os.mkdir(fullpath)

2849

os.mkdir(fullpath)

2849

self.setFile = -1 #inicializo mi contador de seteo

2850

self.setFile = -1 #inicializo mi contador de seteo

2850

else:

2851

else:

2851

filesList = os.listdir( fullpath )

2852

filesList = os.listdir( fullpath )

2852

if len( filesList ) > 0:

2853

if len( filesList ) > 0:

2853

filesList = sorted( filesList, key=str.lower )

2854

filesList = sorted( filesList, key=str.lower )

2854

filen = filesList[-1]

2855

filen = filesList[-1]

2855

2856

if isNumber( filen[8:11] ):

2857

if isNumber( filen[8:11] ):

2857

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

2858

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

2858

else:

2859

else:

2859

self.setFile = -1

2860

self.setFile = -1

2860

else:

2861

else:

2861

self.setFile = -1 #inicializo mi contador de seteo

2862

self.setFile = -1 #inicializo mi contador de seteo

2862

2863

setFile = self.setFile

2864

setFile = self.setFile

2864

setFile += 1

2865

setFile += 1

2865

2866

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

2867

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

2867

timeTuple.tm_year,

2868

timeTuple.tm_year,

2868

timeTuple.tm_yday,

2869

timeTuple.tm_yday,

2869

setFile,

2870

setFile,

2870

ext )

2871

ext )

2871

2872

filename = os.path.join( path, subfolder, file )

2873

filename = os.path.join( path, subfolder, file )

2873

2874

self.blockIndex = 0

2875

self.blockIndex = 0

2875

self.filename = filename

2876

self.filename = filename

2876

self.setFile = setFile

2877

self.setFile = setFile

2877

self.flagIsNewFile = 1

2878

self.flagIsNewFile = 1

2878

2879

print 'Writing the file: %s'%self.filename

2880

print 'Writing the file: %s'%self.filename

2880

2881

self.setFitsHeader(self.dataOut, self.metadatafile)

2882

self.setFitsHeader(self.dataOut, self.metadatafile)

2882

2883

return 1

2884

return 1

2884

2885

def writeBlock(self):

2886

def writeBlock(self):

2886

self.addData(self.dataOut.data_spc)

2887

self.addData(self.dataOut.data_spc)

2887

self.flagIsNewFile = 0

2888

self.flagIsNewFile = 0

2888

2889

2890

def __setNewBlock(self):

2891

def __setNewBlock(self):

2891

2892

if self.flagIsNewFile:

2893

if self.flagIsNewFile:

2893

return 1

2894

return 1

2894

2895

if self.blockIndex < self.dataBlocksPerFile:

2896

if self.blockIndex < self.dataBlocksPerFile:

2896

return 1

2897

return 1

2897

2898

if not( self.setNextFile() ):

2899

if not( self.setNextFile() ):

2899

return 0

2900

return 0

2900

2901

return 1

2902

return 1

2902

2903

def writeNextBlock(self):

2904

def writeNextBlock(self):

2904

if not( self.__setNewBlock() ):

2905

if not( self.__setNewBlock() ):

2905

return 0

2906

return 0

2906

self.writeBlock()

2907

self.writeBlock()

2907

return 1

2908

return 1

2908

2909

def putData(self):

2910

def putData(self):

2910

if self.flagIsNewFile:

2911

if self.flagIsNewFile:

2911

self.setNextFile()

2912

self.setNextFile()

2912

self.writeNextBlock()

2913

self.writeNextBlock()

2913

2914

def run(self, dataOut, **kwargs):

2915

def run(self, dataOut, **kwargs):

2915

if not(self.isConfig):

2916

if not(self.isConfig):

2916

self.setup(dataOut, **kwargs)

2917

self.setup(dataOut, **kwargs)

2917

self.isConfig = True

2918

self.isConfig = True

2918

self.putData()

2919

self.putData()

2919

2920

2921

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages