schain Commit - r434:586e60daf0fb · Jicamarca Repository

             '''
             $Author: murco $
             $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
             '''
             import os, sys
             import glob
             import time
             import numpy
             import fnmatch
             import time, datetime
             from xml.etree.ElementTree import Element, SubElement, ElementTree
             try:
                 import pyfits
             except:
                 print "pyfits module has not been imported, it should be installed to save files in fits format"
             from jrodata import *
             from jroheaderIO import *
             from jroprocessing import *
             LOCALTIME = True #-18000
             def isNumber(str):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float( str )
                     return True
                 except:
                     return False
             def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename,'rb')
                 except:
                     raise IOError, "The file %s can't be opened" %(filename)
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isFileinThisTime(filename, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename,'rb')
                 except:
                     raise IOError, "The file %s can't be opened" %(filename)
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = basicHeaderObj.datatime
                 thisTime = basicHeaderObj.datatime.time()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return None
                 if not ((startTime <= thisTime) and (endTime > thisTime)):
                     return None
                 return thisDatetime
             def getFileFromSet(path,ext,set):
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for file in fileList:
                     try:
                         year = int(file[1:5])
                         doy  = int(file[5:8])
                     except:
                         continue
                     if (os.path.splitext(file)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(file)
                 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
                 if len(myfile)!= 0:
                     return myfile[0]
                 else:
                     filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
                     print 'the filename %s does not exist'%filename
                     print '...going to the last file: '
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for file in fileList:
                     try:
                         year = int(file[1:5])
                         doy  = int(file[5:8])
                     except:
                         continue
                     if (os.path.splitext(file)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(file)
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def checkForRealPath(path, foldercounter, year, doy, set, ext):
                 """
                 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
                 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
                 el path exacto de un determinado file.
                 Example    :
                     nombre correcto del file es  .../.../D2009307/P2009307367.ext
                     Entonces la funcion prueba con las siguientes combinaciones
                         .../.../y2009307367.ext
                         .../.../Y2009307367.ext
                         .../.../x2009307/y2009307367.ext
                         .../.../x2009307/Y2009307367.ext
                         .../.../X2009307/y2009307367.ext
                         .../.../X2009307/Y2009307367.ext
                     siendo para este caso, la ultima combinacion de letras, identica al file buscado
                 Return:
                     Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
                     caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
                     para el filename
                 """
                 fullfilename = None
                 find_flag = False
                 filename = None
                 prefixDirList = [None,'d','D']
                 if ext.lower() == ".r": #voltage
                     prefixFileList = ['d','D']
                 elif ext.lower() == ".pdata": #spectra
                     prefixFileList = ['p','P']
                 else:
                     return None, filename
                 #barrido por las combinaciones posibles
                 for prefixDir in prefixDirList:
                     thispath = path
                     if prefixDir != None:
                         #formo el nombre del directorio xYYYYDDD (x=d o x=D)
                         if foldercounter == 0:
                             thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
                         else:
                             thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
                     for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
                         filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
                         fullfilename = os.path.join( thispath, filename ) #formo el path completo
                         if os.path.exists( fullfilename ): #verifico que exista
                             find_flag = True
                             break
                     if find_flag:
                         break
                 if not(find_flag):
                     return None, filename
                 return fullfilename, filename
             def isDoyFolder(folder):
                 try:
                     year = int(folder[1:5])
                 except:
                     return 0
                 try:
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
             class JRODataIO:
                 c = 3E8
                 isConfig = False
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
                 processingHeaderObj = ProcessingHeader()
                 online = 0
                 dtype = None
                 pathList = []
                 filenameList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagTimeBlock = 0
                 flagIsNewBlock = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
                 ippSeconds = None
                 fileSizeByHeader = None
                 fileIndex = None
                 profileIndex = None
                 blockIndex = None
                 nTotalBlocks = None
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
                 blocksize = None
                 def __init__(self):
                     raise ValueError, "Not implemented"
                 def run(self):
                     raise ValueError, "Not implemented"
                 def getOutput(self):
                     return self.dataOut
             class JRODataReader(JRODataIO, ProcessingUnit):
                 nReadBlocks = 0
                 delay  = 10   #number of seconds waiting a new file
                 nTries  = 3  #quantity tries
                 nFiles = 3   #number of files for searching
                 path = None
                 foldercounter = 0
                 flagNoMoreFiles = 0
                 datetimeList = []
                 __isFirstTimeOnline = 1
                 __printInfo = True
                 profileIndex = None
                 def __init__(self):
                     """
                     """
                     raise ValueError, "This method has not been implemented"
                 def createObjByDefault(self):
                     """
                     """
                     raise ValueError, "This method has not been implemented"
                 def getBlockDimension(self):
                     raise ValueError, "No implemented"
                 def __searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.r',
                                         walk=True):
                     pathList = []
                     if not walk:
                         pathList.append(path)
                     else:
                         dirList = []
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isDoyFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None
                         thisDate = startDate
                         while(thisDate <= endDate):
                             year = thisDate.timetuple().tm_year
                             doy = thisDate.timetuple().tm_yday
                             matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
                             if len(matchlist) == 0:
                                 thisDate += datetime.timedelta(1)
                                 continue
                             for match in matchlist:
                                 pathList.append(os.path.join(path,match,expLabel))
                             thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
                         return None, None
                     print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
                     filenameList = []
                     datetimeList = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             thisDatetime = isFileinThisTime(filename, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "Any file was found for the time range %s - %s" %(startTime, endTime)
                         return None, None
                     print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
                     """
                     Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
                     devuelve el archivo encontrado ademas de otros datos.
                     Input:
                         path           :    carpeta donde estan contenidos los files que contiene data
                         expLabel        :     Nombre del subexperimento (subfolder)
                         ext              :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
                         directory   :    eL directorio donde esta el file encontrado
                         filename    :    el ultimo file de una determinada carpeta
                         year        :    el anho
                         doy         :    el numero de dia del anho
                         set         :    el set del archivo
                     """
                     dirList = []
                     if not walk:
                         fullpath = path
                         foldercounter = 0
                     else:
                         #Filtra solo los directorios
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isDoyFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None, None, None, None, None
                         dirList = sorted( dirList, key=str.lower )
                         doypath = dirList[-1]
                         foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
                         fullpath = os.path.join(path, doypath, expLabel)
                     print "%s folder was found: " %(fullpath )
                     if set == None:
                         filename = getlastFileFromPath(fullpath, ext)
                     else:
                         filename = getFileFromSet(fullpath, ext, set)
                     if not(filename):
                         return None, None, None, None, None, None
                     print "%s file was found" %(filename)
                     if not(self.__verifyFile(os.path.join(fullpath, filename))):
                         return None, None, None, None, None, None
                     year = int( filename[1:5] )
                     doy  = int( filename[5:8] )
                     set  = int( filename[8:11] )
                     return fullpath, foldercounter, filename, year, doy, set
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print "No more Files"
                             return 0
                         filename = self.filenameList[idFile]
                         if not(self.__verifyFile(filename)):
                             continue
                         fileSize = os.path.getsize(filename)
                         fp = open(filename,'rb')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fp = fp
                     print "Setting the file: %s"%self.filename
                     return 1
                 def __setNextFileOnline(self):
                     """
                     Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
                     no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
                     siguientes.
                     Affected:
                         self.flagIsNewFile
                         self.filename
                         self.fileSize
                         self.fp
                         self.set
                         self.flagNoMoreFiles
                     Return:
 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
 : si el file fue abierto con exito y esta listo a ser leido
                     Excepciones:
                         Si un determinado file no puede ser abierto
                     """
                     nFiles = 0
                     fileOk_flag = False
                     firstTime_flag = True
                     self.set += 1
                     if self.set > 999:
                         self.set = 0
                         self.foldercounter += 1
                     #busca el 1er file disponible
                     fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                     if fullfilename:
                         if self.__verifyFile(fullfilename, False):
                             fileOk_flag = True
                     #si no encuentra un file entonces espera y vuelve a buscar
                     if not(fileOk_flag):
                         for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
                             if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
                                 tries = self.nTries
                             else:
                                 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
                             for nTries in range( tries ):
                                 if firstTime_flag:
                                     print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
                                     time.sleep( self.delay )
                                 else:
                                     print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
                                 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                                 if fullfilename:
                                     if self.__verifyFile(fullfilename):
                                         fileOk_flag = True
                                         break
                             if fileOk_flag:
                                 break
                             firstTime_flag = False
                             print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename
                             self.set += 1
                             if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
                                 self.set = 0
                                 self.doy += 1
                                 self.foldercounter = 0
                     if fileOk_flag:
                         self.fileSize = os.path.getsize( fullfilename )
                         self.filename = fullfilename
                         self.flagIsNewFile = 1
                         if self.fp != None: self.fp.close()
                         self.fp = open(fullfilename, 'rb')
                         self.flagNoMoreFiles = 0
                         print 'Setting the file: %s' % fullfilename
                     else:
                         self.fileSize = 0
                         self.filename = None
                         self.flagIsNewFile = 0
                         self.fp = None
                         self.flagNoMoreFiles = 1
                         print 'No more Files'
                     return fileOk_flag
                 def setNextFile(self):
                     if self.fp != None:
                         self.fp.close()
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         return 0
                     self.__readFirstHeader()
                     self.nReadBlocks = 0
                     return 1
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.__rdBasicHeader()
                             return 1
                         if self.fileSize == self.fileSizeByHeader:
             #                self.flagEoF = True
                             return 0
                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         time.sleep( self.delay )
                     return 0
+                def waitDataBlock(self,pointer_location):
+                    currentPointer = pointer_location
+                    neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
+                    for nTries in range( self.nTries ):
+                        self.fp.close()
+                        self.fp = open( self.filename, 'rb' )
+                        self.fp.seek( currentPointer )
+                        self.fileSize = os.path.getsize( self.filename )
+                        currentSize = self.fileSize - currentPointer
+                        if ( currentSize >= neededSize ):
+                            return 1
+                        print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
+                        time.sleep( self.delay )
+                    return 0
                 def __jumpToLastBlock(self):
                     if not(self.__isFirstTimeOnline):
                         return
                     csize = self.fileSize - self.fp.tell()
                     blocksize = self.processingHeaderObj.blockSize
                     #salta el primer bloque de datos
                     if csize > self.processingHeaderObj.blockSize:
                         self.fp.seek(self.fp.tell() + blocksize)
                     else:
                         return
                     csize = self.fileSize - self.fp.tell()
                     neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     while True:
                         if self.fp.tell()<self.fileSize:
                             self.fp.seek(self.fp.tell() + neededsize)
                         else:
                             self.fp.seek(self.fp.tell() - neededsize)
                             break
             #        csize = self.fileSize - self.fp.tell()
             #        neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
             #        factor = int(csize/neededsize)
             #        if factor > 0:
             #            self.fp.seek(self.fp.tell() + factor*neededsize)
                     self.flagIsNewFile = 0
                     self.__isFirstTimeOnline = 0
                 def __setNewBlock(self):
                     if self.fp == None:
                         return 0
                     if self.online:
                         self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         return 1
                     self.lastUTTime = self.basicHeaderObj.utc
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.__rdBasicHeader()
                         return 1
                     if self.__waitNewBlock():
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
                     self.flagTimeBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagTimeBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def __rdProcessingHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.processingHeaderObj.read(fp)
                 def __rdRadarControllerHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.radarControllerHeaderObj.read(fp)
                 def __rdSystemHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.systemHeaderObj.read(fp)
                 def __rdBasicHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.basicHeaderObj.read(fp)
                 def __readFirstHeader(self):
                     self.__rdBasicHeader()
                     self.__rdSystemHeader()
                     self.__rdRadarControllerHeader()
                     self.__rdProcessingHeader()
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
                     else:
                         raise ValueError, 'Data type was not defined'
                     self.dtype = datatype_str
                     self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
             #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
             #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def __verifyFile(self, filename, msgFlag=True):
                     msg = None
                     try:
                         fp = open(filename, 'rb')
                         currentPosition = fp.tell()
                     except:
                         if msgFlag:
                             print "The file %s can't be opened" % (filename)
                         return False
                     neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                     if neededSize == 0:
                         basicHeaderObj = BasicHeader(LOCALTIME)
                         systemHeaderObj = SystemHeader()
                         radarControllerHeaderObj = RadarControllerHeader()
                         processingHeaderObj = ProcessingHeader()
                         try:
                             if not( basicHeaderObj.read(fp) ): raise IOError
                             if not( systemHeaderObj.read(fp) ): raise IOError
                             if not( radarControllerHeaderObj.read(fp) ): raise IOError
                             if not( processingHeaderObj.read(fp) ): raise IOError
                             data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                             neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
                         except:
                             if msgFlag:
                                 print "\tThe file %s is empty or it hasn't enough data" % filename
                             fp.close()
                             return False
                     else:
                         msg = "\tSkipping the file %s due to it hasn't enough data" %filename
                     fp.close()
                     fileSize = os.path.getsize(filename)
                     currentSize = fileSize - currentPosition
                     if currentSize < neededSize:
                         if msgFlag and (msg != None):
                             print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
                         return False
                     return True
                 def setup(self,
                             path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=None,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True):
                     if path == None:
                         raise ValueError, "The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if online:
                         print "Searching files in online mode..."
                         for nTries in range( self.nTries ):
                             fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
                             if fullpath:
                                 break
                             print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
                             time.sleep( self.delay )
                         if not(fullpath):
                             print "There 'isn't valied files in %s" % path
                             return None
                         self.year = year
                         self.doy  = doy
                         self.set  = set - 1
                         self.path = path
                         self.foldercounter = foldercounter
                     else:
                         print "Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
                             print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime())
                             sys.exit(-1)
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                     self.online = online
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "No files"
                         sys.exit(-1)
             #        self.updateDataHeader()
                     return self.dataOut
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds
                     self.dataOut.flagTimeBlock = self.flagTimeBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                 def getFirstHeader(self):
                     raise ValueError, "This method has not been implemented"
                 def getData():
                     raise ValueError, "This method has not been implemented"
                 def hasNotDataInBuffer():
                     raise ValueError, "This method has not been implemented"
                 def readBlock():
                     raise ValueError, "This method has not been implemented"
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print "Number of read blocks per file %04d" %self.nReadBlocks
                 def printTotalBlocks(self):
                     print "Number of read blocks %04d" %self.nTotalBlocks
                 def printNumberOfBlock(self):
                     if self.flagIsNewBlock:
                         print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
                 def printInfo(self):
                     if self.__printInfo == False:
                         return
                     self.basicHeaderObj.printInfo()
                     self.systemHeaderObj.printInfo()
                     self.radarControllerHeaderObj.printInfo()
                     self.processingHeaderObj.printInfo()
                     self.__printInfo = False
                 def run(self, **kwargs):
                     if not(self.isConfig):
             #            self.dataOut = dataOut
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             class JRODataWriter(JRODataIO, Operation):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 blockIndex = 0
                 path = None
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 def __init__(self, dataOut=None):
                     raise ValueError, "Not implemented"
                 def hasAllDataInBuffer(self):
                     raise ValueError, "Not implemented"
                 def setBlockDimension(self):
                     raise ValueError, "Not implemented"
                 def writeBlock(self):
                     raise ValueError, "No implemented"
                 def putData(self):
                     raise ValueError, "No implemented"
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize #bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond  = (self.dataOut.utctime - utc)* 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
                     raise ValueError, "No implemented"
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                     self.dtype = self.dataOut.dtype
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def setNextFile(self):
                     """
                     Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, file )
                     fp = open( filename,'wb' )
                     self.blockIndex = 0
                     #guardando atributos
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.setFirstHeader()
                     print 'Writing the file: %s'%self.filename
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path      :    el path destino en el cual se escribiran los files a crear
                         format    :    formato en el cual sera salvado un file
                         set       :    el setebo del file
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     ext = ext.lower()
                     self.ext = ext
                     self.path = path
                     self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     if not(self.setNextFile()):
                         print "There isn't a next file"
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class VoltageReader(JRODataReader):
                 """
                 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
                 perfiles*alturas*canales) son almacenados en la variable "buffer".
                                          perfiles * alturas * canales
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = VoltageReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         #to get one profile
                         profile =  readerObj.getData()
                         #print the profile
                         print profile
                         #If you want to see all datablock
                         print readerObj.datablock
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 ext = ".r"
                 optchar = "D"
                 dataOut = None
                 def __init__(self):
                     """
                     Inicializador de la clase VoltageReader para la lectura de datos de voltage.
                     Input:
                         dataOut    :    Objeto de la clase Voltage. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Variables afectadas:
                         self.dataOut
                     Return:
                         None
                     """
                     self.isConfig = False
                     self.datablock = None
                     self.utc = 0
                     self.ext = ".r"
                     self.optchar = "D"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.online = 0
                     self.fp = None
                     self.idFile = None
                     self.dtype = None
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.filename = None
                     self.fileSize = None
                     self.firstHeaderSize = 0
                     self.basicHeaderSize = 24
                     self.pathList = []
                     self.filenameList = []
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
                     self.flagNoMoreFiles = 0
                     self.set = 0
                     self.path = None
                     self.profileIndex = 2**32-1
                     self.delay  = 3   #seconds
                     self.nTries  = 3  #quantity tries
                     self.nFiles = 3   #number of files for searching
                     self.nReadBlocks = 0
                     self.flagIsNewFile = 1
                     self.__isFirstTimeOnline = 1
                     self.ippSeconds = 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     self.nTotalBlocks = 0
                     self.blocksize = 0
                     self.dataOut = self.createObjByDefault()
                 def createObjByDefault(self):
                     dataObj = Voltage()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
                         return 1
                     return 0
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.blocksize
                     Return:
                         None
                     """
                     pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
                     self.blocksize = pts2read
                 def readBlock(self):
                     """
                     readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Inputs:
                         None
                     Return:
                         None
                     Affected:
                         self.profileIndex
                         self.datablock
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
+                    current_pointer_location = self.fp.tell()
                     junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
                     try:
                         junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
                     except:
-                        print "The read block (%3d) has not enough data" %self.nReadBlocks
+                        #print "The read block (%3d) has not enough data" %self.nReadBlocks
-                        return 0
+                        if self.waitDataBlock(pointer_location=current_pointer_location):
+                            junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
+                            junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
+            #             return 0
                     junk = numpy.transpose(junk, (2,0,1))
                     self.datablock = junk['real'] + junk['imag']*1j
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.dataOut.dtype = self.dtype
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
                     self.dataOut.ippSeconds = self.ippSeconds
                     self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.flagShiftFFT = False
                     if self.radarControllerHeaderObj.code != None:
                         self.dataOut.nCode = self.radarControllerHeaderObj.nCode
                         self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
                         self.dataOut.code = self.radarControllerHeaderObj.code
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip
                     self.dataOut.flagShiftFFT = False
                 def getData(self):
                     """
                     getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Ademas incrementa el contador del buffer en 1.
                     Return:
                         data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                      buffer. Si no hay mas archivos a leer retorna None.
                     Variables afectadas:
                         self.dataOut
                         self.profileIndex
                     Affected:
                         self.dataOut
                         self.profileIndex
                         self.flagTimeBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             return 0
                         self.getFirstHeader()
                     if self.datablock == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.dataOut.data = self.datablock[:,self.profileIndex,:]
                     self.dataOut.flagNoData = False
                     self.getBasicHeader()
                     self.profileIndex += 1
                     self.dataOut.realtime = self.online
                     return self.dataOut.data
             class VoltageWriter(JRODataWriter):
                 """
                 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".r"
                 optchar = "D"
                 shapeBuffer = None
                 def __init__(self):
                     """
                     Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                     Return: None
                     """
                     self.nTotalBlocks = 0
                     self.profileIndex = 0
                     self.isConfig = False
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.nTotalBlocks = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
                         return 1
                     return 0
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
                                         self.processingHeaderObj.nHeights,
                                         self.systemHeaderObj.nChannels)
                     self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
                                                  self.processingHeaderObj.profilesPerBlock,
                                                  self.processingHeaderObj.nHeights),
                                                  dtype=numpy.dtype('complex64'))
                 def writeBlock(self):
                     """
                     Escribe el buffer en el file designado
                     Affected:
                         self.profileIndex
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.blockIndex
                     Return: None
                     """
                     data = numpy.zeros( self.shapeBuffer, self.dtype )
                     junk = numpy.transpose(self.datablock, (1,2,0))
                     data['real'] = junk.real
                     data['imag'] = junk.imag
                     data = data.reshape( (-1) )
                     data.tofile( self.fp )
                     self.datablock.fill(0)
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.flagIsNewBlock
                         self.profileIndex
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagTimeBlock:
                         self.datablock.fill(0)
                         self.profileIndex = 0
                         self.setNextFile()
                     if self.profileIndex == 0:
                         self.setBasicHeader()
                     self.datablock[:,self.profileIndex,:] = self.dataOut.data
                     self.profileIndex += 1
                     if self.hasAllDataInBuffer():
                         #if self.flagIsNewFile:
                         self.writeNextBlock()
             #            self.setFirstHeader()
                     return 1
                 def __getProcessFlags(self):
                     processFlags = 0
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList =  [PROCFLAG.DATATYPE_CHAR,
                                        PROCFLAG.DATATYPE_SHORT,
                                        PROCFLAG.DATATYPE_LONG,
                                        PROCFLAG.DATATYPE_INT64,
                                        PROCFLAG.DATATYPE_FLOAT,
                                        PROCFLAG.DATATYPE_DOUBLE]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             dtypeValue = datatypeValueList[index]
                             break
                     processFlags += dtypeValue
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code != None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nCohInt > 1:
                         processFlags += PROCFLAG.COHERENT_INTEGRATION
                     return processFlags
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
                     '''
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList = [1,2,4,8,4,8]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             datatypeValue = datatypeValueList[index]
                             break
                     blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * datatypeValue * 2)
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.setBasicHeader()
                     processingHeaderSize = 40 # bytes
                     self.processingHeaderObj.dtype = 0 # Voltage
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.processFlags = self.__getProcessFlags()
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
                     self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
                     self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
             #         if self.dataOut.code != None:
             #             self.processingHeaderObj.code = self.dataOut.code
             #             self.processingHeaderObj.nCode = self.dataOut.nCode
             #             self.processingHeaderObj.nBaud = self.dataOut.nBaud
             #             codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)
             #             processingHeaderSize += codesize
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                         processingHeaderSize += 12
                     self.processingHeaderObj.size = processingHeaderSize
             class SpectraReader(JRODataReader):
                 """
                 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
                 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
                                     paresCanalesIguales    * alturas * perfiles  (Self Spectra)
                                     paresCanalesDiferentes * alturas * perfiles  (Cross Spectra)
                                     canales * alturas                            (DC Channels)
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = SpectraReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         readerObj.getData()
                         print readerObj.data_spc
                         print readerObj.data_cspc
                         print readerObj.data_dc
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 pts2read_SelfSpectra = 0
                 pts2read_CrossSpectra = 0
                 pts2read_DCchannels = 0
                 ext = ".pdata"
                 optchar = "P"
                 dataOut = None
                 nRdChannels = None
                 nRdPairs = None
                 rdPairList = []
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraReader para la lectura de datos de espectros.
                     Inputs:
                         dataOut    :    Objeto de la clase Spectra. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Affected:
                         self.dataOut
                     Return      : None
                     """
                     self.isConfig = False
                     self.pts2read_SelfSpectra = 0
                     self.pts2read_CrossSpectra = 0
                     self.pts2read_DCchannels = 0
                     self.datablock = None
                     self.utc = None
                     self.ext = ".pdata"
                     self.optchar = "P"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.online = 0
                     self.fp = None
                     self.idFile = None
                     self.dtype = None
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.filename = None
                     self.fileSize = None
                     self.firstHeaderSize = 0
                     self.basicHeaderSize = 24
                     self.pathList = []
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
                     self.flagNoMoreFiles = 0
                     self.set = 0
                     self.path = None
                     self.delay  = 60   #seconds
                     self.nTries = 3  #quantity tries
                     self.nFiles = 3   #number of files for searching
                     self.nReadBlocks = 0
                     self.flagIsNewFile = 1
                     self.__isFirstTimeOnline = 1
                     self.ippSeconds = 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     self.nTotalBlocks = 0
                     self.blocksize = 0
                     self.dataOut = self.createObjByDefault()
                     self.profileIndex = 1 #Always
                 def createObjByDefault(self):
                     dataObj = Spectra()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     return 1
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.nRdChannels
                         self.nRdPairs
                         self.pts2read_SelfSpectra
                         self.pts2read_CrossSpectra
                         self.pts2read_DCchannels
                         self.blocksize
                         self.dataOut.nChannels
                         self.dataOut.nPairs
                     Return:
                         None
                     """
                     self.nRdChannels = 0
                     self.nRdPairs = 0
                     self.rdPairList = []
                     for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
                         if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
                             self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
                         else:
                             self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
                             self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
                     pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
                     self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
                     self.blocksize = self.pts2read_SelfSpectra
                     if self.processingHeaderObj.flag_cspc:
                         self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
                         self.blocksize += self.pts2read_CrossSpectra
                     if self.processingHeaderObj.flag_dc:
                         self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
                         self.blocksize += self.pts2read_DCchannels
             #        self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
                 def readBlock(self):
                     """
                     Lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Return: None
                     Variables afectadas:
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
                     blockOk_flag = False
                     fpointer = self.fp.tell()
                     spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
                     spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
                         cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_dc:
                         dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
                         dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
                     if not(self.processingHeaderObj.shif_fft):
                         #desplaza a la derecha en el eje 2 determinadas posiciones
                         shift = int(self.processingHeaderObj.profilesPerBlock/2)
                         spc = numpy.roll( spc, shift , axis=2 )
                         if self.processingHeaderObj.flag_cspc:
                             #desplaza a la derecha en el eje 2 determinadas posiciones
                             cspc = numpy.roll( cspc, shift, axis=2 )
             #            self.processingHeaderObj.shif_fft = True
                     spc = numpy.transpose( spc, (0,2,1) )
                     self.data_spc = spc
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.transpose( cspc, (0,2,1) )
                         self.data_cspc = cspc['real'] + cspc['imag']*1j
                     else:
                         self.data_cspc = None
                     if self.processingHeaderObj.flag_dc:
                         self.data_dc = dc['real'] + dc['imag']*1j
                     else:
                         self.data_dc = None
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.dataOut.dtype = self.dtype
                     self.dataOut.nPairs = self.nRdPairs
                     self.dataOut.pairsList = self.rdPairList
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
                     self.dataOut.ippSeconds = self.ippSeconds
                     self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
                     self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip
                     if self.processingHeaderObj.code != None:
                         self.dataOut.nCode = self.processingHeaderObj.nCode
                         self.dataOut.nBaud = self.processingHeaderObj.nBaud
                         self.dataOut.code = self.processingHeaderObj.code
                         self.dataOut.flagDecodeData = True
                 def getData(self):
                     """
                     Copia el buffer de lectura a la clase "Spectra",
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Return:
 :    Si no hay mas archivos disponibles
 :    Si hizo una buena copia del buffer
                     Affected:
                         self.dataOut
                         self.flagTimeBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             self.dataOut.flagNoData = True
                             return 0
                     #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
                     if self.data_dc == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.getBasicHeader()
                     self.getFirstHeader()
                     self.dataOut.data_spc = self.data_spc
                     self.dataOut.data_cspc = self.data_cspc
                     self.dataOut.data_dc = self.data_dc
                     self.dataOut.flagNoData = False
                     self.dataOut.realtime = self.online
                     return self.dataOut.data_spc
             class SpectraWriter(JRODataWriter):
                 """
                 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".pdata"
                 optchar = "P"
                 shape_spc_Buffer = None
                 shape_cspc_Buffer = None
                 shape_dc_Buffer = None
                 data_spc = None
                 data_cspc = None
                 data_dc = None
             #    dataOut = None
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj
                     Return: None
                     """
                     self.isConfig = False
                     self.nTotalBlocks = 0
                     self.data_spc = None
                     self.data_cspc = None
                     self.data_dc = None
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.nTotalBlocks = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.noMoreFiles = 0
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     return 1
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shape_spc_Buffer = (self.dataOut.nChannels,
                                              self.processingHeaderObj.nHeights,
                                              self.processingHeaderObj.profilesPerBlock)
                     self.shape_cspc_Buffer = (self.dataOut.nPairs,
                                               self.processingHeaderObj.nHeights,
                                               self.processingHeaderObj.profilesPerBlock)
                     self.shape_dc_Buffer = (self.dataOut.nChannels,
                                             self.processingHeaderObj.nHeights)
                 def writeBlock(self):
                     """
                     Escribe el buffer en el file designado
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.nWriteBlocks
                     Return: None
                     """
                     spc = numpy.transpose( self.data_spc, (0,2,1) )
                     if not( self.processingHeaderObj.shif_fft ):
                         spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                     data = spc.reshape((-1))
                     data = data.astype(self.dtype[0])
                     data.tofile(self.fp)
                     if self.data_cspc != None:
                         data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
                         cspc = numpy.transpose( self.data_cspc, (0,2,1) )
                         if not( self.processingHeaderObj.shif_fft ):
                             cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                         data['real'] = cspc.real
                         data['imag'] = cspc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     if self.data_dc != None:
                         data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
                         dc = self.data_dc
                         data['real'] = dc.real
                         data['imag'] = dc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     self.data_spc.fill(0)
                     if self.data_dc != None:
                         self.data_dc.fill(0)
                     if self.data_cspc != None:
                         self.data_cspc.fill(0)
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nWriteBlocks += 1
                     self.blockIndex += 1
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagTimeBlock:
                         self.data_spc.fill(0)
                         self.data_cspc.fill(0)
                         self.data_dc.fill(0)
                         self.setNextFile()
                     if self.flagIsNewFile == 0:
                         self.setBasicHeader()
                     self.data_spc = self.dataOut.data_spc.copy()
                     if self.dataOut.data_cspc != None:
                         self.data_cspc = self.dataOut.data_cspc.copy()
                     self.data_dc = self.dataOut.data_dc.copy()
                     # #self.processingHeaderObj.dataBlocksPerFile)
                     if self.hasAllDataInBuffer():
             #            self.setFirstHeader()
                         self.writeNextBlock()
                     return 1
                 def __getProcessFlags(self):
                     processFlags = 0
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList =  [PROCFLAG.DATATYPE_CHAR,
                                        PROCFLAG.DATATYPE_SHORT,
                                        PROCFLAG.DATATYPE_LONG,
                                        PROCFLAG.DATATYPE_INT64,
                                        PROCFLAG.DATATYPE_FLOAT,
                                        PROCFLAG.DATATYPE_DOUBLE]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             dtypeValue = datatypeValueList[index]
                             break
                     processFlags += dtypeValue
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code != None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nIncohInt > 1:
                         processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                     if self.dataOut.data_dc != None:
                         processFlags += PROCFLAG.SAVE_CHANNELS_DC
                     return processFlags
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
                     '''
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList = [1,2,4,8,4,8]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             datatypeValue = datatypeValueList[index]
                             break
                     pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
                     pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
                     blocksize = (pts2write_SelfSpectra*datatypeValue)
                     if self.dataOut.data_cspc != None:
                         pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
                         blocksize += (pts2write_CrossSpectra*datatypeValue*2)
                     if self.dataOut.data_dc != None:
                         pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
                         blocksize += (pts2write_DCchannels*datatypeValue*2)
                     blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.setBasicHeader()
                     processingHeaderSize = 40 # bytes
                     self.processingHeaderObj.dtype = 1 # Spectra
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.processFlags = self.__getProcessFlags()
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
                     self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
                     self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
                     self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
                     if self.processingHeaderObj.totalSpectra > 0:
                         channelList = []
                         for channel in range(self.dataOut.nChannels):
                             channelList.append(channel)
                             channelList.append(channel)
                         pairsList = []
                         if self.dataOut.nPairs > 0:
                             for pair in self.dataOut.pairsList:
                                 pairsList.append(pair[0])
                                 pairsList.append(pair[1])
                         spectraComb = channelList + pairsList
                         spectraComb = numpy.array(spectraComb,dtype="u1")
                         self.processingHeaderObj.spectraComb = spectraComb
                         sizeOfSpcComb = len(spectraComb)
                         processingHeaderSize += sizeOfSpcComb
             #        The processing header should not have information about code
             #        if self.dataOut.code != None:
             #            self.processingHeaderObj.code = self.dataOut.code
             #            self.processingHeaderObj.nCode = self.dataOut.nCode
             #            self.processingHeaderObj.nBaud = self.dataOut.nBaud
             #            nCodeSize = 4 # bytes
             #            nBaudSize = 4 # bytes
             #            codeSize = 4 # bytes
             #            sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)
             #            processingHeaderSize += sizeOfCode
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                         sizeOfFirstHeight = 4
                         sizeOfdeltaHeight = 4
                         sizeOfnHeights = 4
                         sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows
                         processingHeaderSize += sizeOfWindows
                     self.processingHeaderObj.size = processingHeaderSize
             class SpectraHeisWriter(Operation):
             #    set = None
                 setFile = None
                 idblock = None
                 doypath = None
                 subfolder = None
                 def __init__(self):
                     self.wrObj = FITS()
             #        self.dataOut = dataOut
                     self.nTotalBlocks=0
             #        self.set = None
                     self.setFile = None
                     self.idblock = 0
                     self.wrpath = None
                     self.doypath = None
                     self.subfolder = None
                     self.isConfig = False
                 def isNumber(str):
                     """
                     Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                     Excepciones:
                     Si un determinado string no puede ser convertido a numero
                     Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                     Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                     """
                     try:
                         float( str )
                         return True
                     except:
                         return False
                 def setup(self, dataOut, wrpath):
                     if not(os.path.exists(wrpath)):
                         os.mkdir(wrpath)
                     self.wrpath = wrpath
             #        self.setFile = 0
                     self.dataOut = dataOut
                 def putData(self):
                     name= time.localtime( self.dataOut.utctime)
                     ext=".fits"
                     if self.doypath == None:
                        self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
                        self.doypath = os.path.join( self.wrpath, self.subfolder )
                        os.mkdir(self.doypath)
                     if self.setFile == None:
             #           self.set = self.dataOut.set
                        self.setFile = 0
             #        if self.set != self.dataOut.set:
             ##            self.set = self.dataOut.set
             #            self.setFile = 0
                     #make the filename
                     file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
                     filename = os.path.join(self.wrpath,self.subfolder, file)
                     idblock = numpy.array([self.idblock],dtype="int64")
                     header=self.wrObj.cFImage(idblock=idblock,
                                             year=time.gmtime(self.dataOut.utctime).tm_year,
                                             month=time.gmtime(self.dataOut.utctime).tm_mon,
                                             day=time.gmtime(self.dataOut.utctime).tm_mday,
                                             hour=time.gmtime(self.dataOut.utctime).tm_hour,
                                             minute=time.gmtime(self.dataOut.utctime).tm_min,
                                             second=time.gmtime(self.dataOut.utctime).tm_sec)
                     c=3E8
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
                     colList = []
                     colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
                     colList.append(colFreq)
                     nchannel=self.dataOut.nChannels
                     for i in range(nchannel):
                         col = self.wrObj.writeData(name="PCh"+str(i+1),
                                                      format=str(self.dataOut.nFFTPoints)+'E',
                                                       data=10*numpy.log10(self.dataOut.data_spc[i,:]))
                         colList.append(col)
                     data=self.wrObj.Ctable(colList=colList)
                     self.wrObj.CFile(header,data)
                     self.wrObj.wFile(filename)
                     #update the setFile
                     self.setFile += 1
                     self.idblock += 1
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class FITS:
                 name=None
                 format=None
                 array =None
                 data =None
                 thdulist=None
                 prihdr=None
                 hdu=None
                 def __init__(self):
                     pass
                 def setColF(self,name,format,array):
                     self.name=name
                     self.format=format
                     self.array=array
                     a1=numpy.array([self.array],dtype=numpy.float32)
                     self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
                     return self.col1
             #    def setColP(self,name,format,data):
             #        self.name=name
             #        self.format=format
             #        self.data=data
             #        a2=numpy.array([self.data],dtype=numpy.float32)
             #        self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
             #        return self.col2
                 def writeData(self,name,format,data):
                     self.name=name
                     self.format=format
                     self.data=data
                     a2=numpy.array([self.data],dtype=numpy.float32)
                     self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
                     return self.col2
                 def cFImage(self,idblock,year,month,day,hour,minute,second):
                     self.hdu= pyfits.PrimaryHDU(idblock)
                     self.hdu.header.set("Year",year)
                     self.hdu.header.set("Month",month)
                     self.hdu.header.set("Day",day)
                     self.hdu.header.set("Hour",hour)
                     self.hdu.header.set("Minute",minute)
                     self.hdu.header.set("Second",second)
                     return self.hdu
                 def Ctable(self,colList):
                     self.cols=pyfits.ColDefs(colList)
                     self.tbhdu = pyfits.new_table(self.cols)
                     return self.tbhdu
                 def CFile(self,hdu,tbhdu):
                     self.thdulist=pyfits.HDUList([hdu,tbhdu])
                 def wFile(self,filename):
                     if os.path.isfile(filename):
                         os.remove(filename)
                     self.thdulist.writeto(filename)
             class ParameterConf:
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.name = ''
                     self.value = ''
                 def readXml(self, parmElement):
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                 def getElementName(self):
                     return self.ELEMENTNAME
             class Metadata:
                 def __init__(self, filename):
                     self.parmConfObjList = []
                     self.readXml(filename)
                 def readXml(self, filename):
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     self.projectElement = ElementTree().parse(filename)
                     self.project = self.projectElement.tag
                     parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         self.parmConfObjList.append(parmConfObj)
             class FitsWriter(Operation):
                 def __init__(self):
                     self.isConfig = False
                     self.dataBlocksPerFile = None
                     self.blockIndex = 0
                     self.flagIsNewFile = 1
                     self.fitsObj = None
                     self.optchar = 'P'
                     self.ext = '.fits'
                     self.setFile = 0
                 def setFitsHeader(self, dataOut, metadatafile):
                     header_data = pyfits.PrimaryHDU()
                     metadata4fits = Metadata(metadatafile)
                     for parameter in metadata4fits.parmConfObjList:
                         parm_name = parameter.name
                         parm_value = parameter.value
                         if parm_value == 'fromdatadatetime':
                             value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
                         elif parm_value == 'fromdataheights':
                             value = dataOut.nHeights
                         elif parm_value == 'fromdatachannel':
                             value = dataOut.nChannels
                         elif parm_value == 'fromdatasamples':
                             value = dataOut.nFFTPoints
                         else:
                             value = parm_value
                         header_data.header[parm_name] = value
                     header_data.header['NBLOCK'] = self.blockIndex
                     header_data.writeto(self.filename)
                 def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):
                     self.path = path
                     self.dataOut = dataOut
                     self.metadatafile = metadatafile
                     self.dataBlocksPerFile = dataBlocksPerFile
                 def open(self):
                     self.fitsObj = pyfits.open(self.filename, mode='update')
                 def addData(self, data):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATA'])
                     extension.header['UTCTIME'] = self.dataOut.utctime
                     self.fitsObj.append(extension)
                     self.blockIndex += 1
                     self.fitsObj[0].header['NBLOCK'] = self.blockIndex
                     self.write()
                 def write(self):
                     self.fitsObj.flush(verbose=True)
                     self.fitsObj.close()
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, file )
                     self.blockIndex = 0
                     self.filename = filename
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     print 'Writing the file: %s'%self.filename
                     self.setFitsHeader(self.dataOut, self.metadatafile)
                     return 1
                 def writeBlock(self):
                     self.addData(self.dataOut.data_spc)
                     self.flagIsNewFile = 0
                 def __setNewBlock(self):
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.dataBlocksPerFile:
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def putData(self):
                     if self.flagIsNewFile:
                         self.setNextFile()
                     self.writeNextBlock()
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class FitsReader(ProcessingUnit):
                 __TIMEZONE = time.timezone
                 expName = None
                 datetimestr = None
                 utc = None
                 nChannels = None
                 nSamples = None
                 dataBlocksPerFile = None
                 comments = None
                 lastUTTime = None
                 header_dict = None
                 data = None
                 data_header_dict = None
                 def __init__(self):
                     self.isConfig = False
                     self.ext = '.fits'
                     self.setFile = 0
                     self.flagNoMoreFiles = 0
                     self.flagIsNewFile = 1
                     self.flagTimeBlock = None
                     self.fileIndex = None
                     self.filename = None
                     self.fileSize = None
                     self.fitsObj = None
                     self.nReadBlocks = 0
                     self.nTotalBlocks = 0
                     self.dataOut = self.createObjByDefault()
                     self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
                     self.blockIndex = 1
                 def createObjByDefault(self):
                     dataObj = Fits()
                     return dataObj
                 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
                     try:
                         fitsObj = pyfits.open(filename,'readonly')
                     except:
                         raise IOError, "The file %s can't be opened" %(filename)
                     header = fitsObj[0].header
                     struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
                     utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
                     ltc = utc
                     if useLocalTime:
                         ltc -= time.timezone
                     thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
                     thisTime = thisDatetime.time()
                     if not ((startTime <= thisTime) and (endTime > thisTime)):
                         return None
                     return thisDatetime
                 def __setNextFileOnline(self):
                     raise ValueError, "No implemented"
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print "No more Files"
                             return 0
                         filename = self.filenameList[idFile]
             #            if not(self.__verifyFile(filename)):
             #                continue
                         fileSize = os.path.getsize(filename)
                         fitsObj = pyfits.open(filename,'readonly')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fitsObj = fitsObj
                     print "Setting the file: %s"%self.filename
                     return 1
                 def readHeader(self):
                     headerObj = self.fitsObj[0]
                     self.header_dict = headerObj.header
                     self.expName = headerObj.header['EXPNAME']
                     self.datetimestr = headerObj.header['DATETIME']
                     struct_time = time.strptime(headerObj.header['DATETIME'], "%b %d %Y %H:%M:%S")
             #        self.utc = time.mktime(struct_time) - self.__TIMEZONE
                     self.nChannels = headerObj.header['NCHANNEL']
                     self.nSamples = headerObj.header['NSAMPLE']
                     self.dataBlocksPerFile = headerObj.header['NBLOCK']
                     self.comments = headerObj.header['COMMENT']
                 def setNextFile(self):
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         return 0
                     self.readHeader()
                     self.nReadBlocks = 0
                     self.blockIndex = 1
                     return 1
                 def __searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.fits',
                                         walk=True):
                     pathList = []
                     if not walk:
                         pathList.append(path)
                     else:
                         dirList = []
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isDoyFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None
                         thisDate = startDate
                         while(thisDate <= endDate):
                             year = thisDate.timetuple().tm_year
                             doy = thisDate.timetuple().tm_yday
                             matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
                             if len(matchlist) == 0:
                                 thisDate += datetime.timedelta(1)
                                 continue
                             for match in matchlist:
                                 pathList.append(os.path.join(path,match,expLabel))
                             thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
                         return None, None
                     print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
                     filenameList = []
                     datetimeList = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             thisDatetime = self.isFileinThisTime(filename, startTime, endTime, useLocalTime=True)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "Any file was found for the time range %s - %s" %(startTime, endTime)
                         return None, None
                     print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def setup(self, path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=0,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True):
                     if path == None:
                         raise ValueError, "The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if not(online):
                         print "Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
                             print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime())
                             sys.exit(-1)
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                     self.online = online
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "No files"
                         sys.exit(-1)
                 def readBlock(self):
                     dataObj = self.fitsObj[self.blockIndex]
                     self.data = dataObj.data
                     self.data_header_dict = dataObj.header
                     self.utc = self.data_header_dict['UTCTIME']
                     self.flagIsNewFile = 0
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def __jumpToLastBlock(self):
                     raise ValueError, "No implemented"
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.__rdBasicHeader()
                             return 1
                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         time.sleep( self.delay )
                     return 0
                 def __setNewBlock(self):
                     if self.online:
                         self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         return 1
                     self.lastUTTime = self.utc
                     if self.online:
                         if self.__waitNewBlock():
                             return 1
                     if self.nReadBlocks < self.dataBlocksPerFile:
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.utc - self.lastUTTime
                     self.flagTimeBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagTimeBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     if not(self.readNextBlock()):
                         return 0
                     if self.data == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.dataOut.data = self.data
                     self.dataOut.data_header = self.data_header_dict
                     self.dataOut.utctime = self.utc
                     self.dataOut.header = self.header_dict
                     self.dataOut.expName = self.expName
                     self.dataOut.nChannels = self.nChannels
                     self.dataOut.nSamples = self.nSamples
                     self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
                     self.dataOut.comments = self.comments
                     self.dataOut.flagNoData = False
                     return self.dataOut.data
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()

1

'''

1

'''

2

3

$Author: murco $

3

$Author: murco $

4

$Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $

4

$Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $

5

'''

5

'''

6

7

import os, sys

7

import os, sys

8

import glob

8

import glob

9

import time

9

import time

10

import numpy

10

import numpy

11

import fnmatch

11

import fnmatch

12

import time, datetime

12

import time, datetime

13

from xml.etree.ElementTree import Element, SubElement, ElementTree

13

from xml.etree.ElementTree import Element, SubElement, ElementTree

14

try:

14

try:

15

import pyfits

15

import pyfits

16

except:

16

except:

17

print "pyfits module has not been imported, it should be installed to save files in fits format"

17

print "pyfits module has not been imported, it should be installed to save files in fits format"

18

19

from jrodata import *

19

from jrodata import *

20

from jroheaderIO import *

20

from jroheaderIO import *

21

from jroprocessing import *

21

from jroprocessing import *

22

23

LOCALTIME = True #-18000

23

LOCALTIME = True #-18000

24

25

def isNumber(str):

25

def isNumber(str):

26

"""

26

"""

27

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

27

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

28

29

Excepciones:

29

Excepciones:

30

Si un determinado string no puede ser convertido a numero

30

Si un determinado string no puede ser convertido a numero

31

Input:

31

Input:

32

str, string al cual se le analiza para determinar si convertible a un numero o no

32

str, string al cual se le analiza para determinar si convertible a un numero o no

33

34

Return:

34

Return:

35

True : si el string es uno numerico

35

True : si el string es uno numerico

36

False : no es un string numerico

36

False : no es un string numerico

37

"""

37

"""

38

try:

38

try:

39

float( str )

39

float( str )

40

return True

40

return True

41

except:

41

except:

42

return False

42

return False

43

44

def isThisFileinRange(filename, startUTSeconds, endUTSeconds):

44

def isThisFileinRange(filename, startUTSeconds, endUTSeconds):

45

"""

45

"""

46

Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.

46

Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.

47

48

Inputs:

48

Inputs:

49

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

49

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

50

51

startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en

51

startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en

52

segundos contados desde 01/01/1970.

52

segundos contados desde 01/01/1970.

53

endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en

53

endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en

54

segundos contados desde 01/01/1970.

54

segundos contados desde 01/01/1970.

55

56

Return:

56

Return:

57

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

57

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

58

fecha especificado, de lo contrario retorna False.

58

fecha especificado, de lo contrario retorna False.

59

60

Excepciones:

60

Excepciones:

61

Si el archivo no existe o no puede ser abierto

61

Si el archivo no existe o no puede ser abierto

62

Si la cabecera no puede ser leida.

62

Si la cabecera no puede ser leida.

63

64

"""

64

"""

65

basicHeaderObj = BasicHeader(LOCALTIME)

65

basicHeaderObj = BasicHeader(LOCALTIME)

66

67

try:

67

try:

68

fp = open(filename,'rb')

68

fp = open(filename,'rb')

69

except:

69

except:

70

raise IOError, "The file %s can't be opened" %(filename)

70

raise IOError, "The file %s can't be opened" %(filename)

71

72

sts = basicHeaderObj.read(fp)

72

sts = basicHeaderObj.read(fp)

73

fp.close()

73

fp.close()

74

75

if not(sts):

75

if not(sts):

76

print "Skipping the file %s because it has not a valid header" %(filename)

76

print "Skipping the file %s because it has not a valid header" %(filename)

77

return 0

77

return 0

78

79

if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):

79

if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):

80

return 0

80

return 0

81

82

return 1

82

return 1

83

84

def isFileinThisTime(filename, startTime, endTime):

84

def isFileinThisTime(filename, startTime, endTime):

85

"""

85

"""

86

Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.

86

Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.

87

88

Inputs:

88

Inputs:

89

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

89

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

90

91

startTime : tiempo inicial del rango seleccionado en formato datetime.time

91

startTime : tiempo inicial del rango seleccionado en formato datetime.time

92

93

endTime : tiempo final del rango seleccionado en formato datetime.time

93

endTime : tiempo final del rango seleccionado en formato datetime.time

94

95

Return:

95

Return:

96

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

96

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

97

fecha especificado, de lo contrario retorna False.

97

fecha especificado, de lo contrario retorna False.

98

99

Excepciones:

99

Excepciones:

100

Si el archivo no existe o no puede ser abierto

100

Si el archivo no existe o no puede ser abierto

101

Si la cabecera no puede ser leida.

101

Si la cabecera no puede ser leida.

102

103

"""

103

"""

104

105

106

try:

106

try:

107

fp = open(filename,'rb')

107

fp = open(filename,'rb')

108

except:

108

except:

109

raise IOError, "The file %s can't be opened" %(filename)

109

raise IOError, "The file %s can't be opened" %(filename)

110

111

basicHeaderObj = BasicHeader(LOCALTIME)

111

basicHeaderObj = BasicHeader(LOCALTIME)

112

sts = basicHeaderObj.read(fp)

112

sts = basicHeaderObj.read(fp)

113

fp.close()

113

fp.close()

114

115

thisDatetime = basicHeaderObj.datatime

115

thisDatetime = basicHeaderObj.datatime

116

thisTime = basicHeaderObj.datatime.time()

116

thisTime = basicHeaderObj.datatime.time()

117

118

if not(sts):

118

if not(sts):

119

print "Skipping the file %s because it has not a valid header" %(filename)

119

print "Skipping the file %s because it has not a valid header" %(filename)

120

return None

120

return None

121

122

if not ((startTime <= thisTime) and (endTime > thisTime)):

122

if not ((startTime <= thisTime) and (endTime > thisTime)):

123

return None

123

return None

124

125

return thisDatetime

125

return thisDatetime

126

127

def getFileFromSet(path,ext,set):

127

def getFileFromSet(path,ext,set):

128

validFilelist = []

128

validFilelist = []

129

fileList = os.listdir(path)

129

fileList = os.listdir(path)

130

131

# 0 1234 567 89A BCDE

131

# 0 1234 567 89A BCDE

132

# H YYYY DDD SSS .ext

132

# H YYYY DDD SSS .ext

133

134

for file in fileList:

134

for file in fileList:

135

try:

135

try:

136

year = int(file[1:5])

136

year = int(file[1:5])

137

doy = int(file[5:8])

137

doy = int(file[5:8])

138

139

140

except:

140

except:

141

continue

141

continue

142

143

if (os.path.splitext(file)[-1].lower() != ext.lower()):

143

if (os.path.splitext(file)[-1].lower() != ext.lower()):

144

continue

144

continue

145

146

validFilelist.append(file)

146

validFilelist.append(file)

147

148

myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))

148

myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))

149

150

if len(myfile)!= 0:

150

if len(myfile)!= 0:

151

return myfile[0]

151

return myfile[0]

152

else:

152

else:

153

filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())

153

filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())

154

print 'the filename %s does not exist'%filename

154

print 'the filename %s does not exist'%filename

155

print '...going to the last file: '

155

print '...going to the last file: '

156

157

if validFilelist:

157

if validFilelist:

158

validFilelist = sorted( validFilelist, key=str.lower )

158

validFilelist = sorted( validFilelist, key=str.lower )

159

return validFilelist[-1]

159

return validFilelist[-1]

160

161

return None

161

return None

162

163

164

def getlastFileFromPath(path, ext):

164

def getlastFileFromPath(path, ext):

165

"""

165

"""

166

Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"

166

Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"

167

al final de la depuracion devuelve el ultimo file de la lista que quedo.

167

al final de la depuracion devuelve el ultimo file de la lista que quedo.

168

169

Input:

169

Input:

170

fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta

170

fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta

171

ext : extension de los files contenidos en una carpeta

171

ext : extension de los files contenidos en una carpeta

172

173

Return:

173

Return:

174

El ultimo file de una determinada carpeta, no se considera el path.

174

El ultimo file de una determinada carpeta, no se considera el path.

175

"""

175

"""

176

validFilelist = []

176

validFilelist = []

177

fileList = os.listdir(path)

177

fileList = os.listdir(path)

178

179

# 0 1234 567 89A BCDE

179

# 0 1234 567 89A BCDE

180

# H YYYY DDD SSS .ext

180

# H YYYY DDD SSS .ext

181

182

for file in fileList:

182

for file in fileList:

183

try:

183

try:

184

year = int(file[1:5])

184

year = int(file[1:5])

185

doy = int(file[5:8])

185

doy = int(file[5:8])

186

187

188

except:

188

except:

189

continue

189

continue

190

191

if (os.path.splitext(file)[-1].lower() != ext.lower()):

191

if (os.path.splitext(file)[-1].lower() != ext.lower()):

192

continue

192

continue

193

194

validFilelist.append(file)

194

validFilelist.append(file)

195

196

if validFilelist:

196

if validFilelist:

197

validFilelist = sorted( validFilelist, key=str.lower )

197

validFilelist = sorted( validFilelist, key=str.lower )

198

return validFilelist[-1]

198

return validFilelist[-1]

199

200

return None

200

return None

201

202

def checkForRealPath(path, foldercounter, year, doy, set, ext):

202

def checkForRealPath(path, foldercounter, year, doy, set, ext):

203

"""

203

"""

204

Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,

204

Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,

205

Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar

205

Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar

206

el path exacto de un determinado file.

206

el path exacto de un determinado file.

207

208

Example :

208

Example :

209

nombre correcto del file es .../.../D2009307/P2009307367.ext

209

nombre correcto del file es .../.../D2009307/P2009307367.ext

210

211

Entonces la funcion prueba con las siguientes combinaciones

211

Entonces la funcion prueba con las siguientes combinaciones

212

.../.../y2009307367.ext

212

.../.../y2009307367.ext

213

.../.../Y2009307367.ext

213

.../.../Y2009307367.ext

214

.../.../x2009307/y2009307367.ext

214

.../.../x2009307/y2009307367.ext

215

.../.../x2009307/Y2009307367.ext

215

.../.../x2009307/Y2009307367.ext

216

.../.../X2009307/y2009307367.ext

216

.../.../X2009307/y2009307367.ext

217

.../.../X2009307/Y2009307367.ext

217

.../.../X2009307/Y2009307367.ext

218

siendo para este caso, la ultima combinacion de letras, identica al file buscado

218

siendo para este caso, la ultima combinacion de letras, identica al file buscado

219

220

Return:

220

Return:

221

Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file

221

Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file

222

caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas

222

caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas

223

para el filename

223

para el filename

224

"""

224

"""

225

fullfilename = None

225

fullfilename = None

226

find_flag = False

226

find_flag = False

227

filename = None

227

filename = None

228

229

prefixDirList = [None,'d','D']

229

prefixDirList = [None,'d','D']

230

if ext.lower() == ".r": #voltage

230

if ext.lower() == ".r": #voltage

231

prefixFileList = ['d','D']

231

prefixFileList = ['d','D']

232

elif ext.lower() == ".pdata": #spectra

232

elif ext.lower() == ".pdata": #spectra

233

prefixFileList = ['p','P']

233

prefixFileList = ['p','P']

234

else:

234

else:

235

return None, filename

235

return None, filename

236

237

#barrido por las combinaciones posibles

237

#barrido por las combinaciones posibles

238

for prefixDir in prefixDirList:

238

for prefixDir in prefixDirList:

239

thispath = path

239

thispath = path

240

if prefixDir != None:

240

if prefixDir != None:

241

#formo el nombre del directorio xYYYYDDD (x=d o x=D)

241

#formo el nombre del directorio xYYYYDDD (x=d o x=D)

242

if foldercounter == 0:

242

if foldercounter == 0:

243

thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))

243

thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))

244

else:

244

else:

245

thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))

245

thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))

246

for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"

246

for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"

247

filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext

247

filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext

248

fullfilename = os.path.join( thispath, filename ) #formo el path completo

248

fullfilename = os.path.join( thispath, filename ) #formo el path completo

249

250

if os.path.exists( fullfilename ): #verifico que exista

250

if os.path.exists( fullfilename ): #verifico que exista

251

find_flag = True

251

find_flag = True

252

break

252

break

253

if find_flag:

253

if find_flag:

254

break

254

break

255

256

if not(find_flag):

256

if not(find_flag):

257

return None, filename

257

return None, filename

258

259

return fullfilename, filename

259

return fullfilename, filename

260

261

def isDoyFolder(folder):

261

def isDoyFolder(folder):

262

try:

262

try:

263

year = int(folder[1:5])

263

year = int(folder[1:5])

264

except:

264

except:

265

return 0

265

return 0

266

267

try:

267

try:

268

doy = int(folder[5:8])

268

doy = int(folder[5:8])

269

except:

269

except:

270

return 0

270

return 0

271

272

return 1

272

return 1

273

274

class JRODataIO:

274

class JRODataIO:

275

276

c = 3E8

276

c = 3E8

277

278

isConfig = False

278

isConfig = False

279

280

basicHeaderObj = BasicHeader(LOCALTIME)

280

basicHeaderObj = BasicHeader(LOCALTIME)

281

282

systemHeaderObj = SystemHeader()

282

systemHeaderObj = SystemHeader()

283

284

radarControllerHeaderObj = RadarControllerHeader()

284

radarControllerHeaderObj = RadarControllerHeader()

285

286

processingHeaderObj = ProcessingHeader()

286

processingHeaderObj = ProcessingHeader()

287

288

online = 0

288

online = 0

289

290

dtype = None

290

dtype = None

291

292

pathList = []

292

pathList = []

293

294

filenameList = []

294

filenameList = []

295

296

filename = None

296

filename = None

297

298

ext = None

298

ext = None

299

300

flagIsNewFile = 1

300

flagIsNewFile = 1

301

302

flagTimeBlock = 0

302

flagTimeBlock = 0

303

304

flagIsNewBlock = 0

304

flagIsNewBlock = 0

305

306

fp = None

306

fp = None

307

308

firstHeaderSize = 0

308

firstHeaderSize = 0

309

310

basicHeaderSize = 24

310

basicHeaderSize = 24

311

312

versionFile = 1103

312

versionFile = 1103

313

314

fileSize = None

314

fileSize = None

315

316

ippSeconds = None

316

ippSeconds = None

317

318

fileSizeByHeader = None

318

fileSizeByHeader = None

319

320

fileIndex = None

320

fileIndex = None

321

322

profileIndex = None

322

profileIndex = None

323

324

blockIndex = None

324

blockIndex = None

325

326

nTotalBlocks = None

326

nTotalBlocks = None

327

328

maxTimeStep = 30

328

maxTimeStep = 30

329

330

lastUTTime = None

330

lastUTTime = None

331

332

datablock = None

332

datablock = None

333

334

dataOut = None

334

dataOut = None

335

336

blocksize = None

336

blocksize = None

337

338

def __init__(self):

338

def __init__(self):

339

340

raise ValueError, "Not implemented"

340

raise ValueError, "Not implemented"

341

342

def run(self):

342

def run(self):

343

344

raise ValueError, "Not implemented"

344

raise ValueError, "Not implemented"

345

346

def getOutput(self):

346

def getOutput(self):

347

348

return self.dataOut

348

return self.dataOut

349

350

class JRODataReader(JRODataIO, ProcessingUnit):

350

class JRODataReader(JRODataIO, ProcessingUnit):

351

352

nReadBlocks = 0

352

nReadBlocks = 0

353

354

delay = 10 #number of seconds waiting a new file

354

delay = 10 #number of seconds waiting a new file

355

356

nTries = 3 #quantity tries

356

nTries = 3 #quantity tries

357

358

nFiles = 3 #number of files for searching

358

nFiles = 3 #number of files for searching

359

360

path = None

360

path = None

361

362

foldercounter = 0

362

foldercounter = 0

363

364

flagNoMoreFiles = 0

364

flagNoMoreFiles = 0

365

366

datetimeList = []

366

datetimeList = []

367

368

__isFirstTimeOnline = 1

368

__isFirstTimeOnline = 1

369

370

__printInfo = True

370

__printInfo = True

371

372

profileIndex = None

372

profileIndex = None

373

374

def __init__(self):

374

def __init__(self):

375

376

"""

376

"""

377

378

"""

378

"""

379

380

raise ValueError, "This method has not been implemented"

380

raise ValueError, "This method has not been implemented"

381

382

383

def createObjByDefault(self):

383

def createObjByDefault(self):

384

"""

384

"""

385

386

"""

386

"""

387

raise ValueError, "This method has not been implemented"

387

raise ValueError, "This method has not been implemented"

388

389

def getBlockDimension(self):

389

def getBlockDimension(self):

390

391

raise ValueError, "No implemented"

391

raise ValueError, "No implemented"

392

393

def __searchFilesOffLine(self,

393

def __searchFilesOffLine(self,

394

path,

394

path,

395

startDate,

395

startDate,

396

endDate,

396

endDate,

397

startTime=datetime.time(0,0,0),

397

startTime=datetime.time(0,0,0),

398

endTime=datetime.time(23,59,59),

398

endTime=datetime.time(23,59,59),

399

set=None,

399

set=None,

400

expLabel='',

400

expLabel='',

401

ext='.r',

401

ext='.r',

402

walk=True):

402

walk=True):

403

404

pathList = []

404

pathList = []

405

406

if not walk:

406

if not walk:

407

pathList.append(path)

407

pathList.append(path)

408

409

else:

409

else:

410

dirList = []

410

dirList = []

411

for thisPath in os.listdir(path):

411

for thisPath in os.listdir(path):

412

if not os.path.isdir(os.path.join(path,thisPath)):

412

if not os.path.isdir(os.path.join(path,thisPath)):

413

continue

413

continue

414

if not isDoyFolder(thisPath):

414

if not isDoyFolder(thisPath):

415

continue

415

continue

416

417

dirList.append(thisPath)

417

dirList.append(thisPath)

418

419

if not(dirList):

419

if not(dirList):

420

return None, None

420

return None, None

421

422

thisDate = startDate

422

thisDate = startDate

423

424

while(thisDate <= endDate):

424

while(thisDate <= endDate):

425

year = thisDate.timetuple().tm_year

425

year = thisDate.timetuple().tm_year

426

doy = thisDate.timetuple().tm_yday

426

doy = thisDate.timetuple().tm_yday

427

428

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

428

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

429

if len(matchlist) == 0:

429

if len(matchlist) == 0:

430

thisDate += datetime.timedelta(1)

430

thisDate += datetime.timedelta(1)

431

continue

431

continue

432

for match in matchlist:

432

for match in matchlist:

433

pathList.append(os.path.join(path,match,expLabel))

433

pathList.append(os.path.join(path,match,expLabel))

434

435

thisDate += datetime.timedelta(1)

435

thisDate += datetime.timedelta(1)

436

437

if pathList == []:

437

if pathList == []:

438

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

438

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

439

return None, None

439

return None, None

440

441

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

441

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

442

443

filenameList = []

443

filenameList = []

444

datetimeList = []

444

datetimeList = []

445

446

for i in range(len(pathList)):

446

for i in range(len(pathList)):

447

448

thisPath = pathList[i]

448

thisPath = pathList[i]

449

450

fileList = glob.glob1(thisPath, "*%s" %ext)

450

fileList = glob.glob1(thisPath, "*%s" %ext)

451

fileList.sort()

451

fileList.sort()

452

453

for file in fileList:

453

for file in fileList:

454

455

filename = os.path.join(thisPath,file)

455

filename = os.path.join(thisPath,file)

456

thisDatetime = isFileinThisTime(filename, startTime, endTime)

456

thisDatetime = isFileinThisTime(filename, startTime, endTime)

457

458

if not(thisDatetime):

458

if not(thisDatetime):

459

continue

459

continue

460

461

filenameList.append(filename)

461

filenameList.append(filename)

462

datetimeList.append(thisDatetime)

462

datetimeList.append(thisDatetime)

463

464

if not(filenameList):

464

if not(filenameList):

465

print "Any file was found for the time range %s - %s" %(startTime, endTime)

465

print "Any file was found for the time range %s - %s" %(startTime, endTime)

466

return None, None

466

return None, None

467

468

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

468

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

469

print

469

print

470

471

for i in range(len(filenameList)):

471

for i in range(len(filenameList)):

472

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

472

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

473

474

self.filenameList = filenameList

474

self.filenameList = filenameList

475

self.datetimeList = datetimeList

475

self.datetimeList = datetimeList

476

477

return pathList, filenameList

477

return pathList, filenameList

478

479

def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):

479

def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):

480

481

"""

481

"""

482

Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y

482

Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y

483

devuelve el archivo encontrado ademas de otros datos.

483

devuelve el archivo encontrado ademas de otros datos.

484

485

Input:

485

Input:

486

path : carpeta donde estan contenidos los files que contiene data

486

path : carpeta donde estan contenidos los files que contiene data

487

488

expLabel : Nombre del subexperimento (subfolder)

488

expLabel : Nombre del subexperimento (subfolder)

489

490

ext : extension de los files

490

ext : extension de los files

491

492

walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)

492

walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)

493

494

Return:

494

Return:

495

directory : eL directorio donde esta el file encontrado

495

directory : eL directorio donde esta el file encontrado

496

filename : el ultimo file de una determinada carpeta

496

filename : el ultimo file de una determinada carpeta

497

year : el anho

497

year : el anho

498

doy : el numero de dia del anho

498

doy : el numero de dia del anho

499

set : el set del archivo

499

set : el set del archivo

500

501

502

"""

502

"""

503

dirList = []

503

dirList = []

504

505

if not walk:

505

if not walk:

506

fullpath = path

506

fullpath = path

507

foldercounter = 0

507

foldercounter = 0

508

else:

508

else:

509

#Filtra solo los directorios

509

#Filtra solo los directorios

510

for thisPath in os.listdir(path):

510

for thisPath in os.listdir(path):

511

if not os.path.isdir(os.path.join(path,thisPath)):

511

if not os.path.isdir(os.path.join(path,thisPath)):

512

continue

512

continue

513

if not isDoyFolder(thisPath):

513

if not isDoyFolder(thisPath):

514

continue

514

continue

515

516

dirList.append(thisPath)

516

dirList.append(thisPath)

517

518

if not(dirList):

518

if not(dirList):

519

return None, None, None, None, None, None

519

return None, None, None, None, None, None

520

521

dirList = sorted( dirList, key=str.lower )

521

dirList = sorted( dirList, key=str.lower )

522

523

doypath = dirList[-1]

523

doypath = dirList[-1]

524

foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0

524

foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0

525

fullpath = os.path.join(path, doypath, expLabel)

525

fullpath = os.path.join(path, doypath, expLabel)

526

527

528

print "%s folder was found: " %(fullpath )

528

print "%s folder was found: " %(fullpath )

529

530

if set == None:

530

if set == None:

531

filename = getlastFileFromPath(fullpath, ext)

531

filename = getlastFileFromPath(fullpath, ext)

532

else:

532

else:

533

filename = getFileFromSet(fullpath, ext, set)

533

filename = getFileFromSet(fullpath, ext, set)

534

535

if not(filename):

535

if not(filename):

536

return None, None, None, None, None, None

536

return None, None, None, None, None, None

537

538

print "%s file was found" %(filename)

538

print "%s file was found" %(filename)

539

540

if not(self.__verifyFile(os.path.join(fullpath, filename))):

540

if not(self.__verifyFile(os.path.join(fullpath, filename))):

541

return None, None, None, None, None, None

541

return None, None, None, None, None, None

542

543

year = int( filename[1:5] )

543

year = int( filename[1:5] )

544

doy = int( filename[5:8] )

544

doy = int( filename[5:8] )

545

set = int( filename[8:11] )

545

set = int( filename[8:11] )

546

547

return fullpath, foldercounter, filename, year, doy, set

547

return fullpath, foldercounter, filename, year, doy, set

548

549

def __setNextFileOffline(self):

549

def __setNextFileOffline(self):

550

551

idFile = self.fileIndex

551

idFile = self.fileIndex

552

553

while (True):

553

while (True):

554

idFile += 1

554

idFile += 1

555

if not(idFile < len(self.filenameList)):

555

if not(idFile < len(self.filenameList)):

556

self.flagNoMoreFiles = 1

556

self.flagNoMoreFiles = 1

557

print "No more Files"

557

print "No more Files"

558

return 0

558

return 0

559

560

filename = self.filenameList[idFile]

560

filename = self.filenameList[idFile]

561

562

if not(self.__verifyFile(filename)):

562

if not(self.__verifyFile(filename)):

563

continue

563

continue

564

565

fileSize = os.path.getsize(filename)

565

fileSize = os.path.getsize(filename)

566

fp = open(filename,'rb')

566

fp = open(filename,'rb')

567

break

567

break

568

569

self.flagIsNewFile = 1

569

self.flagIsNewFile = 1

570

self.fileIndex = idFile

570

self.fileIndex = idFile

571

self.filename = filename

571

self.filename = filename

572

self.fileSize = fileSize

572

self.fileSize = fileSize

573

self.fp = fp

573

self.fp = fp

574

575

print "Setting the file: %s"%self.filename

575

print "Setting the file: %s"%self.filename

576

577

return 1

577

return 1

578

579

def __setNextFileOnline(self):

579

def __setNextFileOnline(self):

580

"""

580

"""

581

Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si

581

Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si

582

no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files

582

no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files

583

siguientes.

583

siguientes.

584

585

Affected:

585

Affected:

586

self.flagIsNewFile

586

self.flagIsNewFile

587

self.filename

587

self.filename

588

self.fileSize

588

self.fileSize

589

self.fp

589

self.fp

590

self.set

590

self.set

591

self.flagNoMoreFiles

591

self.flagNoMoreFiles

592

593

Return:

593

Return:

594

0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado

594

0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado

595

1 : si el file fue abierto con exito y esta listo a ser leido

595

1 : si el file fue abierto con exito y esta listo a ser leido

596

597

Excepciones:

597

Excepciones:

598

Si un determinado file no puede ser abierto

598

Si un determinado file no puede ser abierto

599

"""

599

"""

600

nFiles = 0

600

nFiles = 0

601

fileOk_flag = False

601

fileOk_flag = False

602

firstTime_flag = True

602

firstTime_flag = True

603

604

self.set += 1

604

self.set += 1

605

606

if self.set > 999:

606

if self.set > 999:

607

self.set = 0

607

self.set = 0

608

self.foldercounter += 1

608

self.foldercounter += 1

609

610

#busca el 1er file disponible

610

#busca el 1er file disponible

611

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

611

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

612

if fullfilename:

612

if fullfilename:

613

if self.__verifyFile(fullfilename, False):

613

if self.__verifyFile(fullfilename, False):

614

fileOk_flag = True

614

fileOk_flag = True

615

616

#si no encuentra un file entonces espera y vuelve a buscar

616

#si no encuentra un file entonces espera y vuelve a buscar

617

if not(fileOk_flag):

617

if not(fileOk_flag):

618

for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles

618

for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles

619

620

if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces

620

if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces

621

tries = self.nTries

621

tries = self.nTries

622

else:

622

else:

623

tries = 1 #si no es la 1era vez entonces solo lo hace una vez

623

tries = 1 #si no es la 1era vez entonces solo lo hace una vez

624

625

for nTries in range( tries ):

625

for nTries in range( tries ):

626

if firstTime_flag:

626

if firstTime_flag:

627

print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )

627

print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )

628

time.sleep( self.delay )

628

time.sleep( self.delay )

629

else:

629

else:

630

print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)

630

print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)

631

632

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

632

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

633

if fullfilename:

633

if fullfilename:

634

if self.__verifyFile(fullfilename):

634

if self.__verifyFile(fullfilename):

635

fileOk_flag = True

635

fileOk_flag = True

636

break

636

break

637

638

if fileOk_flag:

638

if fileOk_flag:

639

break

639

break

640

641

firstTime_flag = False

641

firstTime_flag = False

642

643

print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename

643

print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename

644

self.set += 1

644

self.set += 1

645

646

if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta

646

if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta

647

self.set = 0

647

self.set = 0

648

self.doy += 1

648

self.doy += 1

649

self.foldercounter = 0

649

self.foldercounter = 0

650

651

if fileOk_flag:

651

if fileOk_flag:

652

self.fileSize = os.path.getsize( fullfilename )

652

self.fileSize = os.path.getsize( fullfilename )

653

self.filename = fullfilename

653

self.filename = fullfilename

654

self.flagIsNewFile = 1

654

self.flagIsNewFile = 1

655

if self.fp != None: self.fp.close()

655

if self.fp != None: self.fp.close()

656

self.fp = open(fullfilename, 'rb')

656

self.fp = open(fullfilename, 'rb')

657

self.flagNoMoreFiles = 0

657

self.flagNoMoreFiles = 0

658

print 'Setting the file: %s' % fullfilename

658

print 'Setting the file: %s' % fullfilename

659

else:

659

else:

660

self.fileSize = 0

660

self.fileSize = 0

661

self.filename = None

661

self.filename = None

662

self.flagIsNewFile = 0

662

self.flagIsNewFile = 0

663

self.fp = None

663

self.fp = None

664

self.flagNoMoreFiles = 1

664

self.flagNoMoreFiles = 1

665

print 'No more Files'

665

print 'No more Files'

666

667

return fileOk_flag

667

return fileOk_flag

668

669

670

def setNextFile(self):

670

def setNextFile(self):

671

if self.fp != None:

671

if self.fp != None:

672

self.fp.close()

672

self.fp.close()

673

674

if self.online:

674

if self.online:

675

newFile = self.__setNextFileOnline()

675

newFile = self.__setNextFileOnline()

676

else:

676

else:

677

newFile = self.__setNextFileOffline()

677

newFile = self.__setNextFileOffline()

678

679

if not(newFile):

679

if not(newFile):

680

return 0

680

return 0

681

682

self.__readFirstHeader()

682

self.__readFirstHeader()

683

self.nReadBlocks = 0

683

self.nReadBlocks = 0

684

return 1

684

return 1

685

686

def __waitNewBlock(self):

686

def __waitNewBlock(self):

687

"""

687

"""

688

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

688

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

689

690

Si el modo de lectura es OffLine siempre retorn 0

690

Si el modo de lectura es OffLine siempre retorn 0

691

"""

691

"""

692

if not self.online:

692

if not self.online:

693

return 0

693

return 0

694

695

if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):

695

if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):

696

return 0

696

return 0

697

698

currentPointer = self.fp.tell()

698

currentPointer = self.fp.tell()

699

700

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

700

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

701

702

for nTries in range( self.nTries ):

702

for nTries in range( self.nTries ):

703

704

self.fp.close()

704

self.fp.close()

705

self.fp = open( self.filename, 'rb' )

705

self.fp = open( self.filename, 'rb' )

706

self.fp.seek( currentPointer )

706

self.fp.seek( currentPointer )

707

708

self.fileSize = os.path.getsize( self.filename )

708

self.fileSize = os.path.getsize( self.filename )

709

currentSize = self.fileSize - currentPointer

709

currentSize = self.fileSize - currentPointer

710

711

if ( currentSize >= neededSize ):

711

if ( currentSize >= neededSize ):

712

self.__rdBasicHeader()

712

self.__rdBasicHeader()

713

return 1

713

return 1

714

715

if self.fileSize == self.fileSizeByHeader:

715

if self.fileSize == self.fileSizeByHeader:

716

# self.flagEoF = True

716

# self.flagEoF = True

717

return 0

717

return 0

718

719

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

719

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

720

time.sleep( self.delay )

720

time.sleep( self.delay )

721

722

723

return 0

723

return 0

724

725

def waitDataBlock(self,pointer_location):

726

727

currentPointer = pointer_location

728

729

neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize

730

731

for nTries in range( self.nTries ):

732

self.fp.close()

733

self.fp = open( self.filename, 'rb' )

734

self.fp.seek( currentPointer )

735

736

self.fileSize = os.path.getsize( self.filename )

737

currentSize = self.fileSize - currentPointer

738

739

if ( currentSize >= neededSize ):

740

return 1

741

742

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

743

time.sleep( self.delay )

744

745

return 0

746

747

725

def __jumpToLastBlock(self):

748

def __jumpToLastBlock(self):

726

749

727

if not(self.__isFirstTimeOnline):

750

if not(self.__isFirstTimeOnline):

728

return

751

return

729

752

730

csize = self.fileSize - self.fp.tell()

753

csize = self.fileSize - self.fp.tell()

731

blocksize = self.processingHeaderObj.blockSize

754

blocksize = self.processingHeaderObj.blockSize

732

755

733

#salta el primer bloque de datos

756

#salta el primer bloque de datos

734

if csize > self.processingHeaderObj.blockSize:

757

if csize > self.processingHeaderObj.blockSize:

735

self.fp.seek(self.fp.tell() + blocksize)

758

self.fp.seek(self.fp.tell() + blocksize)

736

else:

759

else:

737

return

760

return

738

761

739

csize = self.fileSize - self.fp.tell()

762

csize = self.fileSize - self.fp.tell()

740

neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

763

neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

741

while True:

764

while True:

742

765

743

if self.fp.tell()<self.fileSize:

766

if self.fp.tell()<self.fileSize:

744

self.fp.seek(self.fp.tell() + neededsize)

767

self.fp.seek(self.fp.tell() + neededsize)

745

else:

768

else:

746

self.fp.seek(self.fp.tell() - neededsize)

769

self.fp.seek(self.fp.tell() - neededsize)

747

break

770

break

748

771

749

# csize = self.fileSize - self.fp.tell()

772

# csize = self.fileSize - self.fp.tell()

750

# neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

773

# neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

751

# factor = int(csize/neededsize)

774

# factor = int(csize/neededsize)

752

# if factor > 0:

775

# if factor > 0:

753

# self.fp.seek(self.fp.tell() + factor*neededsize)

776

# self.fp.seek(self.fp.tell() + factor*neededsize)

754

777

755

self.flagIsNewFile = 0

778

self.flagIsNewFile = 0

756

self.__isFirstTimeOnline = 0

779

self.__isFirstTimeOnline = 0

757

780

758

781

759

def __setNewBlock(self):

782

def __setNewBlock(self):

760

783

761

if self.fp == None:

784

if self.fp == None:

762

return 0

785

return 0

763

786

764

if self.online:

787

if self.online:

765

self.__jumpToLastBlock()

788

self.__jumpToLastBlock()

766

789

767

if self.flagIsNewFile:

790

if self.flagIsNewFile:

768

return 1

791

return 1

769

792

770

self.lastUTTime = self.basicHeaderObj.utc

793

self.lastUTTime = self.basicHeaderObj.utc

771

currentSize = self.fileSize - self.fp.tell()

794

currentSize = self.fileSize - self.fp.tell()

772

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

795

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

773

796

774

if (currentSize >= neededSize):

797

if (currentSize >= neededSize):

775

self.__rdBasicHeader()

798

self.__rdBasicHeader()

776

return 1

799

return 1

777

800

778

if self.__waitNewBlock():

801

if self.__waitNewBlock():

779

return 1

802

return 1

780

803

781

if not(self.setNextFile()):

804

if not(self.setNextFile()):

782

return 0

805

return 0

783

806

784

deltaTime = self.basicHeaderObj.utc - self.lastUTTime #

807

deltaTime = self.basicHeaderObj.utc - self.lastUTTime #

785

808

786

self.flagTimeBlock = 0

809

self.flagTimeBlock = 0

787

810

788

if deltaTime > self.maxTimeStep:

811

if deltaTime > self.maxTimeStep:

789

self.flagTimeBlock = 1

812

self.flagTimeBlock = 1

790

813

791

return 1

814

return 1

792

815

793

816

794

def readNextBlock(self):

817

def readNextBlock(self):

795

if not(self.__setNewBlock()):

818

if not(self.__setNewBlock()):

796

return 0

819

return 0

797

820

798

if not(self.readBlock()):

821

if not(self.readBlock()):

799

return 0

822

return 0

800

823

801

return 1

824

return 1

802

825

803

def __rdProcessingHeader(self, fp=None):

826

def __rdProcessingHeader(self, fp=None):

804

if fp == None:

827

if fp == None:

805

fp = self.fp

828

fp = self.fp

806

829

807

self.processingHeaderObj.read(fp)

830

self.processingHeaderObj.read(fp)

808

831

809

def __rdRadarControllerHeader(self, fp=None):

832

def __rdRadarControllerHeader(self, fp=None):

810

if fp == None:

833

if fp == None:

811

fp = self.fp

834

fp = self.fp

812

835

813

self.radarControllerHeaderObj.read(fp)

836

self.radarControllerHeaderObj.read(fp)

814

837

815

def __rdSystemHeader(self, fp=None):

838

def __rdSystemHeader(self, fp=None):

816

if fp == None:

839

if fp == None:

817

fp = self.fp

840

fp = self.fp

818

841

819

self.systemHeaderObj.read(fp)

842

self.systemHeaderObj.read(fp)

820

843

821

def __rdBasicHeader(self, fp=None):

844

def __rdBasicHeader(self, fp=None):

822

if fp == None:

845

if fp == None:

823

fp = self.fp

846

fp = self.fp

824

847

825

self.basicHeaderObj.read(fp)

848

self.basicHeaderObj.read(fp)

826

849

827

850

828

def __readFirstHeader(self):

851

def __readFirstHeader(self):

829

self.__rdBasicHeader()

852

self.__rdBasicHeader()

830

self.__rdSystemHeader()

853

self.__rdSystemHeader()

831

self.__rdRadarControllerHeader()

854

self.__rdRadarControllerHeader()

832

self.__rdProcessingHeader()

855

self.__rdProcessingHeader()

833

856

834

self.firstHeaderSize = self.basicHeaderObj.size

857

self.firstHeaderSize = self.basicHeaderObj.size

835

858

836

datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

859

datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

837

if datatype == 0:

860

if datatype == 0:

838

datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])

861

datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])

839

elif datatype == 1:

862

elif datatype == 1:

840

datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])

863

datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])

841

elif datatype == 2:

864

elif datatype == 2:

842

datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])

865

datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])

843

elif datatype == 3:

866

elif datatype == 3:

844

datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])

867

datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])

845

elif datatype == 4:

868

elif datatype == 4:

846

datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])

869

datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])

847

elif datatype == 5:

870

elif datatype == 5:

848

datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])

871

datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])

849

else:

872

else:

850

raise ValueError, 'Data type was not defined'

873

raise ValueError, 'Data type was not defined'

851

874

852

self.dtype = datatype_str

875

self.dtype = datatype_str

853

self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c

876

self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c

854

self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)

877

self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)

855

# self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)

878

# self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)

856

# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)

879

# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)

857

self.getBlockDimension()

880

self.getBlockDimension()

858

881

859

882

860

def __verifyFile(self, filename, msgFlag=True):

883

def __verifyFile(self, filename, msgFlag=True):

861

msg = None

884

msg = None

862

try:

885

try:

863

fp = open(filename, 'rb')

886

fp = open(filename, 'rb')

864

currentPosition = fp.tell()

887

currentPosition = fp.tell()

865

except:

888

except:

866

if msgFlag:

889

if msgFlag:

867

print "The file %s can't be opened" % (filename)

890

print "The file %s can't be opened" % (filename)

868

return False

891

return False

869

892

870

neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize

893

neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize

871

894

872

if neededSize == 0:

895

if neededSize == 0:

873

basicHeaderObj = BasicHeader(LOCALTIME)

896

basicHeaderObj = BasicHeader(LOCALTIME)

874

systemHeaderObj = SystemHeader()

897

systemHeaderObj = SystemHeader()

875

radarControllerHeaderObj = RadarControllerHeader()

898

radarControllerHeaderObj = RadarControllerHeader()

876

processingHeaderObj = ProcessingHeader()

899

processingHeaderObj = ProcessingHeader()

877

900

878

try:

901

try:

879

if not( basicHeaderObj.read(fp) ): raise IOError

902

if not( basicHeaderObj.read(fp) ): raise IOError

880

if not( systemHeaderObj.read(fp) ): raise IOError

903

if not( systemHeaderObj.read(fp) ): raise IOError

881

if not( radarControllerHeaderObj.read(fp) ): raise IOError

904

if not( radarControllerHeaderObj.read(fp) ): raise IOError

882

if not( processingHeaderObj.read(fp) ): raise IOError

905

if not( processingHeaderObj.read(fp) ): raise IOError

883

data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

906

data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

884

907

885

neededSize = processingHeaderObj.blockSize + basicHeaderObj.size

908

neededSize = processingHeaderObj.blockSize + basicHeaderObj.size

886

909

887

except:

910

except:

888

if msgFlag:

911

if msgFlag:

889

print "\tThe file %s is empty or it hasn't enough data" % filename

912

print "\tThe file %s is empty or it hasn't enough data" % filename

890

913

891

fp.close()

914

fp.close()

892

return False

915

return False

893

else:

916

else:

894

msg = "\tSkipping the file %s due to it hasn't enough data" %filename

917

msg = "\tSkipping the file %s due to it hasn't enough data" %filename

895

918

896

fp.close()

919

fp.close()

897

fileSize = os.path.getsize(filename)

920

fileSize = os.path.getsize(filename)

898

currentSize = fileSize - currentPosition

921

currentSize = fileSize - currentPosition

899

if currentSize < neededSize:

922

if currentSize < neededSize:

900

if msgFlag and (msg != None):

923

if msgFlag and (msg != None):

901

print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename

924

print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename

902

return False

925

return False

903

926

904

return True

927

return True

905

928

906

def setup(self,

929

def setup(self,

907

path=None,

930

path=None,

908

startDate=None,

931

startDate=None,

909

endDate=None,

932

endDate=None,

910

startTime=datetime.time(0,0,0),

933

startTime=datetime.time(0,0,0),

911

endTime=datetime.time(23,59,59),

934

endTime=datetime.time(23,59,59),

912

set=None,

935

set=None,

913

expLabel = "",

936

expLabel = "",

914

ext = None,

937

ext = None,

915

online = False,

938

online = False,

916

delay = 60,

939

delay = 60,

917

walk = True):

940

walk = True):

918

941

919

if path == None:

942

if path == None:

920

raise ValueError, "The path is not valid"

943

raise ValueError, "The path is not valid"

921

944

922

if ext == None:

945

if ext == None:

923

ext = self.ext

946

ext = self.ext

924

947

925

if online:

948

if online:

926

print "Searching files in online mode..."

949

print "Searching files in online mode..."

927

950

928

for nTries in range( self.nTries ):

951

for nTries in range( self.nTries ):

929

fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)

952

fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)

930

953

931

if fullpath:

954

if fullpath:

932

break

955

break

933

956

934

print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)

957

print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)

935

time.sleep( self.delay )

958

time.sleep( self.delay )

936

959

937

if not(fullpath):

960

if not(fullpath):

938

print "There 'isn't valied files in %s" % path

961

print "There 'isn't valied files in %s" % path

939

return None

962

return None

940

963

941

self.year = year

964

self.year = year

942

self.doy = doy

965

self.doy = doy

943

self.set = set - 1

966

self.set = set - 1

944

self.path = path

967

self.path = path

945

self.foldercounter = foldercounter

968

self.foldercounter = foldercounter

946

969

947

else:

970

else:

948

print "Searching files in offline mode ..."

971

print "Searching files in offline mode ..."

949

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

972

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

950

startTime=startTime, endTime=endTime,

973

startTime=startTime, endTime=endTime,

951

set=set, expLabel=expLabel, ext=ext,

974

set=set, expLabel=expLabel, ext=ext,

952

walk=walk)

975

walk=walk)

953

976

954

if not(pathList):

977

if not(pathList):

955

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

978

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

956

datetime.datetime.combine(startDate,startTime).ctime(),

979

datetime.datetime.combine(startDate,startTime).ctime(),

957

datetime.datetime.combine(endDate,endTime).ctime())

980

datetime.datetime.combine(endDate,endTime).ctime())

958

981

959

sys.exit(-1)

982

sys.exit(-1)

960

983

961

984

962

self.fileIndex = -1

985

self.fileIndex = -1

963

self.pathList = pathList

986

self.pathList = pathList

964

self.filenameList = filenameList

987

self.filenameList = filenameList

965

988

966

self.online = online

989

self.online = online

967

self.delay = delay

990

self.delay = delay

968

ext = ext.lower()

991

ext = ext.lower()

969

self.ext = ext

992

self.ext = ext

970

993

971

if not(self.setNextFile()):

994

if not(self.setNextFile()):

972

if (startDate!=None) and (endDate!=None):

995

if (startDate!=None) and (endDate!=None):

973

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

996

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

974

elif startDate != None:

997

elif startDate != None:

975

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

998

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

976

else:

999

else:

977

print "No files"

1000

print "No files"

978

1001

979

sys.exit(-1)

1002

sys.exit(-1)

980

1003

981

# self.updateDataHeader()

1004

# self.updateDataHeader()

982

1005

983

return self.dataOut

1006

return self.dataOut

984

1007

985

def getBasicHeader(self):

1008

def getBasicHeader(self):

986

1009

987

self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds

1010

self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds

988

1011

989

self.dataOut.flagTimeBlock = self.flagTimeBlock

1012

self.dataOut.flagTimeBlock = self.flagTimeBlock

990

1013

991

self.dataOut.timeZone = self.basicHeaderObj.timeZone

1014

self.dataOut.timeZone = self.basicHeaderObj.timeZone

992

1015

993

self.dataOut.dstFlag = self.basicHeaderObj.dstFlag

1016

self.dataOut.dstFlag = self.basicHeaderObj.dstFlag

994

1017

995

self.dataOut.errorCount = self.basicHeaderObj.errorCount

1018

self.dataOut.errorCount = self.basicHeaderObj.errorCount

996

1019

997

self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime

1020

self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime

998

1021

999

def getFirstHeader(self):

1022

def getFirstHeader(self):

1000

1023

1001

raise ValueError, "This method has not been implemented"

1024

raise ValueError, "This method has not been implemented"

1002

1025

1003

def getData():

1026

def getData():

1004

1027

1005

raise ValueError, "This method has not been implemented"

1028

raise ValueError, "This method has not been implemented"

1006

1029

1007

def hasNotDataInBuffer():

1030

def hasNotDataInBuffer():

1008

1031

1009

raise ValueError, "This method has not been implemented"

1032

raise ValueError, "This method has not been implemented"

1010

1033

1011

def readBlock():

1034

def readBlock():

1012

1035

1013

raise ValueError, "This method has not been implemented"

1036

raise ValueError, "This method has not been implemented"

1014

1037

1015

def isEndProcess(self):

1038

def isEndProcess(self):

1016

1039

1017

return self.flagNoMoreFiles

1040

return self.flagNoMoreFiles

1018

1041

1019

def printReadBlocks(self):

1042

def printReadBlocks(self):

1020

1043

1021

print "Number of read blocks per file %04d" %self.nReadBlocks

1044

print "Number of read blocks per file %04d" %self.nReadBlocks

1022

1045

1023

def printTotalBlocks(self):

1046

def printTotalBlocks(self):

1024

1047

1025

print "Number of read blocks %04d" %self.nTotalBlocks

1048

print "Number of read blocks %04d" %self.nTotalBlocks

1026

1049

1027

def printNumberOfBlock(self):

1050

def printNumberOfBlock(self):

1028

1051

1029

if self.flagIsNewBlock:

1052

if self.flagIsNewBlock:

1030

print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())

1053

print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())

1031

1054

1032

def printInfo(self):

1055

def printInfo(self):

1033

1056

1034

if self.__printInfo == False:

1057

if self.__printInfo == False:

1035

return

1058

return

1036

1059

1037

self.basicHeaderObj.printInfo()

1060

self.basicHeaderObj.printInfo()

1038

self.systemHeaderObj.printInfo()

1061

self.systemHeaderObj.printInfo()

1039

self.radarControllerHeaderObj.printInfo()

1062

self.radarControllerHeaderObj.printInfo()

1040

self.processingHeaderObj.printInfo()

1063

self.processingHeaderObj.printInfo()

1041

1064

1042

self.__printInfo = False

1065

self.__printInfo = False

1043

1066

1044

1067

1045

def run(self, **kwargs):

1068

def run(self, **kwargs):

1046

1069

1047

if not(self.isConfig):

1070

if not(self.isConfig):

1048

1071

1049

# self.dataOut = dataOut

1072

# self.dataOut = dataOut

1050

self.setup(**kwargs)

1073

self.setup(**kwargs)

1051

self.isConfig = True

1074

self.isConfig = True

1052

1075

1053

self.getData()

1076

self.getData()

1054

1077

1055

class JRODataWriter(JRODataIO, Operation):

1078

class JRODataWriter(JRODataIO, Operation):

1056

1079

1057

"""

1080

"""

1058

Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura

1081

Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura

1059

de los datos siempre se realiza por bloques.

1082

de los datos siempre se realiza por bloques.

1060

"""

1083

"""

1061

1084

1062

blockIndex = 0

1085

blockIndex = 0

1063

1086

1064

path = None

1087

path = None

1065

1088

1066

setFile = None

1089

setFile = None

1067

1090

1068

profilesPerBlock = None

1091

profilesPerBlock = None

1069

1092

1070

blocksPerFile = None

1093

blocksPerFile = None

1071

1094

1072

nWriteBlocks = 0

1095

nWriteBlocks = 0

1073

1096

1074

def __init__(self, dataOut=None):

1097

def __init__(self, dataOut=None):

1075

raise ValueError, "Not implemented"

1098

raise ValueError, "Not implemented"

1076

1099

1077

1100

1078

def hasAllDataInBuffer(self):

1101

def hasAllDataInBuffer(self):

1079

raise ValueError, "Not implemented"

1102

raise ValueError, "Not implemented"

1080

1103

1081

1104

1082

def setBlockDimension(self):

1105

def setBlockDimension(self):

1083

raise ValueError, "Not implemented"

1106

raise ValueError, "Not implemented"

1084

1107

1085

1108

1086

def writeBlock(self):

1109

def writeBlock(self):

1087

raise ValueError, "No implemented"

1110

raise ValueError, "No implemented"

1088

1111

1089

1112

1090

def putData(self):

1113

def putData(self):

1091

raise ValueError, "No implemented"

1114

raise ValueError, "No implemented"

1092

1115

1093

1116

1094

def setBasicHeader(self):

1117

def setBasicHeader(self):

1095

1118

1096

self.basicHeaderObj.size = self.basicHeaderSize #bytes

1119

self.basicHeaderObj.size = self.basicHeaderSize #bytes

1097

self.basicHeaderObj.version = self.versionFile

1120

self.basicHeaderObj.version = self.versionFile

1098

self.basicHeaderObj.dataBlock = self.nTotalBlocks

1121

self.basicHeaderObj.dataBlock = self.nTotalBlocks

1099

1122

1100

utc = numpy.floor(self.dataOut.utctime)

1123

utc = numpy.floor(self.dataOut.utctime)

1101

milisecond = (self.dataOut.utctime - utc)* 1000.0

1124

milisecond = (self.dataOut.utctime - utc)* 1000.0

1102

1125

1103

self.basicHeaderObj.utc = utc

1126

self.basicHeaderObj.utc = utc

1104

self.basicHeaderObj.miliSecond = milisecond

1127

self.basicHeaderObj.miliSecond = milisecond

1105

self.basicHeaderObj.timeZone = self.dataOut.timeZone

1128

self.basicHeaderObj.timeZone = self.dataOut.timeZone

1106

self.basicHeaderObj.dstFlag = self.dataOut.dstFlag

1129

self.basicHeaderObj.dstFlag = self.dataOut.dstFlag

1107

self.basicHeaderObj.errorCount = self.dataOut.errorCount

1130

self.basicHeaderObj.errorCount = self.dataOut.errorCount

1108

1131

1109

def setFirstHeader(self):

1132

def setFirstHeader(self):

1110

"""

1133

"""

1111

Obtiene una copia del First Header

1134

Obtiene una copia del First Header

1112

1135

1113

Affected:

1136

Affected:

1114

1137

1115

self.basicHeaderObj

1138

self.basicHeaderObj

1116

self.systemHeaderObj

1139

self.systemHeaderObj

1117

self.radarControllerHeaderObj

1140

self.radarControllerHeaderObj

1118

self.processingHeaderObj self.

1141

self.processingHeaderObj self.

1119

1142

1120

Return:

1143

Return:

1121

None

1144

None

1122

"""

1145

"""

1123

1146

1124

raise ValueError, "No implemented"

1147

raise ValueError, "No implemented"

1125

1148

1126

def __writeFirstHeader(self):

1149

def __writeFirstHeader(self):

1127

"""

1150

"""

1128

Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)

1151

Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)

1129

1152

1130

Affected:

1153

Affected:

1131

__dataType

1154

__dataType

1132

1155

1133

Return:

1156

Return:

1134

None

1157

None

1135

"""

1158

"""

1136

1159

1137

# CALCULAR PARAMETROS

1160

# CALCULAR PARAMETROS

1138

1161

1139

sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size

1162

sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size

1140

self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader

1163

self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader

1141

1164

1142

self.basicHeaderObj.write(self.fp)

1165

self.basicHeaderObj.write(self.fp)

1143

self.systemHeaderObj.write(self.fp)

1166

self.systemHeaderObj.write(self.fp)

1144

self.radarControllerHeaderObj.write(self.fp)

1167

self.radarControllerHeaderObj.write(self.fp)

1145

self.processingHeaderObj.write(self.fp)

1168

self.processingHeaderObj.write(self.fp)

1146

1169

1147

self.dtype = self.dataOut.dtype

1170

self.dtype = self.dataOut.dtype

1148

1171

1149

def __setNewBlock(self):

1172

def __setNewBlock(self):

1150

"""

1173

"""

1151

Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header

1174

Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header

1152

1175

1153

Return:

1176

Return:

1154

0 : si no pudo escribir nada

1177

0 : si no pudo escribir nada

1155

1 : Si escribio el Basic el First Header

1178

1 : Si escribio el Basic el First Header

1156

"""

1179

"""

1157

if self.fp == None:

1180

if self.fp == None:

1158

self.setNextFile()

1181

self.setNextFile()

1159

1182

1160

if self.flagIsNewFile:

1183

if self.flagIsNewFile:

1161

return 1

1184

return 1

1162

1185

1163

if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:

1186

if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:

1164

self.basicHeaderObj.write(self.fp)

1187

self.basicHeaderObj.write(self.fp)

1165

return 1

1188

return 1

1166

1189

1167

if not( self.setNextFile() ):

1190

if not( self.setNextFile() ):

1168

return 0

1191

return 0

1169

1192

1170

return 1

1193

return 1

1171

1194

1172

1195

1173

def writeNextBlock(self):

1196

def writeNextBlock(self):

1174

"""

1197

"""

1175

Selecciona el bloque siguiente de datos y los escribe en un file

1198

Selecciona el bloque siguiente de datos y los escribe en un file

1176

1199

1177

Return:

1200

Return:

1178

0 : Si no hizo pudo escribir el bloque de datos

1201

0 : Si no hizo pudo escribir el bloque de datos

1179

1 : Si no pudo escribir el bloque de datos

1202

1 : Si no pudo escribir el bloque de datos

1180

"""

1203

"""

1181

if not( self.__setNewBlock() ):

1204

if not( self.__setNewBlock() ):

1182

return 0

1205

return 0

1183

1206

1184

self.writeBlock()

1207

self.writeBlock()

1185

1208

1186

return 1

1209

return 1

1187

1210

1188

def setNextFile(self):

1211

def setNextFile(self):

1189

"""

1212

"""

1190

Determina el siguiente file que sera escrito

1213

Determina el siguiente file que sera escrito

1191

1214

1192

Affected:

1215

Affected:

1193

self.filename

1216

self.filename

1194

self.subfolder

1217

self.subfolder

1195

self.fp

1218

self.fp

1196

self.setFile

1219

self.setFile

1197

self.flagIsNewFile

1220

self.flagIsNewFile

1198

1221

1199

Return:

1222

Return:

1200

0 : Si el archivo no puede ser escrito

1223

0 : Si el archivo no puede ser escrito

1201

1 : Si el archivo esta listo para ser escrito

1224

1 : Si el archivo esta listo para ser escrito

1202

"""

1225

"""

1203

ext = self.ext

1226

ext = self.ext

1204

path = self.path

1227

path = self.path

1205

1228

1206

if self.fp != None:

1229

if self.fp != None:

1207

self.fp.close()

1230

self.fp.close()

1208

1231

1209

timeTuple = time.localtime( self.dataOut.utctime)

1232

timeTuple = time.localtime( self.dataOut.utctime)

1210

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

1233

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

1211

1234

1212

fullpath = os.path.join( path, subfolder )

1235

fullpath = os.path.join( path, subfolder )

1213

if not( os.path.exists(fullpath) ):

1236

if not( os.path.exists(fullpath) ):

1214

os.mkdir(fullpath)

1237

os.mkdir(fullpath)

1215

self.setFile = -1 #inicializo mi contador de seteo

1238

self.setFile = -1 #inicializo mi contador de seteo

1216

else:

1239

else:

1217

filesList = os.listdir( fullpath )

1240

filesList = os.listdir( fullpath )

1218

if len( filesList ) > 0:

1241

if len( filesList ) > 0:

1219

filesList = sorted( filesList, key=str.lower )

1242

filesList = sorted( filesList, key=str.lower )

1220

filen = filesList[-1]

1243

filen = filesList[-1]

1221

# el filename debera tener el siguiente formato

1244

# el filename debera tener el siguiente formato

1222

# 0 1234 567 89A BCDE (hex)

1245

# 0 1234 567 89A BCDE (hex)

1223

# x YYYY DDD SSS .ext

1246

# x YYYY DDD SSS .ext

1224

if isNumber( filen[8:11] ):

1247

if isNumber( filen[8:11] ):

1225

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

1248

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

1226

else:

1249

else:

1227

self.setFile = -1

1250

self.setFile = -1

1228

else:

1251

else:

1229

self.setFile = -1 #inicializo mi contador de seteo

1252

self.setFile = -1 #inicializo mi contador de seteo

1230

1253

1231

setFile = self.setFile

1254

setFile = self.setFile

1232

setFile += 1

1255

setFile += 1

1233

1256

1234

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

1257

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

1235

timeTuple.tm_year,

1258

timeTuple.tm_year,

1236

timeTuple.tm_yday,

1259

timeTuple.tm_yday,

1237

setFile,

1260

setFile,

1238

ext )

1261

ext )

1239

1262

1240

filename = os.path.join( path, subfolder, file )

1263

filename = os.path.join( path, subfolder, file )

1241

1264

1242

fp = open( filename,'wb' )

1265

fp = open( filename,'wb' )

1243

1266

1244

self.blockIndex = 0

1267

self.blockIndex = 0

1245

1268

1246

#guardando atributos

1269

#guardando atributos

1247

self.filename = filename

1270

self.filename = filename

1248

self.subfolder = subfolder

1271

self.subfolder = subfolder

1249

self.fp = fp

1272

self.fp = fp

1250

self.setFile = setFile

1273

self.setFile = setFile

1251

self.flagIsNewFile = 1

1274

self.flagIsNewFile = 1

1252

1275

1253

self.setFirstHeader()

1276

self.setFirstHeader()

1254

1277

1255

print 'Writing the file: %s'%self.filename

1278

print 'Writing the file: %s'%self.filename

1256

1279

1257

self.__writeFirstHeader()

1280

self.__writeFirstHeader()

1258

1281

1259

return 1

1282

return 1

1260

1283

1261

def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):

1284

def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):

1262

"""

1285

"""

1263

Setea el tipo de formato en la cual sera guardada la data y escribe el First Header

1286

Setea el tipo de formato en la cual sera guardada la data y escribe el First Header

1264

1287

1265

Inputs:

1288

Inputs:

1266

path : el path destino en el cual se escribiran los files a crear

1289

path : el path destino en el cual se escribiran los files a crear

1267

format : formato en el cual sera salvado un file

1290

format : formato en el cual sera salvado un file

1268

set : el setebo del file

1291

set : el setebo del file

1269

1292

1270

Return:

1293

Return:

1271

0 : Si no realizo un buen seteo

1294

0 : Si no realizo un buen seteo

1272

1 : Si realizo un buen seteo

1295

1 : Si realizo un buen seteo

1273

"""

1296

"""

1274

1297

1275

if ext == None:

1298

if ext == None:

1276

ext = self.ext

1299

ext = self.ext

1277

1300

1278

ext = ext.lower()

1301

ext = ext.lower()

1279

1302

1280

self.ext = ext

1303

self.ext = ext

1281

1304

1282

self.path = path

1305

self.path = path

1283

1306

1284

self.setFile = set - 1

1307

self.setFile = set - 1

1285

1308

1286

self.blocksPerFile = blocksPerFile

1309

self.blocksPerFile = blocksPerFile

1287

1310

1288

self.profilesPerBlock = profilesPerBlock

1311

self.profilesPerBlock = profilesPerBlock

1289

1312

1290

self.dataOut = dataOut

1313

self.dataOut = dataOut

1291

1314

1292

if not(self.setNextFile()):

1315

if not(self.setNextFile()):

1293

print "There isn't a next file"

1316

print "There isn't a next file"

1294

return 0

1317

return 0

1295

1318

1296

self.setBlockDimension()

1319

self.setBlockDimension()

1297

1320

1298

return 1

1321

return 1

1299

1322

1300

def run(self, dataOut, **kwargs):

1323

def run(self, dataOut, **kwargs):

1301

1324

1302

if not(self.isConfig):

1325

if not(self.isConfig):

1303

1326

1304

self.setup(dataOut, **kwargs)

1327

self.setup(dataOut, **kwargs)

1305

self.isConfig = True

1328

self.isConfig = True

1306

1329

1307

self.putData()

1330

self.putData()

1308

1331

1309

class VoltageReader(JRODataReader):

1332

class VoltageReader(JRODataReader):

1310

"""

1333

"""

1311

Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura

1334

Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura

1312

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:

1335

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:

1313

perfiles*alturas*canales) son almacenados en la variable "buffer".

1336

perfiles*alturas*canales) son almacenados en la variable "buffer".

1314

1337

1315

perfiles * alturas * canales

1338

perfiles * alturas * canales

1316

1339

1317

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1340

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1318

RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la

1341

RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la

1319

cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de

1342

cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de

1320

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1343

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1321

1344

1322

Example:

1345

Example:

1323

1346

1324

dpath = "/home/myuser/data"

1347

dpath = "/home/myuser/data"

1325

1348

1326

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1349

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1327

1350

1328

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1351

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1329

1352

1330

readerObj = VoltageReader()

1353

readerObj = VoltageReader()

1331

1354

1332

readerObj.setup(dpath, startTime, endTime)

1355

readerObj.setup(dpath, startTime, endTime)

1333

1356

1334

while(True):

1357

while(True):

1335

1358

1336

#to get one profile

1359

#to get one profile

1337

profile = readerObj.getData()

1360

profile = readerObj.getData()

1338

1361

1339

#print the profile

1362

#print the profile

1340

print profile

1363

print profile

1341

1364

1342

#If you want to see all datablock

1365

#If you want to see all datablock

1343

print readerObj.datablock

1366

print readerObj.datablock

1344

1367

1345

if readerObj.flagNoMoreFiles:

1368

if readerObj.flagNoMoreFiles:

1346

break

1369

break

1347

1370

1348

"""

1371

"""

1349

1372

1350

ext = ".r"

1373

ext = ".r"

1351

1374

1352

optchar = "D"

1375

optchar = "D"

1353

dataOut = None

1376

dataOut = None

1354

1377

1355

1378

1356

def __init__(self):

1379

def __init__(self):

1357

"""

1380

"""

1358

Inicializador de la clase VoltageReader para la lectura de datos de voltage.

1381

Inicializador de la clase VoltageReader para la lectura de datos de voltage.

1359

1382

1360

Input:

1383

Input:

1361

dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para

1384

dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para

1362

almacenar un perfil de datos cada vez que se haga un requerimiento

1385

almacenar un perfil de datos cada vez que se haga un requerimiento

1363

(getData). El perfil sera obtenido a partir del buffer de datos,

1386

(getData). El perfil sera obtenido a partir del buffer de datos,

1364

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1387

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1365

bloque de datos.

1388

bloque de datos.

1366

Si este parametro no es pasado se creara uno internamente.

1389

Si este parametro no es pasado se creara uno internamente.

1367

1390

1368

Variables afectadas:

1391

Variables afectadas:

1369

self.dataOut

1392

self.dataOut

1370

1393

1371

Return:

1394

Return:

1372

None

1395

None

1373

"""

1396

"""

1374

1397

1375

self.isConfig = False

1398

self.isConfig = False

1376

1399

1377

self.datablock = None

1400

self.datablock = None

1378

1401

1379

self.utc = 0

1402

self.utc = 0

1380

1403

1381

self.ext = ".r"

1404

self.ext = ".r"

1382

1405

1383

self.optchar = "D"

1406

self.optchar = "D"

1384

1407

1385

self.basicHeaderObj = BasicHeader(LOCALTIME)

1408

self.basicHeaderObj = BasicHeader(LOCALTIME)

1386

1409

1387

self.systemHeaderObj = SystemHeader()

1410

self.systemHeaderObj = SystemHeader()

1388

1411

1389

self.radarControllerHeaderObj = RadarControllerHeader()

1412

self.radarControllerHeaderObj = RadarControllerHeader()

1390

1413

1391

self.processingHeaderObj = ProcessingHeader()

1414

self.processingHeaderObj = ProcessingHeader()

1392

1415

1393

self.online = 0

1416

self.online = 0

1394

1417

1395

self.fp = None

1418

self.fp = None

1396

1419

1397

self.idFile = None

1420

self.idFile = None

1398

1421

1399

self.dtype = None

1422

self.dtype = None

1400

1423

1401

self.fileSizeByHeader = None

1424

self.fileSizeByHeader = None

1402

1425

1403

self.filenameList = []

1426

self.filenameList = []

1404

1427

1405

self.filename = None

1428

self.filename = None

1406

1429

1407

self.fileSize = None

1430

self.fileSize = None

1408

1431

1409

self.firstHeaderSize = 0

1432

self.firstHeaderSize = 0

1410

1433

1411

self.basicHeaderSize = 24

1434

self.basicHeaderSize = 24

1412

1435

1413

self.pathList = []

1436

self.pathList = []

1414

1437

1415

self.filenameList = []

1438

self.filenameList = []

1416

1439

1417

self.lastUTTime = 0

1440

self.lastUTTime = 0

1418

1441

1419

self.maxTimeStep = 30

1442

self.maxTimeStep = 30

1420

1443

1421

self.flagNoMoreFiles = 0

1444

self.flagNoMoreFiles = 0

1422

1445

1423

self.set = 0

1446

self.set = 0

1424

1447

1425

self.path = None

1448

self.path = None

1426

1449

1427

self.profileIndex = 2**32-1

1450

self.profileIndex = 2**32-1

1428

1451

1429

self.delay = 3 #seconds

1452

self.delay = 3 #seconds

1430

1453

1431

self.nTries = 3 #quantity tries

1454

self.nTries = 3 #quantity tries

1432

1455

1433

self.nFiles = 3 #number of files for searching

1456

self.nFiles = 3 #number of files for searching

1434

1457

1435

self.nReadBlocks = 0

1458

self.nReadBlocks = 0

1436

1459

1437

self.flagIsNewFile = 1

1460

self.flagIsNewFile = 1

1438

1461

1439

self.__isFirstTimeOnline = 1

1462

self.__isFirstTimeOnline = 1

1440

1463

1441

self.ippSeconds = 0

1464

self.ippSeconds = 0

1442

1465

1443

self.flagTimeBlock = 0

1466

self.flagTimeBlock = 0

1444

1467

1445

self.flagIsNewBlock = 0

1468

self.flagIsNewBlock = 0

1446

1469

1447

self.nTotalBlocks = 0

1470

self.nTotalBlocks = 0

1448

1471

1449

self.blocksize = 0

1472

self.blocksize = 0

1450

1473

1451

self.dataOut = self.createObjByDefault()

1474

self.dataOut = self.createObjByDefault()

1452

1475

1453

def createObjByDefault(self):

1476

def createObjByDefault(self):

1454

1477

1455

dataObj = Voltage()

1478

dataObj = Voltage()

1456

1479

1457

return dataObj

1480

return dataObj

1458

1481

1459

def __hasNotDataInBuffer(self):

1482

def __hasNotDataInBuffer(self):

1460

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1483

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1461

return 1

1484

return 1

1462

return 0

1485

return 0

1463

1486

1464

1487

1465

def getBlockDimension(self):

1488

def getBlockDimension(self):

1466

"""

1489

"""

1467

Obtiene la cantidad de puntos a leer por cada bloque de datos

1490

Obtiene la cantidad de puntos a leer por cada bloque de datos

1468

1491

1469

Affected:

1492

Affected:

1470

self.blocksize

1493

self.blocksize

1471

1494

1472

Return:

1495

Return:

1473

None

1496

None

1474

"""

1497

"""

1475

pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels

1498

pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels

1476

self.blocksize = pts2read

1499

self.blocksize = pts2read

1477

1500

1478

1501

1479

def readBlock(self):

1502

def readBlock(self):

1480

"""

1503

"""

1481

readBlock lee el bloque de datos desde la posicion actual del puntero del archivo

1504

readBlock lee el bloque de datos desde la posicion actual del puntero del archivo

1482

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

1505

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

1483

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

1506

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

1484

es seteado a 0

1507

es seteado a 0

1485

1508

1486

Inputs:

1509

Inputs:

1487

None

1510

None

1488

1511

1489

Return:

1512

Return:

1490

None

1513

None

1491

1514

1492

Affected:

1515

Affected:

1493

self.profileIndex

1516

self.profileIndex

1494

self.datablock

1517

self.datablock

1495

self.flagIsNewFile

1518

self.flagIsNewFile

1496

self.flagIsNewBlock

1519

self.flagIsNewBlock

1497

self.nTotalBlocks

1520

self.nTotalBlocks

1498

1521

1499

Exceptions:

1522

Exceptions:

1500

Si un bloque leido no es un bloque valido

1523

Si un bloque leido no es un bloque valido

1501

"""

1524

"""

1502

1525

current_pointer_location = self.fp.tell()

1503

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1526

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1504

1527

1505

try:

1528

try:

1506

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1529

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1507

except:

1530

except:

1508

print "The read block (%3d) has not enough data" %self.nReadBlocks

1531

#print "The read block (%3d) has not enough data" %self.nReadBlocks

1509

~~return~~ 0

1532

1533

if self.waitDataBlock(pointer_location=current_pointer_location):

1534

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1535

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1536

# return 0

1510

1537

1511

junk = numpy.transpose(junk, (2,0,1))

1538

junk = numpy.transpose(junk, (2,0,1))

1512

self.datablock = junk['real'] + junk['imag']*1j

1539

self.datablock = junk['real'] + junk['imag']*1j

1513

1540

1514

self.profileIndex = 0

1541

self.profileIndex = 0

1515

1542

1516

self.flagIsNewFile = 0

1543

self.flagIsNewFile = 0

1517

self.flagIsNewBlock = 1

1544

self.flagIsNewBlock = 1

1518

1545

1519

self.nTotalBlocks += 1

1546

self.nTotalBlocks += 1

1520

self.nReadBlocks += 1

1547

self.nReadBlocks += 1

1521

1548

1522

return 1

1549

return 1

1523

1550

1524

def getFirstHeader(self):

1551

def getFirstHeader(self):

1525

1552

1526

self.dataOut.dtype = self.dtype

1553

self.dataOut.dtype = self.dtype

1527

1554

1528

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

1555

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

1529

1556

1530

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

1557

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

1531

1558

1532

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

1559

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

1533

1560

1534

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

1561

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

1535

1562

1536

self.dataOut.ippSeconds = self.ippSeconds

1563

self.dataOut.ippSeconds = self.ippSeconds

1537

1564

1538

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt

1565

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt

1539

1566

1540

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

1567

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

1541

1568

1542

self.dataOut.flagShiftFFT = False

1569

self.dataOut.flagShiftFFT = False

1543

1570

1544

if self.radarControllerHeaderObj.code != None:

1571

if self.radarControllerHeaderObj.code != None:

1545

1572

1546

self.dataOut.nCode = self.radarControllerHeaderObj.nCode

1573

self.dataOut.nCode = self.radarControllerHeaderObj.nCode

1547

1574

1548

self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud

1575

self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud

1549

1576

1550

self.dataOut.code = self.radarControllerHeaderObj.code

1577

self.dataOut.code = self.radarControllerHeaderObj.code

1551

1578

1552

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

1579

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

1553

1580

1554

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

1581

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

1555

1582

1556

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

1583

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

1557

1584

1558

self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip

1585

self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip

1559

1586

1560

self.dataOut.flagShiftFFT = False

1587

self.dataOut.flagShiftFFT = False

1561

1588

1562

def getData(self):

1589

def getData(self):

1563

"""

1590

"""

1564

getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"

1591

getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"

1565

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

1592

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

1566

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

1593

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

1567

1594

1568

Ademas incrementa el contador del buffer en 1.

1595

Ademas incrementa el contador del buffer en 1.

1569

1596

1570

Return:

1597

Return:

1571

data : retorna un perfil de voltages (alturas * canales) copiados desde el

1598

data : retorna un perfil de voltages (alturas * canales) copiados desde el

1572

buffer. Si no hay mas archivos a leer retorna None.

1599

buffer. Si no hay mas archivos a leer retorna None.

1573

1600

1574

Variables afectadas:

1601

Variables afectadas:

1575

self.dataOut

1602

self.dataOut

1576

self.profileIndex

1603

self.profileIndex

1577

1604

1578

Affected:

1605

Affected:

1579

self.dataOut

1606

self.dataOut

1580

self.profileIndex

1607

self.profileIndex

1581

self.flagTimeBlock

1608

self.flagTimeBlock

1582

self.flagIsNewBlock

1609

self.flagIsNewBlock

1583

"""

1610

"""

1584

1611

1585

if self.flagNoMoreFiles:

1612

if self.flagNoMoreFiles:

1586

self.dataOut.flagNoData = True

1613

self.dataOut.flagNoData = True

1587

print 'Process finished'

1614

print 'Process finished'

1588

return 0

1615

return 0

1589

1616

1590

self.flagTimeBlock = 0

1617

self.flagTimeBlock = 0

1591

self.flagIsNewBlock = 0

1618

self.flagIsNewBlock = 0

1592

1619

1593

if self.__hasNotDataInBuffer():

1620

if self.__hasNotDataInBuffer():

1594

1621

1595

if not( self.readNextBlock() ):

1622

if not( self.readNextBlock() ):

1596

return 0

1623

return 0

1597

1624

1598

self.getFirstHeader()

1625

self.getFirstHeader()

1599

1626

1600

if self.datablock == None:

1627

if self.datablock == None:

1601

self.dataOut.flagNoData = True

1628

self.dataOut.flagNoData = True

1602

return 0

1629

return 0

1603

1630

1604

self.dataOut.data = self.datablock[:,self.profileIndex,:]

1631

self.dataOut.data = self.datablock[:,self.profileIndex,:]

1605

1632

1606

self.dataOut.flagNoData = False

1633

self.dataOut.flagNoData = False

1607

1634

1608

self.getBasicHeader()

1635

self.getBasicHeader()

1609

1636

1610

self.profileIndex += 1

1637

self.profileIndex += 1

1611

1638

1612

self.dataOut.realtime = self.online

1639

self.dataOut.realtime = self.online

1613

1640

1614

return self.dataOut.data

1641

return self.dataOut.data

1615

1642

1616

1643

1617

class VoltageWriter(JRODataWriter):

1644

class VoltageWriter(JRODataWriter):

1618

"""

1645

"""

1619

Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura

1646

Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura

1620

de los datos siempre se realiza por bloques.

1647

de los datos siempre se realiza por bloques.

1621

"""

1648

"""

1622

1649

1623

ext = ".r"

1650

ext = ".r"

1624

1651

1625

optchar = "D"

1652

optchar = "D"

1626

1653

1627

shapeBuffer = None

1654

shapeBuffer = None

1628

1655

1629

1656

1630

def __init__(self):

1657

def __init__(self):

1631

"""

1658

"""

1632

Inicializador de la clase VoltageWriter para la escritura de datos de espectros.

1659

Inicializador de la clase VoltageWriter para la escritura de datos de espectros.

1633

1660

1634

Affected:

1661

Affected:

1635

self.dataOut

1662

self.dataOut

1636

1663

1637

Return: None

1664

Return: None

1638

"""

1665

"""

1639

1666

1640

self.nTotalBlocks = 0

1667

self.nTotalBlocks = 0

1641

1668

1642

self.profileIndex = 0

1669

self.profileIndex = 0

1643

1670

1644

self.isConfig = False

1671

self.isConfig = False

1645

1672

1646

self.fp = None

1673

self.fp = None

1647

1674

1648

self.flagIsNewFile = 1

1675

self.flagIsNewFile = 1

1649

1676

1650

self.nTotalBlocks = 0

1677

self.nTotalBlocks = 0

1651

1678

1652

self.flagIsNewBlock = 0

1679

self.flagIsNewBlock = 0

1653

1680

1654

self.setFile = None

1681

self.setFile = None

1655

1682

1656

self.dtype = None

1683

self.dtype = None

1657

1684

1658

self.path = None

1685

self.path = None

1659

1686

1660

self.filename = None

1687

self.filename = None

1661

1688

1662

self.basicHeaderObj = BasicHeader(LOCALTIME)

1689

self.basicHeaderObj = BasicHeader(LOCALTIME)

1663

1690

1664

self.systemHeaderObj = SystemHeader()

1691

self.systemHeaderObj = SystemHeader()

1665

1692

1666

self.radarControllerHeaderObj = RadarControllerHeader()

1693

self.radarControllerHeaderObj = RadarControllerHeader()

1667

1694

1668

self.processingHeaderObj = ProcessingHeader()

1695

self.processingHeaderObj = ProcessingHeader()

1669

1696

1670

def hasAllDataInBuffer(self):

1697

def hasAllDataInBuffer(self):

1671

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1698

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1672

return 1

1699

return 1

1673

return 0

1700

return 0

1674

1701

1675

1702

1676

def setBlockDimension(self):

1703

def setBlockDimension(self):

1677

"""

1704

"""

1678

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

1705

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

1679

1706

1680

Affected:

1707

Affected:

1681

self.shape_spc_Buffer

1708

self.shape_spc_Buffer

1682

self.shape_cspc_Buffer

1709

self.shape_cspc_Buffer

1683

self.shape_dc_Buffer

1710

self.shape_dc_Buffer

1684

1711

1685

Return: None

1712

Return: None

1686

"""

1713

"""

1687

self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,

1714

self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,

1688

self.processingHeaderObj.nHeights,

1715

self.processingHeaderObj.nHeights,

1689

self.systemHeaderObj.nChannels)

1716

self.systemHeaderObj.nChannels)

1690

1717

1691

self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,

1718

self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,

1692

self.processingHeaderObj.profilesPerBlock,

1719

self.processingHeaderObj.profilesPerBlock,

1693

self.processingHeaderObj.nHeights),

1720

self.processingHeaderObj.nHeights),

1694

dtype=numpy.dtype('complex64'))

1721

dtype=numpy.dtype('complex64'))

1695

1722

1696

1723

1697

def writeBlock(self):

1724

def writeBlock(self):

1698

"""

1725

"""

1699

Escribe el buffer en el file designado

1726

Escribe el buffer en el file designado

1700

1727

1701

Affected:

1728

Affected:

1702

self.profileIndex

1729

self.profileIndex

1703

self.flagIsNewFile

1730

self.flagIsNewFile

1704

self.flagIsNewBlock

1731

self.flagIsNewBlock

1705

self.nTotalBlocks

1732

self.nTotalBlocks

1706

self.blockIndex

1733

self.blockIndex

1707

1734

1708

Return: None

1735

Return: None

1709

"""

1736

"""

1710

data = numpy.zeros( self.shapeBuffer, self.dtype )

1737

data = numpy.zeros( self.shapeBuffer, self.dtype )

1711

1738

1712

junk = numpy.transpose(self.datablock, (1,2,0))

1739

junk = numpy.transpose(self.datablock, (1,2,0))

1713

1740

1714

data['real'] = junk.real

1741

data['real'] = junk.real

1715

data['imag'] = junk.imag

1742

data['imag'] = junk.imag

1716

1743

1717

data = data.reshape( (-1) )

1744

data = data.reshape( (-1) )

1718

1745

1719

data.tofile( self.fp )

1746

data.tofile( self.fp )

1720

1747

1721

self.datablock.fill(0)

1748

self.datablock.fill(0)

1722

1749

1723

self.profileIndex = 0

1750

self.profileIndex = 0

1724

self.flagIsNewFile = 0

1751

self.flagIsNewFile = 0

1725

self.flagIsNewBlock = 1

1752

self.flagIsNewBlock = 1

1726

1753

1727

self.blockIndex += 1

1754

self.blockIndex += 1

1728

self.nTotalBlocks += 1

1755

self.nTotalBlocks += 1

1729

1756

1730

def putData(self):

1757

def putData(self):

1731

"""

1758

"""

1732

Setea un bloque de datos y luego los escribe en un file

1759

Setea un bloque de datos y luego los escribe en un file

1733

1760

1734

Affected:

1761

Affected:

1735

self.flagIsNewBlock

1762

self.flagIsNewBlock

1736

self.profileIndex

1763

self.profileIndex

1737

1764

1738

Return:

1765

Return:

1739

0 : Si no hay data o no hay mas files que puedan escribirse

1766

0 : Si no hay data o no hay mas files que puedan escribirse

1740

1 : Si se escribio la data de un bloque en un file

1767

1 : Si se escribio la data de un bloque en un file

1741

"""

1768

"""

1742

if self.dataOut.flagNoData:

1769

if self.dataOut.flagNoData:

1743

return 0

1770

return 0

1744

1771

1745

self.flagIsNewBlock = 0

1772

self.flagIsNewBlock = 0

1746

1773

1747

if self.dataOut.flagTimeBlock:

1774

if self.dataOut.flagTimeBlock:

1748

1775

1749

self.datablock.fill(0)

1776

self.datablock.fill(0)

1750

self.profileIndex = 0

1777

self.profileIndex = 0

1751

self.setNextFile()

1778

self.setNextFile()

1752

1779

1753

if self.profileIndex == 0:

1780

if self.profileIndex == 0:

1754

self.setBasicHeader()

1781

self.setBasicHeader()

1755

1782

1756

self.datablock[:,self.profileIndex,:] = self.dataOut.data

1783

self.datablock[:,self.profileIndex,:] = self.dataOut.data

1757

1784

1758

self.profileIndex += 1

1785

self.profileIndex += 1

1759

1786

1760

if self.hasAllDataInBuffer():

1787

if self.hasAllDataInBuffer():

1761

#if self.flagIsNewFile:

1788

#if self.flagIsNewFile:

1762

self.writeNextBlock()

1789

self.writeNextBlock()

1763

# self.setFirstHeader()

1790

# self.setFirstHeader()

1764

1791

1765

return 1

1792

return 1

1766

1793

1767

def __getProcessFlags(self):

1794

def __getProcessFlags(self):

1768

1795

1769

processFlags = 0

1796

processFlags = 0

1770

1797

1771

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1798

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1772

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1799

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1773

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1800

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1774

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1801

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1775

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1802

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1776

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1803

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1777

1804

1778

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1805

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1779

1806

1780

1807

1781

1808

1782

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

1809

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

1783

PROCFLAG.DATATYPE_SHORT,

1810

PROCFLAG.DATATYPE_SHORT,

1784

PROCFLAG.DATATYPE_LONG,

1811

PROCFLAG.DATATYPE_LONG,

1785

PROCFLAG.DATATYPE_INT64,

1812

PROCFLAG.DATATYPE_INT64,

1786

PROCFLAG.DATATYPE_FLOAT,

1813

PROCFLAG.DATATYPE_FLOAT,

1787

PROCFLAG.DATATYPE_DOUBLE]

1814

PROCFLAG.DATATYPE_DOUBLE]

1788

1815

1789

1816

1790

for index in range(len(dtypeList)):

1817

for index in range(len(dtypeList)):

1791

if self.dataOut.dtype == dtypeList[index]:

1818

if self.dataOut.dtype == dtypeList[index]:

1792

dtypeValue = datatypeValueList[index]

1819

dtypeValue = datatypeValueList[index]

1793

break

1820

break

1794

1821

1795

processFlags += dtypeValue

1822

processFlags += dtypeValue

1796

1823

1797

if self.dataOut.flagDecodeData:

1824

if self.dataOut.flagDecodeData:

1798

processFlags += PROCFLAG.DECODE_DATA

1825

processFlags += PROCFLAG.DECODE_DATA

1799

1826

1800

if self.dataOut.flagDeflipData:

1827

if self.dataOut.flagDeflipData:

1801

processFlags += PROCFLAG.DEFLIP_DATA

1828

processFlags += PROCFLAG.DEFLIP_DATA

1802

1829

1803

if self.dataOut.code != None:

1830

if self.dataOut.code != None:

1804

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

1831

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

1805

1832

1806

if self.dataOut.nCohInt > 1:

1833

if self.dataOut.nCohInt > 1:

1807

processFlags += PROCFLAG.COHERENT_INTEGRATION

1834

processFlags += PROCFLAG.COHERENT_INTEGRATION

1808

1835

1809

return processFlags

1836

return processFlags

1810

1837

1811

1838

1812

def __getBlockSize(self):

1839

def __getBlockSize(self):

1813

'''

1840

'''

1814

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage

1841

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage

1815

'''

1842

'''

1816

1843

1817

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1844

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1818

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1845

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1819

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1846

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1820

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1847

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1821

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1848

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1822

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1849

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1823

1850

1824

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1851

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1825

datatypeValueList = [1,2,4,8,4,8]

1852

datatypeValueList = [1,2,4,8,4,8]

1826

for index in range(len(dtypeList)):

1853

for index in range(len(dtypeList)):

1827

if self.dataOut.dtype == dtypeList[index]:

1854

if self.dataOut.dtype == dtypeList[index]:

1828

datatypeValue = datatypeValueList[index]

1855

datatypeValue = datatypeValueList[index]

1829

break

1856

break

1830

1857

1831

blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * datatypeValue * 2)

1858

blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * datatypeValue * 2)

1832

1859

1833

return blocksize

1860

return blocksize

1834

1861

1835

def setFirstHeader(self):

1862

def setFirstHeader(self):

1836

1863

1837

"""

1864

"""

1838

Obtiene una copia del First Header

1865

Obtiene una copia del First Header

1839

1866

1840

Affected:

1867

Affected:

1841

self.systemHeaderObj

1868

self.systemHeaderObj

1842

self.radarControllerHeaderObj

1869

self.radarControllerHeaderObj

1843

self.dtype

1870

self.dtype

1844

1871

1845

Return:

1872

Return:

1846

None

1873

None

1847

"""

1874

"""

1848

1875

1849

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

1876

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

1850

self.systemHeaderObj.nChannels = self.dataOut.nChannels

1877

self.systemHeaderObj.nChannels = self.dataOut.nChannels

1851

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

1878

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

1852

1879

1853

self.setBasicHeader()

1880

self.setBasicHeader()

1854

1881

1855

processingHeaderSize = 40 # bytes

1882

processingHeaderSize = 40 # bytes

1856

self.processingHeaderObj.dtype = 0 # Voltage

1883

self.processingHeaderObj.dtype = 0 # Voltage

1857

self.processingHeaderObj.blockSize = self.__getBlockSize()

1884

self.processingHeaderObj.blockSize = self.__getBlockSize()

1858

self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock

1885

self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock

1859

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

1886

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

1860

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

1887

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

1861

self.processingHeaderObj.processFlags = self.__getProcessFlags()

1888

self.processingHeaderObj.processFlags = self.__getProcessFlags()

1862

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt

1889

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt

1863

self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage

1890

self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage

1864

self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage

1891

self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage

1865

1892

1866

# if self.dataOut.code != None:

1893

# if self.dataOut.code != None:

1867

# self.processingHeaderObj.code = self.dataOut.code

1894

# self.processingHeaderObj.code = self.dataOut.code

1868

# self.processingHeaderObj.nCode = self.dataOut.nCode

1895

# self.processingHeaderObj.nCode = self.dataOut.nCode

1869

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

1896

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

1870

# codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)

1897

# codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)

1871

# processingHeaderSize += codesize

1898

# processingHeaderSize += codesize

1872

1899

1873

if self.processingHeaderObj.nWindows != 0:

1900

if self.processingHeaderObj.nWindows != 0:

1874

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

1901

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

1875

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

1902

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

1876

self.processingHeaderObj.nHeights = self.dataOut.nHeights

1903

self.processingHeaderObj.nHeights = self.dataOut.nHeights

1877

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

1904

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

1878

processingHeaderSize += 12

1905

processingHeaderSize += 12

1879

1906

1880

self.processingHeaderObj.size = processingHeaderSize

1907

self.processingHeaderObj.size = processingHeaderSize

1881

1908

1882

class SpectraReader(JRODataReader):

1909

class SpectraReader(JRODataReader):

1883

"""

1910

"""

1884

Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura

1911

Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura

1885

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)

1912

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)

1886

son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.

1913

son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.

1887

1914

1888

paresCanalesIguales * alturas * perfiles (Self Spectra)

1915

paresCanalesIguales * alturas * perfiles (Self Spectra)

1889

paresCanalesDiferentes * alturas * perfiles (Cross Spectra)

1916

paresCanalesDiferentes * alturas * perfiles (Cross Spectra)

1890

canales * alturas (DC Channels)

1917

canales * alturas (DC Channels)

1891

1918

1892

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1919

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1893

RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la

1920

RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la

1894

cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de

1921

cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de

1895

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1922

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1896

1923

1897

Example:

1924

Example:

1898

dpath = "/home/myuser/data"

1925

dpath = "/home/myuser/data"

1899

1926

1900

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1927

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1901

1928

1902

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1929

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1903

1930

1904

readerObj = SpectraReader()

1931

readerObj = SpectraReader()

1905

1932

1906

readerObj.setup(dpath, startTime, endTime)

1933

readerObj.setup(dpath, startTime, endTime)

1907

1934

1908

while(True):

1935

while(True):

1909

1936

1910

readerObj.getData()

1937

readerObj.getData()

1911

1938

1912

print readerObj.data_spc

1939

print readerObj.data_spc

1913

1940

1914

print readerObj.data_cspc

1941

print readerObj.data_cspc

1915

1942

1916

print readerObj.data_dc

1943

print readerObj.data_dc

1917

1944

1918

if readerObj.flagNoMoreFiles:

1945

if readerObj.flagNoMoreFiles:

1919

break

1946

break

1920

1947

1921

"""

1948

"""

1922

1949

1923

pts2read_SelfSpectra = 0

1950

pts2read_SelfSpectra = 0

1924

1951

1925

pts2read_CrossSpectra = 0

1952

pts2read_CrossSpectra = 0

1926

1953

1927

pts2read_DCchannels = 0

1954

pts2read_DCchannels = 0

1928

1955

1929

ext = ".pdata"

1956

ext = ".pdata"

1930

1957

1931

optchar = "P"

1958

optchar = "P"

1932

1959

1933

dataOut = None

1960

dataOut = None

1934

1961

1935

nRdChannels = None

1962

nRdChannels = None

1936

1963

1937

nRdPairs = None

1964

nRdPairs = None

1938

1965

1939

rdPairList = []

1966

rdPairList = []

1940

1967

1941

def __init__(self):

1968

def __init__(self):

1942

"""

1969

"""

1943

Inicializador de la clase SpectraReader para la lectura de datos de espectros.

1970

Inicializador de la clase SpectraReader para la lectura de datos de espectros.

1944

1971

1945

Inputs:

1972

Inputs:

1946

dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para

1973

dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para

1947

almacenar un perfil de datos cada vez que se haga un requerimiento

1974

almacenar un perfil de datos cada vez que se haga un requerimiento

1948

(getData). El perfil sera obtenido a partir del buffer de datos,

1975

(getData). El perfil sera obtenido a partir del buffer de datos,

1949

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1976

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1950

bloque de datos.

1977

bloque de datos.

1951

Si este parametro no es pasado se creara uno internamente.

1978

Si este parametro no es pasado se creara uno internamente.

1952

1979

1953

Affected:

1980

Affected:

1954

self.dataOut

1981

self.dataOut

1955

1982

1956

Return : None

1983

Return : None

1957

"""

1984

"""

1958

1985

1959

self.isConfig = False

1986

self.isConfig = False

1960

1987

1961

self.pts2read_SelfSpectra = 0

1988

self.pts2read_SelfSpectra = 0

1962

1989

1963

self.pts2read_CrossSpectra = 0

1990

self.pts2read_CrossSpectra = 0

1964

1991

1965

self.pts2read_DCchannels = 0

1992

self.pts2read_DCchannels = 0

1966

1993

1967

self.datablock = None

1994

self.datablock = None

1968

1995

1969

self.utc = None

1996

self.utc = None

1970

1997

1971

self.ext = ".pdata"

1998

self.ext = ".pdata"

1972

1999

1973

self.optchar = "P"

2000

self.optchar = "P"

1974

2001

1975

self.basicHeaderObj = BasicHeader(LOCALTIME)

2002

self.basicHeaderObj = BasicHeader(LOCALTIME)

1976

2003

1977

self.systemHeaderObj = SystemHeader()

2004

self.systemHeaderObj = SystemHeader()

1978

2005

1979

self.radarControllerHeaderObj = RadarControllerHeader()

2006

self.radarControllerHeaderObj = RadarControllerHeader()

1980

2007

1981

self.processingHeaderObj = ProcessingHeader()

2008

self.processingHeaderObj = ProcessingHeader()

1982

2009

1983

self.online = 0

2010

self.online = 0

1984

2011

1985

self.fp = None

2012

self.fp = None

1986

2013

1987

self.idFile = None

2014

self.idFile = None

1988

2015

1989

self.dtype = None

2016

self.dtype = None

1990

2017

1991

self.fileSizeByHeader = None

2018

self.fileSizeByHeader = None

1992

2019

1993

self.filenameList = []

2020

self.filenameList = []

1994

2021

1995

self.filename = None

2022

self.filename = None

1996

2023

1997

self.fileSize = None

2024

self.fileSize = None

1998

2025

1999

self.firstHeaderSize = 0

2026

self.firstHeaderSize = 0

2000

2027

2001

self.basicHeaderSize = 24

2028

self.basicHeaderSize = 24

2002

2029

2003

self.pathList = []

2030

self.pathList = []

2004

2031

2005

self.lastUTTime = 0

2032

self.lastUTTime = 0

2006

2033

2007

self.maxTimeStep = 30

2034

self.maxTimeStep = 30

2008

2035

2009

self.flagNoMoreFiles = 0

2036

self.flagNoMoreFiles = 0

2010

2037

2011

self.set = 0

2038

self.set = 0

2012

2039

2013

self.path = None

2040

self.path = None

2014

2041

2015

self.delay = 60 #seconds

2042

self.delay = 60 #seconds

2016

2043

2017

self.nTries = 3 #quantity tries

2044

self.nTries = 3 #quantity tries

2018

2045

2019

self.nFiles = 3 #number of files for searching

2046

self.nFiles = 3 #number of files for searching

2020

2047

2021

self.nReadBlocks = 0

2048

self.nReadBlocks = 0

2022

2049

2023

self.flagIsNewFile = 1

2050

self.flagIsNewFile = 1

2024

2051

2025

self.__isFirstTimeOnline = 1

2052

self.__isFirstTimeOnline = 1

2026

2053

2027

self.ippSeconds = 0

2054

self.ippSeconds = 0

2028

2055

2029

self.flagTimeBlock = 0

2056

self.flagTimeBlock = 0

2030

2057

2031

self.flagIsNewBlock = 0

2058

self.flagIsNewBlock = 0

2032

2059

2033

self.nTotalBlocks = 0

2060

self.nTotalBlocks = 0

2034

2061

2035

self.blocksize = 0

2062

self.blocksize = 0

2036

2063

2037

self.dataOut = self.createObjByDefault()

2064

self.dataOut = self.createObjByDefault()

2038

2065

2039

self.profileIndex = 1 #Always

2066

self.profileIndex = 1 #Always

2040

2067

2041

2068

2042

def createObjByDefault(self):

2069

def createObjByDefault(self):

2043

2070

2044

dataObj = Spectra()

2071

dataObj = Spectra()

2045

2072

2046

return dataObj

2073

return dataObj

2047

2074

2048

def __hasNotDataInBuffer(self):

2075

def __hasNotDataInBuffer(self):

2049

return 1

2076

return 1

2050

2077

2051

2078

2052

def getBlockDimension(self):

2079

def getBlockDimension(self):

2053

"""

2080

"""

2054

Obtiene la cantidad de puntos a leer por cada bloque de datos

2081

Obtiene la cantidad de puntos a leer por cada bloque de datos

2055

2082

2056

Affected:

2083

Affected:

2057

self.nRdChannels

2084

self.nRdChannels

2058

self.nRdPairs

2085

self.nRdPairs

2059

self.pts2read_SelfSpectra

2086

self.pts2read_SelfSpectra

2060

self.pts2read_CrossSpectra

2087

self.pts2read_CrossSpectra

2061

self.pts2read_DCchannels

2088

self.pts2read_DCchannels

2062

self.blocksize

2089

self.blocksize

2063

self.dataOut.nChannels

2090

self.dataOut.nChannels

2064

self.dataOut.nPairs

2091

self.dataOut.nPairs

2065

2092

2066

Return:

2093

Return:

2067

None

2094

None

2068

"""

2095

"""

2069

self.nRdChannels = 0

2096

self.nRdChannels = 0

2070

self.nRdPairs = 0

2097

self.nRdPairs = 0

2071

self.rdPairList = []

2098

self.rdPairList = []

2072

2099

2073

for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):

2100

for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):

2074

if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:

2101

if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:

2075

self.nRdChannels = self.nRdChannels + 1 #par de canales iguales

2102

self.nRdChannels = self.nRdChannels + 1 #par de canales iguales

2076

else:

2103

else:

2077

self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes

2104

self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes

2078

self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))

2105

self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))

2079

2106

2080

pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock

2107

pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock

2081

2108

2082

self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)

2109

self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)

2083

self.blocksize = self.pts2read_SelfSpectra

2110

self.blocksize = self.pts2read_SelfSpectra

2084

2111

2085

if self.processingHeaderObj.flag_cspc:

2112

if self.processingHeaderObj.flag_cspc:

2086

self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)

2113

self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)

2087

self.blocksize += self.pts2read_CrossSpectra

2114

self.blocksize += self.pts2read_CrossSpectra

2088

2115

2089

if self.processingHeaderObj.flag_dc:

2116

if self.processingHeaderObj.flag_dc:

2090

self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)

2117

self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)

2091

self.blocksize += self.pts2read_DCchannels

2118

self.blocksize += self.pts2read_DCchannels

2092

2119

2093

# self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels

2120

# self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels

2094

2121

2095

2122

2096

def readBlock(self):

2123

def readBlock(self):

2097

"""

2124

"""

2098

Lee el bloque de datos desde la posicion actual del puntero del archivo

2125

Lee el bloque de datos desde la posicion actual del puntero del archivo

2099

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

2126

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

2100

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

2127

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

2101

es seteado a 0

2128

es seteado a 0

2102

2129

2103

Return: None

2130

Return: None

2104

2131

2105

Variables afectadas:

2132

Variables afectadas:

2106

2133

2107

self.flagIsNewFile

2134

self.flagIsNewFile

2108

self.flagIsNewBlock

2135

self.flagIsNewBlock

2109

self.nTotalBlocks

2136

self.nTotalBlocks

2110

self.data_spc

2137

self.data_spc

2111

self.data_cspc

2138

self.data_cspc

2112

self.data_dc

2139

self.data_dc

2113

2140

2114

Exceptions:

2141

Exceptions:

2115

Si un bloque leido no es un bloque valido

2142

Si un bloque leido no es un bloque valido

2116

"""

2143

"""

2117

blockOk_flag = False

2144

blockOk_flag = False

2118

fpointer = self.fp.tell()

2145

fpointer = self.fp.tell()

2119

2146

2120

spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )

2147

spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )

2121

spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2148

spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2122

2149

2123

if self.processingHeaderObj.flag_cspc:

2150

if self.processingHeaderObj.flag_cspc:

2124

cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )

2151

cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )

2125

cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2152

cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2126

2153

2127

if self.processingHeaderObj.flag_dc:

2154

if self.processingHeaderObj.flag_dc:

2128

dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )

2155

dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )

2129

dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D

2156

dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D

2130

2157

2131

2158

2132

if not(self.processingHeaderObj.shif_fft):

2159

if not(self.processingHeaderObj.shif_fft):

2133

#desplaza a la derecha en el eje 2 determinadas posiciones

2160

#desplaza a la derecha en el eje 2 determinadas posiciones

2134

shift = int(self.processingHeaderObj.profilesPerBlock/2)

2161

shift = int(self.processingHeaderObj.profilesPerBlock/2)

2135

spc = numpy.roll( spc, shift , axis=2 )

2162

spc = numpy.roll( spc, shift , axis=2 )

2136

2163

2137

if self.processingHeaderObj.flag_cspc:

2164

if self.processingHeaderObj.flag_cspc:

2138

#desplaza a la derecha en el eje 2 determinadas posiciones

2165

#desplaza a la derecha en el eje 2 determinadas posiciones

2139

cspc = numpy.roll( cspc, shift, axis=2 )

2166

cspc = numpy.roll( cspc, shift, axis=2 )

2140

2167

2141

# self.processingHeaderObj.shif_fft = True

2168

# self.processingHeaderObj.shif_fft = True

2142

2169

2143

spc = numpy.transpose( spc, (0,2,1) )

2170

spc = numpy.transpose( spc, (0,2,1) )

2144

self.data_spc = spc

2171

self.data_spc = spc

2145

2172

2146

if self.processingHeaderObj.flag_cspc:

2173

if self.processingHeaderObj.flag_cspc:

2147

cspc = numpy.transpose( cspc, (0,2,1) )

2174

cspc = numpy.transpose( cspc, (0,2,1) )

2148

self.data_cspc = cspc['real'] + cspc['imag']*1j

2175

self.data_cspc = cspc['real'] + cspc['imag']*1j

2149

else:

2176

else:

2150

self.data_cspc = None

2177

self.data_cspc = None

2151

2178

2152

if self.processingHeaderObj.flag_dc:

2179

if self.processingHeaderObj.flag_dc:

2153

self.data_dc = dc['real'] + dc['imag']*1j

2180

self.data_dc = dc['real'] + dc['imag']*1j

2154

else:

2181

else:

2155

self.data_dc = None

2182

self.data_dc = None

2156

2183

2157

self.flagIsNewFile = 0

2184

self.flagIsNewFile = 0

2158

self.flagIsNewBlock = 1

2185

self.flagIsNewBlock = 1

2159

2186

2160

self.nTotalBlocks += 1

2187

self.nTotalBlocks += 1

2161

self.nReadBlocks += 1

2188

self.nReadBlocks += 1

2162

2189

2163

return 1

2190

return 1

2164

2191

2165

def getFirstHeader(self):

2192

def getFirstHeader(self):

2166

2193

2167

self.dataOut.dtype = self.dtype

2194

self.dataOut.dtype = self.dtype

2168

2195

2169

self.dataOut.nPairs = self.nRdPairs

2196

self.dataOut.nPairs = self.nRdPairs

2170

2197

2171

self.dataOut.pairsList = self.rdPairList

2198

self.dataOut.pairsList = self.rdPairList

2172

2199

2173

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

2200

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

2174

2201

2175

self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock

2202

self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock

2176

2203

2177

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

2204

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

2178

2205

2179

self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt

2206

self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt

2180

2207

2181

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

2208

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

2182

2209

2183

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

2210

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

2184

2211

2185

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

2212

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

2186

2213

2187

self.dataOut.ippSeconds = self.ippSeconds

2214

self.dataOut.ippSeconds = self.ippSeconds

2188

2215

2189

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints

2216

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints

2190

2217

2191

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

2218

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

2192

2219

2193

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

2220

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

2194

2221

2195

self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft

2222

self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft

2196

2223

2197

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

2224

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

2198

2225

2199

self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip

2226

self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip

2200

2227

2201

if self.processingHeaderObj.code != None:

2228

if self.processingHeaderObj.code != None:

2202

2229

2203

self.dataOut.nCode = self.processingHeaderObj.nCode

2230

self.dataOut.nCode = self.processingHeaderObj.nCode

2204

2231

2205

self.dataOut.nBaud = self.processingHeaderObj.nBaud

2232

self.dataOut.nBaud = self.processingHeaderObj.nBaud

2206

2233

2207

self.dataOut.code = self.processingHeaderObj.code

2234

self.dataOut.code = self.processingHeaderObj.code

2208

2235

2209

self.dataOut.flagDecodeData = True

2236

self.dataOut.flagDecodeData = True

2210

2237

2211

def getData(self):

2238

def getData(self):

2212

"""

2239

"""

2213

Copia el buffer de lectura a la clase "Spectra",

2240

Copia el buffer de lectura a la clase "Spectra",

2214

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

2241

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

2215

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

2242

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

2216

2243

2217

Return:

2244

Return:

2218

0 : Si no hay mas archivos disponibles

2245

0 : Si no hay mas archivos disponibles

2219

1 : Si hizo una buena copia del buffer

2246

1 : Si hizo una buena copia del buffer

2220

2247

2221

Affected:

2248

Affected:

2222

self.dataOut

2249

self.dataOut

2223

2250

2224

self.flagTimeBlock

2251

self.flagTimeBlock

2225

self.flagIsNewBlock

2252

self.flagIsNewBlock

2226

"""

2253

"""

2227

2254

2228

if self.flagNoMoreFiles:

2255

if self.flagNoMoreFiles:

2229

self.dataOut.flagNoData = True

2256

self.dataOut.flagNoData = True

2230

print 'Process finished'

2257

print 'Process finished'

2231

return 0

2258

return 0

2232

2259

2233

self.flagTimeBlock = 0

2260

self.flagTimeBlock = 0

2234

self.flagIsNewBlock = 0

2261

self.flagIsNewBlock = 0

2235

2262

2236

if self.__hasNotDataInBuffer():

2263

if self.__hasNotDataInBuffer():

2237

2264

2238

if not( self.readNextBlock() ):

2265

if not( self.readNextBlock() ):

2239

self.dataOut.flagNoData = True

2266

self.dataOut.flagNoData = True

2240

return 0

2267

return 0

2241

2268

2242

#data es un numpy array de 3 dmensiones (perfiles, alturas y canales)

2269

#data es un numpy array de 3 dmensiones (perfiles, alturas y canales)

2243

2270

2244

if self.data_dc == None:

2271

if self.data_dc == None:

2245

self.dataOut.flagNoData = True

2272

self.dataOut.flagNoData = True

2246

return 0

2273

return 0

2247

2274

2248

self.getBasicHeader()

2275

self.getBasicHeader()

2249

2276

2250

self.getFirstHeader()

2277

self.getFirstHeader()

2251

2278

2252

self.dataOut.data_spc = self.data_spc

2279

self.dataOut.data_spc = self.data_spc

2253

2280

2254

self.dataOut.data_cspc = self.data_cspc

2281

self.dataOut.data_cspc = self.data_cspc

2255

2282

2256

self.dataOut.data_dc = self.data_dc

2283

self.dataOut.data_dc = self.data_dc

2257

2284

2258

self.dataOut.flagNoData = False

2285

self.dataOut.flagNoData = False

2259

2286

2260

self.dataOut.realtime = self.online

2287

self.dataOut.realtime = self.online

2261

2288

2262

return self.dataOut.data_spc

2289

return self.dataOut.data_spc

2263

2290

2264

2291

2265

class SpectraWriter(JRODataWriter):

2292

class SpectraWriter(JRODataWriter):

2266

2293

2267

"""

2294

"""

2268

Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura

2295

Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura

2269

de los datos siempre se realiza por bloques.

2296

de los datos siempre se realiza por bloques.

2270

"""

2297

"""

2271

2298

2272

ext = ".pdata"

2299

ext = ".pdata"

2273

2300

2274

optchar = "P"

2301

optchar = "P"

2275

2302

2276

shape_spc_Buffer = None

2303

shape_spc_Buffer = None

2277

2304

2278

shape_cspc_Buffer = None

2305

shape_cspc_Buffer = None

2279

2306

2280

shape_dc_Buffer = None

2307

shape_dc_Buffer = None

2281

2308

2282

data_spc = None

2309

data_spc = None

2283

2310

2284

data_cspc = None

2311

data_cspc = None

2285

2312

2286

data_dc = None

2313

data_dc = None

2287

2314

2288

# dataOut = None

2315

# dataOut = None

2289

2316

2290

def __init__(self):

2317

def __init__(self):

2291

"""

2318

"""

2292

Inicializador de la clase SpectraWriter para la escritura de datos de espectros.

2319

Inicializador de la clase SpectraWriter para la escritura de datos de espectros.

2293

2320

2294

Affected:

2321

Affected:

2295

self.dataOut

2322

self.dataOut

2296

self.basicHeaderObj

2323

self.basicHeaderObj

2297

self.systemHeaderObj

2324

self.systemHeaderObj

2298

self.radarControllerHeaderObj

2325

self.radarControllerHeaderObj

2299

self.processingHeaderObj

2326

self.processingHeaderObj

2300

2327

2301

Return: None

2328

Return: None

2302

"""

2329

"""

2303

2330

2304

self.isConfig = False

2331

self.isConfig = False

2305

2332

2306

self.nTotalBlocks = 0

2333

self.nTotalBlocks = 0

2307

2334

2308

self.data_spc = None

2335

self.data_spc = None

2309

2336

2310

self.data_cspc = None

2337

self.data_cspc = None

2311

2338

2312

self.data_dc = None

2339

self.data_dc = None

2313

2340

2314

self.fp = None

2341

self.fp = None

2315

2342

2316

self.flagIsNewFile = 1

2343

self.flagIsNewFile = 1

2317

2344

2318

self.nTotalBlocks = 0

2345

self.nTotalBlocks = 0

2319

2346

2320

self.flagIsNewBlock = 0

2347

self.flagIsNewBlock = 0

2321

2348

2322

self.setFile = None

2349

self.setFile = None

2323

2350

2324

self.dtype = None

2351

self.dtype = None

2325

2352

2326

self.path = None

2353

self.path = None

2327

2354

2328

self.noMoreFiles = 0

2355

self.noMoreFiles = 0

2329

2356

2330

self.filename = None

2357

self.filename = None

2331

2358

2332

self.basicHeaderObj = BasicHeader(LOCALTIME)

2359

self.basicHeaderObj = BasicHeader(LOCALTIME)

2333

2360

2334

self.systemHeaderObj = SystemHeader()

2361

self.systemHeaderObj = SystemHeader()

2335

2362

2336

self.radarControllerHeaderObj = RadarControllerHeader()

2363

self.radarControllerHeaderObj = RadarControllerHeader()

2337

2364

2338

self.processingHeaderObj = ProcessingHeader()

2365

self.processingHeaderObj = ProcessingHeader()

2339

2366

2340

2367

2341

def hasAllDataInBuffer(self):

2368

def hasAllDataInBuffer(self):

2342

return 1

2369

return 1

2343

2370

2344

2371

2345

def setBlockDimension(self):

2372

def setBlockDimension(self):

2346

"""

2373

"""

2347

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

2374

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

2348

2375

2349

Affected:

2376

Affected:

2350

self.shape_spc_Buffer

2377

self.shape_spc_Buffer

2351

self.shape_cspc_Buffer

2378

self.shape_cspc_Buffer

2352

self.shape_dc_Buffer

2379

self.shape_dc_Buffer

2353

2380

2354

Return: None

2381

Return: None

2355

"""

2382

"""

2356

self.shape_spc_Buffer = (self.dataOut.nChannels,

2383

self.shape_spc_Buffer = (self.dataOut.nChannels,

2357

self.processingHeaderObj.nHeights,

2384

self.processingHeaderObj.nHeights,

2358

self.processingHeaderObj.profilesPerBlock)

2385

self.processingHeaderObj.profilesPerBlock)

2359

2386

2360

self.shape_cspc_Buffer = (self.dataOut.nPairs,

2387

self.shape_cspc_Buffer = (self.dataOut.nPairs,

2361

self.processingHeaderObj.nHeights,

2388

self.processingHeaderObj.nHeights,

2362

self.processingHeaderObj.profilesPerBlock)

2389

self.processingHeaderObj.profilesPerBlock)

2363

2390

2364

self.shape_dc_Buffer = (self.dataOut.nChannels,

2391

self.shape_dc_Buffer = (self.dataOut.nChannels,

2365

self.processingHeaderObj.nHeights)

2392

self.processingHeaderObj.nHeights)

2366

2393

2367

2394

2368

def writeBlock(self):

2395

def writeBlock(self):

2369

"""

2396

"""

2370

Escribe el buffer en el file designado

2397

Escribe el buffer en el file designado

2371

2398

2372

Affected:

2399

Affected:

2373

self.data_spc

2400

self.data_spc

2374

self.data_cspc

2401

self.data_cspc

2375

self.data_dc

2402

self.data_dc

2376

self.flagIsNewFile

2403

self.flagIsNewFile

2377

self.flagIsNewBlock

2404

self.flagIsNewBlock

2378

self.nTotalBlocks

2405

self.nTotalBlocks

2379

self.nWriteBlocks

2406

self.nWriteBlocks

2380

2407

2381

Return: None

2408

Return: None

2382

"""

2409

"""

2383

2410

2384

spc = numpy.transpose( self.data_spc, (0,2,1) )

2411

spc = numpy.transpose( self.data_spc, (0,2,1) )

2385

if not( self.processingHeaderObj.shif_fft ):

2412

if not( self.processingHeaderObj.shif_fft ):

2386

spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2413

spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2387

data = spc.reshape((-1))

2414

data = spc.reshape((-1))

2388

data = data.astype(self.dtype[0])

2415

data = data.astype(self.dtype[0])

2389

data.tofile(self.fp)

2416

data.tofile(self.fp)

2390

2417

2391

if self.data_cspc != None:

2418

if self.data_cspc != None:

2392

data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )

2419

data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )

2393

cspc = numpy.transpose( self.data_cspc, (0,2,1) )

2420

cspc = numpy.transpose( self.data_cspc, (0,2,1) )

2394

if not( self.processingHeaderObj.shif_fft ):

2421

if not( self.processingHeaderObj.shif_fft ):

2395

cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2422

cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2396

data['real'] = cspc.real

2423

data['real'] = cspc.real

2397

data['imag'] = cspc.imag

2424

data['imag'] = cspc.imag

2398

data = data.reshape((-1))

2425

data = data.reshape((-1))

2399

data.tofile(self.fp)

2426

data.tofile(self.fp)

2400

2427

2401

if self.data_dc != None:

2428

if self.data_dc != None:

2402

data = numpy.zeros( self.shape_dc_Buffer, self.dtype )

2429

data = numpy.zeros( self.shape_dc_Buffer, self.dtype )

2403

dc = self.data_dc

2430

dc = self.data_dc

2404

data['real'] = dc.real

2431

data['real'] = dc.real

2405

data['imag'] = dc.imag

2432

data['imag'] = dc.imag

2406

data = data.reshape((-1))

2433

data = data.reshape((-1))

2407

data.tofile(self.fp)

2434

data.tofile(self.fp)

2408

2435

2409

self.data_spc.fill(0)

2436

self.data_spc.fill(0)

2410

2437

2411

if self.data_dc != None:

2438

if self.data_dc != None:

2412

self.data_dc.fill(0)

2439

self.data_dc.fill(0)

2413

2440

2414

if self.data_cspc != None:

2441

if self.data_cspc != None:

2415

self.data_cspc.fill(0)

2442

self.data_cspc.fill(0)

2416

2443

2417

self.flagIsNewFile = 0

2444

self.flagIsNewFile = 0

2418

self.flagIsNewBlock = 1

2445

self.flagIsNewBlock = 1

2419

self.nTotalBlocks += 1

2446

self.nTotalBlocks += 1

2420

self.nWriteBlocks += 1

2447

self.nWriteBlocks += 1

2421

self.blockIndex += 1

2448

self.blockIndex += 1

2422

2449

2423

2450

2424

def putData(self):

2451

def putData(self):

2425

"""

2452

"""

2426

Setea un bloque de datos y luego los escribe en un file

2453

Setea un bloque de datos y luego los escribe en un file

2427

2454

2428

Affected:

2455

Affected:

2429

self.data_spc

2456

self.data_spc

2430

self.data_cspc

2457

self.data_cspc

2431

self.data_dc

2458

self.data_dc

2432

2459

2433

Return:

2460

Return:

2434

0 : Si no hay data o no hay mas files que puedan escribirse

2461

0 : Si no hay data o no hay mas files que puedan escribirse

2435

1 : Si se escribio la data de un bloque en un file

2462

1 : Si se escribio la data de un bloque en un file

2436

"""

2463

"""

2437

2464

2438

if self.dataOut.flagNoData:

2465

if self.dataOut.flagNoData:

2439

return 0

2466

return 0

2440

2467

2441

self.flagIsNewBlock = 0

2468

self.flagIsNewBlock = 0

2442

2469

2443

if self.dataOut.flagTimeBlock:

2470

if self.dataOut.flagTimeBlock:

2444

self.data_spc.fill(0)

2471

self.data_spc.fill(0)

2445

self.data_cspc.fill(0)

2472

self.data_cspc.fill(0)

2446

self.data_dc.fill(0)

2473

self.data_dc.fill(0)

2447

self.setNextFile()

2474

self.setNextFile()

2448

2475

2449

if self.flagIsNewFile == 0:

2476

if self.flagIsNewFile == 0:

2450

self.setBasicHeader()

2477

self.setBasicHeader()

2451

2478

2452

self.data_spc = self.dataOut.data_spc.copy()

2479

self.data_spc = self.dataOut.data_spc.copy()

2453

if self.dataOut.data_cspc != None:

2480

if self.dataOut.data_cspc != None:

2454

self.data_cspc = self.dataOut.data_cspc.copy()

2481

self.data_cspc = self.dataOut.data_cspc.copy()

2455

self.data_dc = self.dataOut.data_dc.copy()

2482

self.data_dc = self.dataOut.data_dc.copy()

2456

2483

2457

# #self.processingHeaderObj.dataBlocksPerFile)

2484

# #self.processingHeaderObj.dataBlocksPerFile)

2458

if self.hasAllDataInBuffer():

2485

if self.hasAllDataInBuffer():

2459

# self.setFirstHeader()

2486

# self.setFirstHeader()

2460

self.writeNextBlock()

2487

self.writeNextBlock()

2461

2488

2462

return 1

2489

return 1

2463

2490

2464

2491

2465

def __getProcessFlags(self):

2492

def __getProcessFlags(self):

2466

2493

2467

processFlags = 0

2494

processFlags = 0

2468

2495

2469

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2496

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2470

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2497

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2471

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2498

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2472

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2499

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2473

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2500

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2474

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2501

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2475

2502

2476

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2503

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2477

2504

2478

2505

2479

2506

2480

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

2507

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

2481

PROCFLAG.DATATYPE_SHORT,

2508

PROCFLAG.DATATYPE_SHORT,

2482

PROCFLAG.DATATYPE_LONG,

2509

PROCFLAG.DATATYPE_LONG,

2483

PROCFLAG.DATATYPE_INT64,

2510

PROCFLAG.DATATYPE_INT64,

2484

PROCFLAG.DATATYPE_FLOAT,

2511

PROCFLAG.DATATYPE_FLOAT,

2485

PROCFLAG.DATATYPE_DOUBLE]

2512

PROCFLAG.DATATYPE_DOUBLE]

2486

2513

2487

2514

2488

for index in range(len(dtypeList)):

2515

for index in range(len(dtypeList)):

2489

if self.dataOut.dtype == dtypeList[index]:

2516

if self.dataOut.dtype == dtypeList[index]:

2490

dtypeValue = datatypeValueList[index]

2517

dtypeValue = datatypeValueList[index]

2491

break

2518

break

2492

2519

2493

processFlags += dtypeValue

2520

processFlags += dtypeValue

2494

2521

2495

if self.dataOut.flagDecodeData:

2522

if self.dataOut.flagDecodeData:

2496

processFlags += PROCFLAG.DECODE_DATA

2523

processFlags += PROCFLAG.DECODE_DATA

2497

2524

2498

if self.dataOut.flagDeflipData:

2525

if self.dataOut.flagDeflipData:

2499

processFlags += PROCFLAG.DEFLIP_DATA

2526

processFlags += PROCFLAG.DEFLIP_DATA

2500

2527

2501

if self.dataOut.code != None:

2528

if self.dataOut.code != None:

2502

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

2529

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

2503

2530

2504

if self.dataOut.nIncohInt > 1:

2531

if self.dataOut.nIncohInt > 1:

2505

processFlags += PROCFLAG.INCOHERENT_INTEGRATION

2532

processFlags += PROCFLAG.INCOHERENT_INTEGRATION

2506

2533

2507

if self.dataOut.data_dc != None:

2534

if self.dataOut.data_dc != None:

2508

processFlags += PROCFLAG.SAVE_CHANNELS_DC

2535

processFlags += PROCFLAG.SAVE_CHANNELS_DC

2509

2536

2510

return processFlags

2537

return processFlags

2511

2538

2512

2539

2513

def __getBlockSize(self):

2540

def __getBlockSize(self):

2514

'''

2541

'''

2515

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra

2542

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra

2516

'''

2543

'''

2517

2544

2518

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2545

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2519

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2546

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2520

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2547

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2521

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2548

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2522

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2549

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2523

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2550

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2524

2551

2525

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2552

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2526

datatypeValueList = [1,2,4,8,4,8]

2553

datatypeValueList = [1,2,4,8,4,8]

2527

for index in range(len(dtypeList)):

2554

for index in range(len(dtypeList)):

2528

if self.dataOut.dtype == dtypeList[index]:

2555

if self.dataOut.dtype == dtypeList[index]:

2529

datatypeValue = datatypeValueList[index]

2556

datatypeValue = datatypeValueList[index]

2530

break

2557

break

2531

2558

2532

2559

2533

pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints

2560

pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints

2534

2561

2535

pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)

2562

pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)

2536

blocksize = (pts2write_SelfSpectra*datatypeValue)

2563

blocksize = (pts2write_SelfSpectra*datatypeValue)

2537

2564

2538

if self.dataOut.data_cspc != None:

2565

if self.dataOut.data_cspc != None:

2539

pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)

2566

pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)

2540

blocksize += (pts2write_CrossSpectra*datatypeValue*2)

2567

blocksize += (pts2write_CrossSpectra*datatypeValue*2)

2541

2568

2542

if self.dataOut.data_dc != None:

2569

if self.dataOut.data_dc != None:

2543

pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)

2570

pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)

2544

blocksize += (pts2write_DCchannels*datatypeValue*2)

2571

blocksize += (pts2write_DCchannels*datatypeValue*2)

2545

2572

2546

blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO

2573

blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO

2547

2574

2548

return blocksize

2575

return blocksize

2549

2576

2550

def setFirstHeader(self):

2577

def setFirstHeader(self):

2551

2578

2552

"""

2579

"""

2553

Obtiene una copia del First Header

2580

Obtiene una copia del First Header

2554

2581

2555

Affected:

2582

Affected:

2556

self.systemHeaderObj

2583

self.systemHeaderObj

2557

self.radarControllerHeaderObj

2584

self.radarControllerHeaderObj

2558

self.dtype

2585

self.dtype

2559

2586

2560

Return:

2587

Return:

2561

None

2588

None

2562

"""

2589

"""

2563

2590

2564

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

2591

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

2565

self.systemHeaderObj.nChannels = self.dataOut.nChannels

2592

self.systemHeaderObj.nChannels = self.dataOut.nChannels

2566

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

2593

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

2567

2594

2568

self.setBasicHeader()

2595

self.setBasicHeader()

2569

2596

2570

processingHeaderSize = 40 # bytes

2597

processingHeaderSize = 40 # bytes

2571

self.processingHeaderObj.dtype = 1 # Spectra

2598

self.processingHeaderObj.dtype = 1 # Spectra

2572

self.processingHeaderObj.blockSize = self.__getBlockSize()

2599

self.processingHeaderObj.blockSize = self.__getBlockSize()

2573

self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints

2600

self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints

2574

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

2601

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

2575

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

2602

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

2576

self.processingHeaderObj.processFlags = self.__getProcessFlags()

2603

self.processingHeaderObj.processFlags = self.__getProcessFlags()

2577

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval

2604

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval

2578

self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt

2605

self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt

2579

self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels

2606

self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels

2580

self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT

2607

self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT

2581

2608

2582

if self.processingHeaderObj.totalSpectra > 0:

2609

if self.processingHeaderObj.totalSpectra > 0:

2583

channelList = []

2610

channelList = []

2584

for channel in range(self.dataOut.nChannels):

2611

for channel in range(self.dataOut.nChannels):

2585

channelList.append(channel)

2612

channelList.append(channel)

2586

channelList.append(channel)

2613

channelList.append(channel)

2587

2614

2588

pairsList = []

2615

pairsList = []

2589

if self.dataOut.nPairs > 0:

2616

if self.dataOut.nPairs > 0:

2590

for pair in self.dataOut.pairsList:

2617

for pair in self.dataOut.pairsList:

2591

pairsList.append(pair[0])

2618

pairsList.append(pair[0])

2592

pairsList.append(pair[1])

2619

pairsList.append(pair[1])

2593

2620

2594

spectraComb = channelList + pairsList

2621

spectraComb = channelList + pairsList

2595

spectraComb = numpy.array(spectraComb,dtype="u1")

2622

spectraComb = numpy.array(spectraComb,dtype="u1")

2596

self.processingHeaderObj.spectraComb = spectraComb

2623

self.processingHeaderObj.spectraComb = spectraComb

2597

sizeOfSpcComb = len(spectraComb)

2624

sizeOfSpcComb = len(spectraComb)

2598

processingHeaderSize += sizeOfSpcComb

2625

processingHeaderSize += sizeOfSpcComb

2599

2626

2600

# The processing header should not have information about code

2627

# The processing header should not have information about code

2601

# if self.dataOut.code != None:

2628

# if self.dataOut.code != None:

2602

# self.processingHeaderObj.code = self.dataOut.code

2629

# self.processingHeaderObj.code = self.dataOut.code

2603

# self.processingHeaderObj.nCode = self.dataOut.nCode

2630

# self.processingHeaderObj.nCode = self.dataOut.nCode

2604

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

2631

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

2605

# nCodeSize = 4 # bytes

2632

# nCodeSize = 4 # bytes

2606

# nBaudSize = 4 # bytes

2633

# nBaudSize = 4 # bytes

2607

# codeSize = 4 # bytes

2634

# codeSize = 4 # bytes

2608

# sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)

2635

# sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)

2609

# processingHeaderSize += sizeOfCode

2636

# processingHeaderSize += sizeOfCode

2610

2637

2611

if self.processingHeaderObj.nWindows != 0:

2638

if self.processingHeaderObj.nWindows != 0:

2612

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

2639

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

2613

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2640

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2614

self.processingHeaderObj.nHeights = self.dataOut.nHeights

2641

self.processingHeaderObj.nHeights = self.dataOut.nHeights

2615

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

2642

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

2616

sizeOfFirstHeight = 4

2643

sizeOfFirstHeight = 4

2617

sizeOfdeltaHeight = 4

2644

sizeOfdeltaHeight = 4

2618

sizeOfnHeights = 4

2645

sizeOfnHeights = 4

2619

sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows

2646

sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows

2620

processingHeaderSize += sizeOfWindows

2647

processingHeaderSize += sizeOfWindows

2621

2648

2622

self.processingHeaderObj.size = processingHeaderSize

2649

self.processingHeaderObj.size = processingHeaderSize

2623

2650

2624

class SpectraHeisWriter(Operation):

2651

class SpectraHeisWriter(Operation):

2625

# set = None

2652

# set = None

2626

setFile = None

2653

setFile = None

2627

idblock = None

2654

idblock = None

2628

doypath = None

2655

doypath = None

2629

subfolder = None

2656

subfolder = None

2630

2657

2631

def __init__(self):

2658

def __init__(self):

2632

self.wrObj = FITS()

2659

self.wrObj = FITS()

2633

# self.dataOut = dataOut

2660

# self.dataOut = dataOut

2634

self.nTotalBlocks=0

2661

self.nTotalBlocks=0

2635

# self.set = None

2662

# self.set = None

2636

self.setFile = None

2663

self.setFile = None

2637

self.idblock = 0

2664

self.idblock = 0

2638

self.wrpath = None

2665

self.wrpath = None

2639

self.doypath = None

2666

self.doypath = None

2640

self.subfolder = None

2667

self.subfolder = None

2641

self.isConfig = False

2668

self.isConfig = False

2642

2669

2643

def isNumber(str):

2670

def isNumber(str):

2644

"""

2671

"""

2645

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

2672

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

2646

2673

2647

Excepciones:

2674

Excepciones:

2648

Si un determinado string no puede ser convertido a numero

2675

Si un determinado string no puede ser convertido a numero

2649

Input:

2676

Input:

2650

str, string al cual se le analiza para determinar si convertible a un numero o no

2677

str, string al cual se le analiza para determinar si convertible a un numero o no

2651

2678

2652

Return:

2679

Return:

2653

True : si el string es uno numerico

2680

True : si el string es uno numerico

2654

False : no es un string numerico

2681

False : no es un string numerico

2655

"""

2682

"""

2656

try:

2683

try:

2657

float( str )

2684

float( str )

2658

return True

2685

return True

2659

except:

2686

except:

2660

return False

2687

return False

2661

2688

2662

def setup(self, dataOut, wrpath):

2689

def setup(self, dataOut, wrpath):

2663

2690

2664

if not(os.path.exists(wrpath)):

2691

if not(os.path.exists(wrpath)):

2665

os.mkdir(wrpath)

2692

os.mkdir(wrpath)

2666

2693

2667

self.wrpath = wrpath

2694

self.wrpath = wrpath

2668

# self.setFile = 0

2695

# self.setFile = 0

2669

self.dataOut = dataOut

2696

self.dataOut = dataOut

2670

2697

2671

def putData(self):

2698

def putData(self):

2672

name= time.localtime( self.dataOut.utctime)

2699

name= time.localtime( self.dataOut.utctime)

2673

ext=".fits"

2700

ext=".fits"

2674

2701

2675

if self.doypath == None:

2702

if self.doypath == None:

2676

self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))

2703

self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))

2677

self.doypath = os.path.join( self.wrpath, self.subfolder )

2704

self.doypath = os.path.join( self.wrpath, self.subfolder )

2678

os.mkdir(self.doypath)

2705

os.mkdir(self.doypath)

2679

2706

2680

if self.setFile == None:

2707

if self.setFile == None:

2681

# self.set = self.dataOut.set

2708

# self.set = self.dataOut.set

2682

self.setFile = 0

2709

self.setFile = 0

2683

# if self.set != self.dataOut.set:

2710

# if self.set != self.dataOut.set:

2684

## self.set = self.dataOut.set

2711

## self.set = self.dataOut.set

2685

# self.setFile = 0

2712

# self.setFile = 0

2686

2713

2687

#make the filename

2714

#make the filename

2688

file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)

2715

file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)

2689

2716

2690

filename = os.path.join(self.wrpath,self.subfolder, file)

2717

filename = os.path.join(self.wrpath,self.subfolder, file)

2691

2718

2692

idblock = numpy.array([self.idblock],dtype="int64")

2719

idblock = numpy.array([self.idblock],dtype="int64")

2693

header=self.wrObj.cFImage(idblock=idblock,

2720

header=self.wrObj.cFImage(idblock=idblock,

2694

year=time.gmtime(self.dataOut.utctime).tm_year,

2721

year=time.gmtime(self.dataOut.utctime).tm_year,

2695

month=time.gmtime(self.dataOut.utctime).tm_mon,

2722

month=time.gmtime(self.dataOut.utctime).tm_mon,

2696

day=time.gmtime(self.dataOut.utctime).tm_mday,

2723

day=time.gmtime(self.dataOut.utctime).tm_mday,

2697

hour=time.gmtime(self.dataOut.utctime).tm_hour,

2724

hour=time.gmtime(self.dataOut.utctime).tm_hour,

2698

minute=time.gmtime(self.dataOut.utctime).tm_min,

2725

minute=time.gmtime(self.dataOut.utctime).tm_min,

2699

second=time.gmtime(self.dataOut.utctime).tm_sec)

2726

second=time.gmtime(self.dataOut.utctime).tm_sec)

2700

2727

2701

c=3E8

2728

c=3E8

2702

deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2729

deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2703

freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))

2730

freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))

2704

2731

2705

colList = []

2732

colList = []

2706

2733

2707

colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)

2734

colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)

2708

2735

2709

colList.append(colFreq)

2736

colList.append(colFreq)

2710

2737

2711

nchannel=self.dataOut.nChannels

2738

nchannel=self.dataOut.nChannels

2712

2739

2713

for i in range(nchannel):

2740

for i in range(nchannel):

2714

col = self.wrObj.writeData(name="PCh"+str(i+1),

2741

col = self.wrObj.writeData(name="PCh"+str(i+1),

2715

format=str(self.dataOut.nFFTPoints)+'E',

2742

format=str(self.dataOut.nFFTPoints)+'E',

2716

data=10*numpy.log10(self.dataOut.data_spc[i,:]))

2743

data=10*numpy.log10(self.dataOut.data_spc[i,:]))

2717

2744

2718

colList.append(col)

2745

colList.append(col)

2719

2746

2720

data=self.wrObj.Ctable(colList=colList)

2747

data=self.wrObj.Ctable(colList=colList)

2721

2748

2722

self.wrObj.CFile(header,data)

2749

self.wrObj.CFile(header,data)

2723

2750

2724

self.wrObj.wFile(filename)

2751

self.wrObj.wFile(filename)

2725

2752

2726

#update the setFile

2753

#update the setFile

2727

self.setFile += 1

2754

self.setFile += 1

2728

self.idblock += 1

2755

self.idblock += 1

2729

2756

2730

return 1

2757

return 1

2731

2758

2732

def run(self, dataOut, **kwargs):

2759

def run(self, dataOut, **kwargs):

2733

2760

2734

if not(self.isConfig):

2761

if not(self.isConfig):

2735

2762

2736

self.setup(dataOut, **kwargs)

2763

self.setup(dataOut, **kwargs)

2737

self.isConfig = True

2764

self.isConfig = True

2738

2765

2739

self.putData()

2766

self.putData()

2740

2767

2741

2768

2742

class FITS:

2769

class FITS:

2743

name=None

2770

name=None

2744

format=None

2771

format=None

2745

array =None

2772

array =None

2746

data =None

2773

data =None

2747

thdulist=None

2774

thdulist=None

2748

prihdr=None

2775

prihdr=None

2749

hdu=None

2776

hdu=None

2750

2777

2751

def __init__(self):

2778

def __init__(self):

2752

2779

2753

pass

2780

pass

2754

2781

2755

def setColF(self,name,format,array):

2782

def setColF(self,name,format,array):

2756

self.name=name

2783

self.name=name

2757

self.format=format

2784

self.format=format

2758

self.array=array

2785

self.array=array

2759

a1=numpy.array([self.array],dtype=numpy.float32)

2786

a1=numpy.array([self.array],dtype=numpy.float32)

2760

self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)

2787

self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)

2761

return self.col1

2788

return self.col1

2762

2789

2763

# def setColP(self,name,format,data):

2790

# def setColP(self,name,format,data):

2764

# self.name=name

2791

# self.name=name

2765

# self.format=format

2792

# self.format=format

2766

# self.data=data

2793

# self.data=data

2767

# a2=numpy.array([self.data],dtype=numpy.float32)

2794

# a2=numpy.array([self.data],dtype=numpy.float32)

2768

# self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)

2795

# self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)

2769

# return self.col2

2796

# return self.col2

2770

2797

2771

2798

2772

def writeData(self,name,format,data):

2799

def writeData(self,name,format,data):

2773

self.name=name

2800

self.name=name

2774

self.format=format

2801

self.format=format

2775

self.data=data

2802

self.data=data

2776

a2=numpy.array([self.data],dtype=numpy.float32)

2803

a2=numpy.array([self.data],dtype=numpy.float32)

2777

self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)

2804

self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)

2778

return self.col2

2805

return self.col2

2779

2806

2780

def cFImage(self,idblock,year,month,day,hour,minute,second):

2807

def cFImage(self,idblock,year,month,day,hour,minute,second):

2781

self.hdu= pyfits.PrimaryHDU(idblock)

2808

self.hdu= pyfits.PrimaryHDU(idblock)

2782

self.hdu.header.set("Year",year)

2809

self.hdu.header.set("Year",year)

2783

self.hdu.header.set("Month",month)

2810

self.hdu.header.set("Month",month)

2784

self.hdu.header.set("Day",day)

2811

self.hdu.header.set("Day",day)

2785

self.hdu.header.set("Hour",hour)

2812

self.hdu.header.set("Hour",hour)

2786

self.hdu.header.set("Minute",minute)

2813

self.hdu.header.set("Minute",minute)

2787

self.hdu.header.set("Second",second)

2814

self.hdu.header.set("Second",second)

2788

return self.hdu

2815

return self.hdu

2789

2816

2790

2817

2791

def Ctable(self,colList):

2818

def Ctable(self,colList):

2792

self.cols=pyfits.ColDefs(colList)

2819

self.cols=pyfits.ColDefs(colList)

2793

self.tbhdu = pyfits.new_table(self.cols)

2820

self.tbhdu = pyfits.new_table(self.cols)

2794

return self.tbhdu

2821

return self.tbhdu

2795

2822

2796

2823

2797

def CFile(self,hdu,tbhdu):

2824

def CFile(self,hdu,tbhdu):

2798

self.thdulist=pyfits.HDUList([hdu,tbhdu])

2825

self.thdulist=pyfits.HDUList([hdu,tbhdu])

2799

2826

2800

def wFile(self,filename):

2827

def wFile(self,filename):

2801

if os.path.isfile(filename):

2828

if os.path.isfile(filename):

2802

os.remove(filename)

2829

os.remove(filename)

2803

self.thdulist.writeto(filename)

2830

self.thdulist.writeto(filename)

2804

2831

2805

2832

2806

class ParameterConf:

2833

class ParameterConf:

2807

ELEMENTNAME = 'Parameter'

2834

ELEMENTNAME = 'Parameter'

2808

def __init__(self):

2835

def __init__(self):

2809

self.name = ''

2836

self.name = ''

2810

self.value = ''

2837

self.value = ''

2811

2838

2812

def readXml(self, parmElement):

2839

def readXml(self, parmElement):

2813

self.name = parmElement.get('name')

2840

self.name = parmElement.get('name')

2814

self.value = parmElement.get('value')

2841

self.value = parmElement.get('value')

2815

2842

2816

def getElementName(self):

2843

def getElementName(self):

2817

return self.ELEMENTNAME

2844

return self.ELEMENTNAME

2818

2845

2819

class Metadata:

2846

class Metadata:

2820

2847

2821

def __init__(self, filename):

2848

def __init__(self, filename):

2822

self.parmConfObjList = []

2849

self.parmConfObjList = []

2823

self.readXml(filename)

2850

self.readXml(filename)

2824

2851

2825

def readXml(self, filename):

2852

def readXml(self, filename):

2826

self.projectElement = None

2853

self.projectElement = None

2827

self.procUnitConfObjDict = {}

2854

self.procUnitConfObjDict = {}

2828

self.projectElement = ElementTree().parse(filename)

2855

self.projectElement = ElementTree().parse(filename)

2829

self.project = self.projectElement.tag

2856

self.project = self.projectElement.tag

2830

2857

2831

parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())

2858

parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())

2832

2859

2833

for parmElement in parmElementList:

2860

for parmElement in parmElementList:

2834

parmConfObj = ParameterConf()

2861

parmConfObj = ParameterConf()

2835

parmConfObj.readXml(parmElement)

2862

parmConfObj.readXml(parmElement)

2836

self.parmConfObjList.append(parmConfObj)

2863

self.parmConfObjList.append(parmConfObj)

2837

2864

2838

class FitsWriter(Operation):

2865

class FitsWriter(Operation):

2839

2866

2840

def __init__(self):

2867

def __init__(self):

2841

self.isConfig = False

2868

self.isConfig = False

2842

self.dataBlocksPerFile = None

2869

self.dataBlocksPerFile = None

2843

self.blockIndex = 0

2870

self.blockIndex = 0

2844

self.flagIsNewFile = 1

2871

self.flagIsNewFile = 1

2845

self.fitsObj = None

2872

self.fitsObj = None

2846

self.optchar = 'P'

2873

self.optchar = 'P'

2847

self.ext = '.fits'

2874

self.ext = '.fits'

2848

self.setFile = 0

2875

self.setFile = 0

2849

2876

2850

def setFitsHeader(self, dataOut, metadatafile):

2877

def setFitsHeader(self, dataOut, metadatafile):

2851

2878

2852

header_data = pyfits.PrimaryHDU()

2879

header_data = pyfits.PrimaryHDU()

2853

2880

2854

metadata4fits = Metadata(metadatafile)

2881

metadata4fits = Metadata(metadatafile)

2855

for parameter in metadata4fits.parmConfObjList:

2882

for parameter in metadata4fits.parmConfObjList:

2856

parm_name = parameter.name

2883

parm_name = parameter.name

2857

parm_value = parameter.value

2884

parm_value = parameter.value

2858

2885

2859

if parm_value == 'fromdatadatetime':

2886

if parm_value == 'fromdatadatetime':

2860

value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())

2887

value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())

2861

elif parm_value == 'fromdataheights':

2888

elif parm_value == 'fromdataheights':

2862

value = dataOut.nHeights

2889

value = dataOut.nHeights

2863

elif parm_value == 'fromdatachannel':

2890

elif parm_value == 'fromdatachannel':

2864

value = dataOut.nChannels

2891

value = dataOut.nChannels

2865

elif parm_value == 'fromdatasamples':

2892

elif parm_value == 'fromdatasamples':

2866

value = dataOut.nFFTPoints

2893

value = dataOut.nFFTPoints

2867

else:

2894

else:

2868

value = parm_value

2895

value = parm_value

2869

2896

2870

header_data.header[parm_name] = value

2897

header_data.header[parm_name] = value

2871

2898

2872

header_data.header['NBLOCK'] = self.blockIndex

2899

header_data.header['NBLOCK'] = self.blockIndex

2873

2900

2874

header_data.writeto(self.filename)

2901

header_data.writeto(self.filename)

2875

2902

2876

2903

2877

def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):

2904

def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):

2878

2905

2879

self.path = path

2906

self.path = path

2880

self.dataOut = dataOut

2907

self.dataOut = dataOut

2881

self.metadatafile = metadatafile

2908

self.metadatafile = metadatafile

2882

self.dataBlocksPerFile = dataBlocksPerFile

2909

self.dataBlocksPerFile = dataBlocksPerFile

2883

2910

2884

def open(self):

2911

def open(self):

2885

self.fitsObj = pyfits.open(self.filename, mode='update')

2912

self.fitsObj = pyfits.open(self.filename, mode='update')

2886

2913

2887

2914

2888

def addData(self, data):

2915

def addData(self, data):

2889

self.open()

2916

self.open()

2890

extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATA'])

2917

extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATA'])

2891

extension.header['UTCTIME'] = self.dataOut.utctime

2918

extension.header['UTCTIME'] = self.dataOut.utctime

2892

self.fitsObj.append(extension)

2919

self.fitsObj.append(extension)

2893

self.blockIndex += 1

2920

self.blockIndex += 1

2894

self.fitsObj[0].header['NBLOCK'] = self.blockIndex

2921

self.fitsObj[0].header['NBLOCK'] = self.blockIndex

2895

2922

2896

self.write()

2923

self.write()

2897

2924

2898

def write(self):

2925

def write(self):

2899

2926

2900

self.fitsObj.flush(verbose=True)

2927

self.fitsObj.flush(verbose=True)

2901

self.fitsObj.close()

2928

self.fitsObj.close()

2902

2929

2903

2930

2904

def setNextFile(self):

2931

def setNextFile(self):

2905

2932

2906

ext = self.ext

2933

ext = self.ext

2907

path = self.path

2934

path = self.path

2908

2935

2909

timeTuple = time.localtime( self.dataOut.utctime)

2936

timeTuple = time.localtime( self.dataOut.utctime)

2910

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

2937

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

2911

2938

2912

fullpath = os.path.join( path, subfolder )

2939

fullpath = os.path.join( path, subfolder )

2913

if not( os.path.exists(fullpath) ):

2940

if not( os.path.exists(fullpath) ):

2914

os.mkdir(fullpath)

2941

os.mkdir(fullpath)

2915

self.setFile = -1 #inicializo mi contador de seteo

2942

self.setFile = -1 #inicializo mi contador de seteo

2916

else:

2943

else:

2917

filesList = os.listdir( fullpath )

2944

filesList = os.listdir( fullpath )

2918

if len( filesList ) > 0:

2945

if len( filesList ) > 0:

2919

filesList = sorted( filesList, key=str.lower )

2946

filesList = sorted( filesList, key=str.lower )

2920

filen = filesList[-1]

2947

filen = filesList[-1]

2921

2948

2922

if isNumber( filen[8:11] ):

2949

if isNumber( filen[8:11] ):

2923

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

2950

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

2924

else:

2951

else:

2925

self.setFile = -1

2952

self.setFile = -1

2926

else:

2953

else:

2927

self.setFile = -1 #inicializo mi contador de seteo

2954

self.setFile = -1 #inicializo mi contador de seteo

2928

2955

2929

setFile = self.setFile

2956

setFile = self.setFile

2930

setFile += 1

2957

setFile += 1

2931

2958

2932

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

2959

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

2933

timeTuple.tm_year,

2960

timeTuple.tm_year,

2934

timeTuple.tm_yday,

2961

timeTuple.tm_yday,

2935

setFile,

2962

setFile,

2936

ext )

2963

ext )

2937

2964

2938

filename = os.path.join( path, subfolder, file )

2965

filename = os.path.join( path, subfolder, file )

2939

2966

2940

self.blockIndex = 0

2967

self.blockIndex = 0

2941

self.filename = filename

2968

self.filename = filename

2942

self.setFile = setFile

2969

self.setFile = setFile

2943

self.flagIsNewFile = 1

2970

self.flagIsNewFile = 1

2944

2971

2945

print 'Writing the file: %s'%self.filename

2972

print 'Writing the file: %s'%self.filename

2946

2973

2947

self.setFitsHeader(self.dataOut, self.metadatafile)

2974

self.setFitsHeader(self.dataOut, self.metadatafile)

2948

2975

2949

return 1

2976

return 1

2950

2977

2951

def writeBlock(self):

2978

def writeBlock(self):

2952

self.addData(self.dataOut.data_spc)

2979

self.addData(self.dataOut.data_spc)

2953

self.flagIsNewFile = 0

2980

self.flagIsNewFile = 0

2954

2981

2955

2982

2956

def __setNewBlock(self):

2983

def __setNewBlock(self):

2957

2984

2958

if self.flagIsNewFile:

2985

if self.flagIsNewFile:

2959

return 1

2986

return 1

2960

2987

2961

if self.blockIndex < self.dataBlocksPerFile:

2988

if self.blockIndex < self.dataBlocksPerFile:

2962

return 1

2989

return 1

2963

2990

2964

if not( self.setNextFile() ):

2991

if not( self.setNextFile() ):

2965

return 0

2992

return 0

2966

2993

2967

return 1

2994

return 1

2968

2995

2969

def writeNextBlock(self):

2996

def writeNextBlock(self):

2970

if not( self.__setNewBlock() ):

2997

if not( self.__setNewBlock() ):

2971

return 0

2998

return 0

2972

self.writeBlock()

2999

self.writeBlock()

2973

return 1

3000

return 1

2974

3001

2975

def putData(self):

3002

def putData(self):

2976

if self.flagIsNewFile:

3003

if self.flagIsNewFile:

2977

self.setNextFile()

3004

self.setNextFile()

2978

self.writeNextBlock()

3005

self.writeNextBlock()

2979

3006

2980

def run(self, dataOut, **kwargs):

3007

def run(self, dataOut, **kwargs):

2981

if not(self.isConfig):

3008

if not(self.isConfig):

2982

self.setup(dataOut, **kwargs)

3009

self.setup(dataOut, **kwargs)

2983

self.isConfig = True

3010

self.isConfig = True

2984

self.putData()

3011

self.putData()

2985

3012

2986

3013

2987

class FitsReader(ProcessingUnit):

3014

class FitsReader(ProcessingUnit):

2988

3015

2989

__TIMEZONE = time.timezone

3016

__TIMEZONE = time.timezone

2990

3017

2991

expName = None

3018

expName = None

2992

datetimestr = None

3019

datetimestr = None

2993

utc = None

3020

utc = None

2994

nChannels = None

3021

nChannels = None

2995

nSamples = None

3022

nSamples = None

2996

dataBlocksPerFile = None

3023

dataBlocksPerFile = None

2997

comments = None

3024

comments = None

2998

lastUTTime = None

3025

lastUTTime = None

2999

header_dict = None

3026

header_dict = None

3000

data = None

3027

data = None

3001

data_header_dict = None

3028

data_header_dict = None

3002

3029

3003

def __init__(self):

3030

def __init__(self):

3004

self.isConfig = False

3031

self.isConfig = False

3005

self.ext = '.fits'

3032

self.ext = '.fits'

3006

self.setFile = 0

3033

self.setFile = 0

3007

self.flagNoMoreFiles = 0

3034

self.flagNoMoreFiles = 0

3008

self.flagIsNewFile = 1

3035

self.flagIsNewFile = 1

3009

self.flagTimeBlock = None

3036

self.flagTimeBlock = None

3010

self.fileIndex = None

3037

self.fileIndex = None

3011

self.filename = None

3038

self.filename = None

3012

self.fileSize = None

3039

self.fileSize = None

3013

self.fitsObj = None

3040

self.fitsObj = None

3014

self.nReadBlocks = 0

3041

self.nReadBlocks = 0

3015

self.nTotalBlocks = 0

3042

self.nTotalBlocks = 0

3016

self.dataOut = self.createObjByDefault()

3043

self.dataOut = self.createObjByDefault()

3017

self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()

3044

self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()

3018

self.blockIndex = 1

3045

self.blockIndex = 1

3019

3046

3020

def createObjByDefault(self):

3047

def createObjByDefault(self):

3021

3048

3022

dataObj = Fits()

3049

dataObj = Fits()

3023

3050

3024

return dataObj

3051

return dataObj

3025

3052

3026

def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):

3053

def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):

3027

try:

3054

try:

3028

fitsObj = pyfits.open(filename,'readonly')

3055

fitsObj = pyfits.open(filename,'readonly')

3029

except:

3056

except:

3030

raise IOError, "The file %s can't be opened" %(filename)

3057

raise IOError, "The file %s can't be opened" %(filename)

3031

3058

3032

header = fitsObj[0].header

3059

header = fitsObj[0].header

3033

struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")

3060

struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")

3034

utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS

3061

utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS

3035

3062

3036

ltc = utc

3063

ltc = utc

3037

if useLocalTime:

3064

if useLocalTime:

3038

ltc -= time.timezone

3065

ltc -= time.timezone

3039

thisDatetime = datetime.datetime.utcfromtimestamp(ltc)

3066

thisDatetime = datetime.datetime.utcfromtimestamp(ltc)

3040

thisTime = thisDatetime.time()

3067

thisTime = thisDatetime.time()

3041

3068

3042

if not ((startTime <= thisTime) and (endTime > thisTime)):

3069

if not ((startTime <= thisTime) and (endTime > thisTime)):

3043

return None

3070

return None

3044

3071

3045

return thisDatetime

3072

return thisDatetime

3046

3073

3047

def __setNextFileOnline(self):

3074

def __setNextFileOnline(self):

3048

raise ValueError, "No implemented"

3075

raise ValueError, "No implemented"

3049

3076

3050

def __setNextFileOffline(self):

3077

def __setNextFileOffline(self):

3051

idFile = self.fileIndex

3078

idFile = self.fileIndex

3052

3079

3053

while (True):

3080

while (True):

3054

idFile += 1

3081

idFile += 1

3055

if not(idFile < len(self.filenameList)):

3082

if not(idFile < len(self.filenameList)):

3056

self.flagNoMoreFiles = 1

3083

self.flagNoMoreFiles = 1

3057

print "No more Files"

3084

print "No more Files"

3058

return 0

3085

return 0

3059

3086

3060

filename = self.filenameList[idFile]

3087

filename = self.filenameList[idFile]

3061

3088

3062

# if not(self.__verifyFile(filename)):

3089

# if not(self.__verifyFile(filename)):

3063

# continue

3090

# continue

3064

3091

3065

fileSize = os.path.getsize(filename)

3092

fileSize = os.path.getsize(filename)

3066

fitsObj = pyfits.open(filename,'readonly')

3093

fitsObj = pyfits.open(filename,'readonly')

3067

break

3094

break

3068

3095

3069

self.flagIsNewFile = 1

3096

self.flagIsNewFile = 1

3070

self.fileIndex = idFile

3097

self.fileIndex = idFile

3071

self.filename = filename

3098

self.filename = filename

3072

self.fileSize = fileSize

3099

self.fileSize = fileSize

3073

self.fitsObj = fitsObj

3100

self.fitsObj = fitsObj

3074

3101

3075

print "Setting the file: %s"%self.filename

3102

print "Setting the file: %s"%self.filename

3076

3103

3077

return 1

3104

return 1

3078

3105

3079

def readHeader(self):

3106

def readHeader(self):

3080

headerObj = self.fitsObj[0]

3107

headerObj = self.fitsObj[0]

3081

3108

3082

self.header_dict = headerObj.header

3109

self.header_dict = headerObj.header

3083

self.expName = headerObj.header['EXPNAME']

3110

self.expName = headerObj.header['EXPNAME']

3084

self.datetimestr = headerObj.header['DATETIME']

3111

self.datetimestr = headerObj.header['DATETIME']

3085

struct_time = time.strptime(headerObj.header['DATETIME'], "%b %d %Y %H:%M:%S")

3112

struct_time = time.strptime(headerObj.header['DATETIME'], "%b %d %Y %H:%M:%S")

3086

# self.utc = time.mktime(struct_time) - self.__TIMEZONE

3113

# self.utc = time.mktime(struct_time) - self.__TIMEZONE

3087

self.nChannels = headerObj.header['NCHANNEL']

3114

self.nChannels = headerObj.header['NCHANNEL']

3088

self.nSamples = headerObj.header['NSAMPLE']

3115

self.nSamples = headerObj.header['NSAMPLE']

3089

self.dataBlocksPerFile = headerObj.header['NBLOCK']

3116

self.dataBlocksPerFile = headerObj.header['NBLOCK']

3090

self.comments = headerObj.header['COMMENT']

3117

self.comments = headerObj.header['COMMENT']

3091

3118

3092

3119

3093

def setNextFile(self):

3120

def setNextFile(self):

3094

3121

3095

if self.online:

3122

if self.online:

3096

newFile = self.__setNextFileOnline()

3123

newFile = self.__setNextFileOnline()

3097

else:

3124

else:

3098

newFile = self.__setNextFileOffline()

3125

newFile = self.__setNextFileOffline()

3099

3126

3100

if not(newFile):

3127

if not(newFile):

3101

return 0

3128

return 0

3102

3129

3103

self.readHeader()

3130

self.readHeader()

3104

3131

3105

self.nReadBlocks = 0

3132

self.nReadBlocks = 0

3106

self.blockIndex = 1

3133

self.blockIndex = 1

3107

return 1

3134

return 1

3108

3135

3109

def __searchFilesOffLine(self,

3136

def __searchFilesOffLine(self,

3110

path,

3137

path,

3111

startDate,

3138

startDate,

3112

endDate,

3139

endDate,

3113

startTime=datetime.time(0,0,0),

3140

startTime=datetime.time(0,0,0),

3114

endTime=datetime.time(23,59,59),

3141

endTime=datetime.time(23,59,59),

3115

set=None,

3142

set=None,

3116

expLabel='',

3143

expLabel='',

3117

ext='.fits',

3144

ext='.fits',

3118

walk=True):

3145

walk=True):

3119

3146

3120

pathList = []

3147

pathList = []

3121

3148

3122

if not walk:

3149

if not walk:

3123

pathList.append(path)

3150

pathList.append(path)

3124

3151

3125

else:

3152

else:

3126

dirList = []

3153

dirList = []

3127

for thisPath in os.listdir(path):

3154

for thisPath in os.listdir(path):

3128

if not os.path.isdir(os.path.join(path,thisPath)):

3155

if not os.path.isdir(os.path.join(path,thisPath)):

3129

continue

3156

continue

3130

if not isDoyFolder(thisPath):

3157

if not isDoyFolder(thisPath):

3131

continue

3158

continue

3132

3159

3133

dirList.append(thisPath)

3160

dirList.append(thisPath)

3134

3161

3135

if not(dirList):

3162

if not(dirList):

3136

return None, None

3163

return None, None

3137

3164

3138

thisDate = startDate

3165

thisDate = startDate

3139

3166

3140

while(thisDate <= endDate):

3167

while(thisDate <= endDate):

3141

year = thisDate.timetuple().tm_year

3168

year = thisDate.timetuple().tm_year

3142

doy = thisDate.timetuple().tm_yday

3169

doy = thisDate.timetuple().tm_yday

3143

3170

3144

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

3171

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

3145

if len(matchlist) == 0:

3172

if len(matchlist) == 0:

3146

thisDate += datetime.timedelta(1)

3173

thisDate += datetime.timedelta(1)

3147

continue

3174

continue

3148

for match in matchlist:

3175

for match in matchlist:

3149

pathList.append(os.path.join(path,match,expLabel))

3176

pathList.append(os.path.join(path,match,expLabel))

3150

3177

3151

thisDate += datetime.timedelta(1)

3178

thisDate += datetime.timedelta(1)

3152

3179

3153

if pathList == []:

3180

if pathList == []:

3154

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

3181

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

3155

return None, None

3182

return None, None

3156

3183

3157

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

3184

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

3158

3185

3159

filenameList = []

3186

filenameList = []

3160

datetimeList = []

3187

datetimeList = []

3161

3188

3162

for i in range(len(pathList)):

3189

for i in range(len(pathList)):

3163

3190

3164

thisPath = pathList[i]

3191

thisPath = pathList[i]

3165

3192

3166

fileList = glob.glob1(thisPath, "*%s" %ext)

3193

fileList = glob.glob1(thisPath, "*%s" %ext)

3167

fileList.sort()

3194

fileList.sort()

3168

3195

3169

for file in fileList:

3196

for file in fileList:

3170

3197

3171

filename = os.path.join(thisPath,file)

3198

filename = os.path.join(thisPath,file)

3172

thisDatetime = self.isFileinThisTime(filename, startTime, endTime, useLocalTime=True)

3199

thisDatetime = self.isFileinThisTime(filename, startTime, endTime, useLocalTime=True)

3173

3200

3174

if not(thisDatetime):

3201

if not(thisDatetime):

3175

continue

3202

continue

3176

3203

3177

filenameList.append(filename)

3204

filenameList.append(filename)

3178

datetimeList.append(thisDatetime)

3205

datetimeList.append(thisDatetime)

3179

3206

3180

if not(filenameList):

3207

if not(filenameList):

3181

print "Any file was found for the time range %s - %s" %(startTime, endTime)

3208

print "Any file was found for the time range %s - %s" %(startTime, endTime)

3182

return None, None

3209

return None, None

3183

3210

3184

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

3211

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

3185

print

3212

print

3186

3213

3187

for i in range(len(filenameList)):

3214

for i in range(len(filenameList)):

3188

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

3215

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

3189

3216

3190

self.filenameList = filenameList

3217

self.filenameList = filenameList

3191

self.datetimeList = datetimeList

3218

self.datetimeList = datetimeList

3192

3219

3193

return pathList, filenameList

3220

return pathList, filenameList

3194

3221

3195

def setup(self, path=None,

3222

def setup(self, path=None,

3196

startDate=None,

3223

startDate=None,

3197

endDate=None,

3224

endDate=None,

3198

startTime=datetime.time(0,0,0),

3225

startTime=datetime.time(0,0,0),

3199

endTime=datetime.time(23,59,59),

3226

endTime=datetime.time(23,59,59),

3200

set=0,

3227

set=0,

3201

expLabel = "",

3228

expLabel = "",

3202

ext = None,

3229

ext = None,

3203

online = False,

3230

online = False,

3204

delay = 60,

3231

delay = 60,

3205

walk = True):

3232

walk = True):

3206

3233

3207

if path == None:

3234

if path == None:

3208

raise ValueError, "The path is not valid"

3235

raise ValueError, "The path is not valid"

3209

3236

3210

if ext == None:

3237

if ext == None:

3211

ext = self.ext

3238

ext = self.ext

3212

3239

3213

if not(online):

3240

if not(online):

3214

print "Searching files in offline mode ..."

3241

print "Searching files in offline mode ..."

3215

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

3242

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

3216

startTime=startTime, endTime=endTime,

3243

startTime=startTime, endTime=endTime,

3217

set=set, expLabel=expLabel, ext=ext,

3244

set=set, expLabel=expLabel, ext=ext,

3218

walk=walk)

3245

walk=walk)

3219

3246

3220

if not(pathList):

3247

if not(pathList):

3221

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

3248

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

3222

datetime.datetime.combine(startDate,startTime).ctime(),

3249

datetime.datetime.combine(startDate,startTime).ctime(),

3223

datetime.datetime.combine(endDate,endTime).ctime())

3250

datetime.datetime.combine(endDate,endTime).ctime())

3224

3251

3225

sys.exit(-1)

3252

sys.exit(-1)

3226

3253

3227

self.fileIndex = -1

3254

self.fileIndex = -1

3228

self.pathList = pathList

3255

self.pathList = pathList

3229

self.filenameList = filenameList

3256

self.filenameList = filenameList

3230

3257

3231

self.online = online

3258

self.online = online

3232

self.delay = delay

3259

self.delay = delay

3233

ext = ext.lower()

3260

ext = ext.lower()

3234

self.ext = ext

3261

self.ext = ext

3235

3262

3236

if not(self.setNextFile()):

3263

if not(self.setNextFile()):

3237

if (startDate!=None) and (endDate!=None):

3264

if (startDate!=None) and (endDate!=None):

3238

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

3265

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

3239

elif startDate != None:

3266

elif startDate != None:

3240

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

3267

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

3241

else:

3268

else:

3242

print "No files"

3269

print "No files"

3243

3270

3244

sys.exit(-1)

3271

sys.exit(-1)

3245

3272

3246

3273

3247

3274

3248

def readBlock(self):

3275

def readBlock(self):

3249

dataObj = self.fitsObj[self.blockIndex]

3276

dataObj = self.fitsObj[self.blockIndex]

3250

3277

3251

self.data = dataObj.data

3278

self.data = dataObj.data

3252

self.data_header_dict = dataObj.header

3279

self.data_header_dict = dataObj.header

3253

self.utc = self.data_header_dict['UTCTIME']

3280

self.utc = self.data_header_dict['UTCTIME']

3254

3281

3255

self.flagIsNewFile = 0

3282

self.flagIsNewFile = 0

3256

self.blockIndex += 1

3283

self.blockIndex += 1

3257

self.nTotalBlocks += 1

3284

self.nTotalBlocks += 1

3258

self.nReadBlocks += 1

3285

self.nReadBlocks += 1

3259

3286

3260

return 1

3287

return 1

3261

3288

3262

def __jumpToLastBlock(self):

3289

def __jumpToLastBlock(self):

3263

raise ValueError, "No implemented"

3290

raise ValueError, "No implemented"

3264

3291

3265

def __waitNewBlock(self):

3292

def __waitNewBlock(self):

3266

"""

3293

"""

3267

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

3294

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

3268

3295

3269

Si el modo de lectura es OffLine siempre retorn 0

3296

Si el modo de lectura es OffLine siempre retorn 0

3270

"""

3297

"""

3271

if not self.online:

3298

if not self.online:

3272

return 0

3299

return 0

3273

3300

3274

if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):

3301

if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):

3275

return 0

3302

return 0

3276

3303

3277

currentPointer = self.fp.tell()

3304

currentPointer = self.fp.tell()

3278

3305

3279

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

3306

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

3280

3307

3281

for nTries in range( self.nTries ):

3308

for nTries in range( self.nTries ):

3282

3309

3283

self.fp.close()

3310

self.fp.close()

3284

self.fp = open( self.filename, 'rb' )

3311

self.fp = open( self.filename, 'rb' )

3285

self.fp.seek( currentPointer )

3312

self.fp.seek( currentPointer )

3286

3313

3287

self.fileSize = os.path.getsize( self.filename )

3314

self.fileSize = os.path.getsize( self.filename )

3288

currentSize = self.fileSize - currentPointer

3315

currentSize = self.fileSize - currentPointer

3289

3316

3290

if ( currentSize >= neededSize ):

3317

if ( currentSize >= neededSize ):

3291

self.__rdBasicHeader()

3318

self.__rdBasicHeader()

3292

return 1

3319

return 1

3293

3320

3294

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

3321

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

3295

time.sleep( self.delay )

3322

time.sleep( self.delay )

3296

3323

3297

3324

3298

return 0

3325

return 0

3299

3326

3300

def __setNewBlock(self):

3327

def __setNewBlock(self):

3301

3328

3302

if self.online:

3329

if self.online:

3303

self.__jumpToLastBlock()

3330

self.__jumpToLastBlock()

3304

3331

3305

if self.flagIsNewFile:

3332

if self.flagIsNewFile:

3306

return 1

3333

return 1

3307

3334

3308

self.lastUTTime = self.utc

3335

self.lastUTTime = self.utc

3309

3336

3310

if self.online:

3337

if self.online:

3311

if self.__waitNewBlock():

3338

if self.__waitNewBlock():

3312

return 1

3339

return 1

3313

3340

3314

if self.nReadBlocks < self.dataBlocksPerFile:

3341

if self.nReadBlocks < self.dataBlocksPerFile:

3315

return 1

3342

return 1

3316

3343

3317

if not(self.setNextFile()):

3344

if not(self.setNextFile()):

3318

return 0

3345

return 0

3319

3346

3320

deltaTime = self.utc - self.lastUTTime

3347

deltaTime = self.utc - self.lastUTTime

3321

3348

3322

self.flagTimeBlock = 0

3349

self.flagTimeBlock = 0

3323

3350

3324

if deltaTime > self.maxTimeStep:

3351

if deltaTime > self.maxTimeStep:

3325

self.flagTimeBlock = 1

3352

self.flagTimeBlock = 1

3326

3353

3327

return 1

3354

return 1

3328

3355

3329

3356

3330

def readNextBlock(self):

3357

def readNextBlock(self):

3331

if not(self.__setNewBlock()):

3358

if not(self.__setNewBlock()):

3332

return 0

3359

return 0

3333

3360

3334

if not(self.readBlock()):

3361

if not(self.readBlock()):

3335

return 0

3362

return 0

3336

3363

3337

return 1

3364

return 1

3338

3365

3339

3366

3340

def getData(self):

3367

def getData(self):

3341

3368

3342

if self.flagNoMoreFiles:

3369

if self.flagNoMoreFiles:

3343

self.dataOut.flagNoData = True

3370

self.dataOut.flagNoData = True

3344

print 'Process finished'

3371

print 'Process finished'

3345

return 0

3372

return 0

3346

3373

3347

self.flagTimeBlock = 0

3374

self.flagTimeBlock = 0

3348

self.flagIsNewBlock = 0

3375

self.flagIsNewBlock = 0

3349

3376

3350

if not(self.readNextBlock()):

3377

if not(self.readNextBlock()):

3351

return 0

3378

return 0

3352

3379

3353

if self.data == None:

3380

if self.data == None:

3354

self.dataOut.flagNoData = True

3381

self.dataOut.flagNoData = True

3355

return 0

3382

return 0

3356

3383

3357

self.dataOut.data = self.data

3384

self.dataOut.data = self.data

3358

self.dataOut.data_header = self.data_header_dict

3385

self.dataOut.data_header = self.data_header_dict

3359

self.dataOut.utctime = self.utc

3386

self.dataOut.utctime = self.utc

3360

3387

3361

self.dataOut.header = self.header_dict

3388

self.dataOut.header = self.header_dict

3362

self.dataOut.expName = self.expName

3389

self.dataOut.expName = self.expName

3363

self.dataOut.nChannels = self.nChannels

3390

self.dataOut.nChannels = self.nChannels

3364

self.dataOut.nSamples = self.nSamples

3391

self.dataOut.nSamples = self.nSamples

3365

self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile

3392

self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile

3366

self.dataOut.comments = self.comments

3393

self.dataOut.comments = self.comments

3367

3394

3368

self.dataOut.flagNoData = False

3395

self.dataOut.flagNoData = False

3369

3396

3370

return self.dataOut.data

3397

return self.dataOut.data

3371

3398

3372

def run(self, **kwargs):

3399

def run(self, **kwargs):

3373

3400

3374

if not(self.isConfig):

3401

if not(self.isConfig):

3375

self.setup(**kwargs)

3402

self.setup(**kwargs)

3376

self.isConfig = True

3403

self.isConfig = True

3377

3404

3378

self.getData() No newline at end of file

3405

self.getData()

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages