schain Commit - r449:336ad0c1a9b5 · Jicamarca Repository

             '''
             $Author: murco $
             $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
             '''
             import os, sys
             import glob
             import time
             import numpy
             import fnmatch
             import time, datetime
             from xml.etree.ElementTree import Element, SubElement, ElementTree
             try:
                 import pyfits
             except:
                 print "pyfits module has not been imported, it should be installed to save files in fits format"
             from jrodata import *
             from jroheaderIO import *
             from jroprocessing import *
             LOCALTIME = True #-18000
             def isNumber(str):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float( str )
                     return True
                 except:
                     return False
             def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename,'rb')
                 except:
                     raise IOError, "The file %s can't be opened" %(filename)
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isFileinThisTime(filename, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename,'rb')
                 except:
                     raise IOError, "The file %s can't be opened" %(filename)
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = basicHeaderObj.datatime
                 thisTime = basicHeaderObj.datatime.time()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return None
                 if not ((startTime <= thisTime) and (endTime > thisTime)):
                     return None
                 return thisDatetime
             def getFileFromSet(path,ext,set):
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for file in fileList:
                     try:
                         year = int(file[1:5])
                         doy  = int(file[5:8])
                     except:
                         continue
                     if (os.path.splitext(file)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(file)
                 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
                 if len(myfile)!= 0:
                     return myfile[0]
                 else:
                     filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
                     print 'the filename %s does not exist'%filename
                     print '...going to the last file: '
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for file in fileList:
                     try:
                         year = int(file[1:5])
                         doy  = int(file[5:8])
                     except:
                         continue
                     if (os.path.splitext(file)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(file)
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def checkForRealPath(path, foldercounter, year, doy, set, ext):
                 """
                 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
                 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
                 el path exacto de un determinado file.
                 Example    :
                     nombre correcto del file es  .../.../D2009307/P2009307367.ext
                     Entonces la funcion prueba con las siguientes combinaciones
                         .../.../y2009307367.ext
                         .../.../Y2009307367.ext
                         .../.../x2009307/y2009307367.ext
                         .../.../x2009307/Y2009307367.ext
                         .../.../X2009307/y2009307367.ext
                         .../.../X2009307/Y2009307367.ext
                     siendo para este caso, la ultima combinacion de letras, identica al file buscado
                 Return:
                     Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
                     caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
                     para el filename
                 """
                 fullfilename = None
                 find_flag = False
                 filename = None
                 prefixDirList = [None,'d','D']
                 if ext.lower() == ".r": #voltage
                     prefixFileList = ['d','D']
                 elif ext.lower() == ".pdata": #spectra
                     prefixFileList = ['p','P']
                 else:
                     return None, filename
                 #barrido por las combinaciones posibles
                 for prefixDir in prefixDirList:
                     thispath = path
                     if prefixDir != None:
                         #formo el nombre del directorio xYYYYDDD (x=d o x=D)
                         if foldercounter == 0:
                             thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
                         else:
                             thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
                     for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
                         filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
                         fullfilename = os.path.join( thispath, filename ) #formo el path completo
                         if os.path.exists( fullfilename ): #verifico que exista
                             find_flag = True
                             break
                     if find_flag:
                         break
                 if not(find_flag):
                     return None, filename
                 return fullfilename, filename
             def isDoyFolder(folder):
                 try:
                     year = int(folder[1:5])
                 except:
                     return 0
                 try:
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
             class JRODataIO:
                 c = 3E8
                 isConfig = False
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
                 processingHeaderObj = ProcessingHeader()
                 online = 0
                 dtype = None
                 pathList = []
                 filenameList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagTimeBlock = 0
                 flagIsNewBlock = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
                 ippSeconds = None
                 fileSizeByHeader = None
                 fileIndex = None
                 profileIndex = None
                 blockIndex = None
                 nTotalBlocks = None
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
                 blocksize = None
                 def __init__(self):
                     raise ValueError, "Not implemented"
                 def run(self):
                     raise ValueError, "Not implemented"
                 def getOutput(self):
                     return self.dataOut
             class JRODataReader(JRODataIO, ProcessingUnit):
                 nReadBlocks = 0
                 delay  = 10   #number of seconds waiting a new file
                 nTries  = 3  #quantity tries
                 nFiles = 3   #number of files for searching
                 path = None
                 foldercounter = 0
                 flagNoMoreFiles = 0
                 datetimeList = []
                 __isFirstTimeOnline = 1
                 __printInfo = True
                 profileIndex = None
                 def __init__(self):
                     """
                     """
                     raise ValueError, "This method has not been implemented"
                 def createObjByDefault(self):
                     """
                     """
                     raise ValueError, "This method has not been implemented"
                 def getBlockDimension(self):
                     raise ValueError, "No implemented"
                 def __searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.r',
                                         walk=True):
                     pathList = []
                     if not walk:
-                        pathList.append(path)
+                        #pathList.append(path)
+                        multi_path = path.split(',')
+                        for single_path in multi_path:
+                            pathList.append(single_path)
                     else:
-                        dirList = []
+                        #dirList = []
-                        for thisPath in os.listdir(path):
+                        multi_path = path.split(',')
-                            if not os.path.isdir(os.path.join(path,thisPath)):
+                        for single_path in multi_path:
-                                continue
+                            dirList = []
-                            if not isDoyFolder(thisPath):
+                            for thisPath in os.listdir(single_path):
-                                continue
+                                if not os.path.isdir(os.path.join(single_path,thisPath)):
+                                    continue
+                                if not isDoyFolder(thisPath):
+                                    continue
-                            dirList.append(thisPath)
+                                dirList.append(thisPath)
-                        if not(dirList):
+                            if not(dirList):
-                            return None, None
+                                return None, None
-                        thisDate = startDate
+                            thisDate = startDate
-                        while(thisDate <= endDate):
+                            while(thisDate <= endDate):
-                            year = thisDate.timetuple().tm_year
+                                year = thisDate.timetuple().tm_year
-                            doy = thisDate.timetuple().tm_yday
+                                doy = thisDate.timetuple().tm_yday
-                            matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
+                                matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
-                            if len(matchlist) == 0:
+                                if len(matchlist) == 0:
+                                    thisDate += datetime.timedelta(1)
+                                    continue
+                                for match in matchlist:
+                                    pathList.append(os.path.join(single_path,match,expLabel))
                                 thisDate += datetime.timedelta(1)
-                                continue
-                            for match in matchlist:
-                                pathList.append(os.path.join(path,match,expLabel))
-                            thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
                         return None, None
                     print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
                     filenameList = []
                     datetimeList = []
+                    pathDict = {}
+                    filenameList_to_sort = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
+                        pathDict.setdefault(fileList[0])
+                        pathDict[fileList[0]] = i
+                        filenameList_to_sort.append(fileList[0])
+                    filenameList_to_sort.sort()
+                    for file in filenameList_to_sort:
+                        thisPath = pathList[pathDict[file]]
+                        fileList = glob.glob1(thisPath, "*%s" %ext)
+                        fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             thisDatetime = isFileinThisTime(filename, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "Any file was found for the time range %s - %s" %(startTime, endTime)
                         return None, None
                     print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
                     """
                     Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
                     devuelve el archivo encontrado ademas de otros datos.
                     Input:
                         path           :    carpeta donde estan contenidos los files que contiene data
                         expLabel        :     Nombre del subexperimento (subfolder)
                         ext              :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
                         directory   :    eL directorio donde esta el file encontrado
                         filename    :    el ultimo file de una determinada carpeta
                         year        :    el anho
                         doy         :    el numero de dia del anho
                         set         :    el set del archivo
                     """
                     dirList = []
                     if not walk:
                         fullpath = path
                         foldercounter = 0
                     else:
                         #Filtra solo los directorios
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isDoyFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None, None, None, None, None
                         dirList = sorted( dirList, key=str.lower )
                         doypath = dirList[-1]
                         foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
                         fullpath = os.path.join(path, doypath, expLabel)
                     print "%s folder was found: " %(fullpath )
                     if set == None:
                         filename = getlastFileFromPath(fullpath, ext)
                     else:
                         filename = getFileFromSet(fullpath, ext, set)
                     if not(filename):
                         return None, None, None, None, None, None
                     print "%s file was found" %(filename)
                     if not(self.__verifyFile(os.path.join(fullpath, filename))):
                         return None, None, None, None, None, None
                     year = int( filename[1:5] )
                     doy  = int( filename[5:8] )
                     set  = int( filename[8:11] )
                     return fullpath, foldercounter, filename, year, doy, set
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print "No more Files"
                             return 0
                         filename = self.filenameList[idFile]
                         if not(self.__verifyFile(filename)):
                             continue
                         fileSize = os.path.getsize(filename)
                         fp = open(filename,'rb')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fp = fp
                     print "Setting the file: %s"%self.filename
                     return 1
                 def __setNextFileOnline(self):
                     """
                     Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
                     no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
                     siguientes.
                     Affected:
                         self.flagIsNewFile
                         self.filename
                         self.fileSize
                         self.fp
                         self.set
                         self.flagNoMoreFiles
                     Return:
 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
 : si el file fue abierto con exito y esta listo a ser leido
                     Excepciones:
                         Si un determinado file no puede ser abierto
                     """
                     nFiles = 0
                     fileOk_flag = False
                     firstTime_flag = True
                     self.set += 1
                     if self.set > 999:
                         self.set = 0
                         self.foldercounter += 1
                     #busca el 1er file disponible
                     fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                     if fullfilename:
                         if self.__verifyFile(fullfilename, False):
                             fileOk_flag = True
                     #si no encuentra un file entonces espera y vuelve a buscar
                     if not(fileOk_flag):
                         for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
                             if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
                                 tries = self.nTries
                             else:
                                 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
                             for nTries in range( tries ):
                                 if firstTime_flag:
                                     print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
                                     time.sleep( self.delay )
                                 else:
                                     print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
                                 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                                 if fullfilename:
                                     if self.__verifyFile(fullfilename):
                                         fileOk_flag = True
                                         break
                             if fileOk_flag:
                                 break
                             firstTime_flag = False
                             print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename
                             self.set += 1
                             if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
                                 self.set = 0
                                 self.doy += 1
                                 self.foldercounter = 0
                     if fileOk_flag:
                         self.fileSize = os.path.getsize( fullfilename )
                         self.filename = fullfilename
                         self.flagIsNewFile = 1
                         if self.fp != None: self.fp.close()
                         self.fp = open(fullfilename, 'rb')
                         self.flagNoMoreFiles = 0
                         print 'Setting the file: %s' % fullfilename
                     else:
                         self.fileSize = 0
                         self.filename = None
                         self.flagIsNewFile = 0
                         self.fp = None
                         self.flagNoMoreFiles = 1
                         print 'No more Files'
                     return fileOk_flag
                 def setNextFile(self):
                     if self.fp != None:
                         self.fp.close()
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         return 0
                     self.__readFirstHeader()
                     self.nReadBlocks = 0
                     return 1
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.__rdBasicHeader()
                             return 1
                         if self.fileSize == self.fileSizeByHeader:
             #                self.flagEoF = True
                             return 0
                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         time.sleep( self.delay )
                     return 0
                 def waitDataBlock(self,pointer_location):
                     currentPointer = pointer_location
                     neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             return 1
                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         time.sleep( self.delay )
                     return 0
                 def __jumpToLastBlock(self):
                     if not(self.__isFirstTimeOnline):
                         return
                     csize = self.fileSize - self.fp.tell()
                     blocksize = self.processingHeaderObj.blockSize
                     #salta el primer bloque de datos
                     if csize > self.processingHeaderObj.blockSize:
                         self.fp.seek(self.fp.tell() + blocksize)
                     else:
                         return
                     csize = self.fileSize - self.fp.tell()
                     neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     while True:
                         if self.fp.tell()<self.fileSize:
                             self.fp.seek(self.fp.tell() + neededsize)
                         else:
                             self.fp.seek(self.fp.tell() - neededsize)
                             break
             #        csize = self.fileSize - self.fp.tell()
             #        neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
             #        factor = int(csize/neededsize)
             #        if factor > 0:
             #            self.fp.seek(self.fp.tell() + factor*neededsize)
                     self.flagIsNewFile = 0
                     self.__isFirstTimeOnline = 0
                 def __setNewBlock(self):
                     if self.fp == None:
                         return 0
                     if self.online:
                         self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         return 1
                     self.lastUTTime = self.basicHeaderObj.utc
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.__rdBasicHeader()
                         return 1
                     if self.__waitNewBlock():
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
                     self.flagTimeBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagTimeBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def __rdProcessingHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.processingHeaderObj.read(fp)
                 def __rdRadarControllerHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.radarControllerHeaderObj.read(fp)
                 def __rdSystemHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.systemHeaderObj.read(fp)
                 def __rdBasicHeader(self, fp=None):
                     if fp == None:
                         fp = self.fp
                     self.basicHeaderObj.read(fp)
                 def __readFirstHeader(self):
                     self.__rdBasicHeader()
                     self.__rdSystemHeader()
                     self.__rdRadarControllerHeader()
                     self.__rdProcessingHeader()
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
                     else:
                         raise ValueError, 'Data type was not defined'
                     self.dtype = datatype_str
                     self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
             #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
             #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def __verifyFile(self, filename, msgFlag=True):
                     msg = None
                     try:
                         fp = open(filename, 'rb')
                         currentPosition = fp.tell()
                     except:
                         if msgFlag:
                             print "The file %s can't be opened" % (filename)
                         return False
                     neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                     if neededSize == 0:
                         basicHeaderObj = BasicHeader(LOCALTIME)
                         systemHeaderObj = SystemHeader()
                         radarControllerHeaderObj = RadarControllerHeader()
                         processingHeaderObj = ProcessingHeader()
                         try:
                             if not( basicHeaderObj.read(fp) ): raise IOError
                             if not( systemHeaderObj.read(fp) ): raise IOError
                             if not( radarControllerHeaderObj.read(fp) ): raise IOError
                             if not( processingHeaderObj.read(fp) ): raise IOError
                             data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                             neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
                         except:
                             if msgFlag:
                                 print "\tThe file %s is empty or it hasn't enough data" % filename
                             fp.close()
                             return False
                     else:
                         msg = "\tSkipping the file %s due to it hasn't enough data" %filename
                     fp.close()
                     fileSize = os.path.getsize(filename)
                     currentSize = fileSize - currentPosition
                     if currentSize < neededSize:
                         if msgFlag and (msg != None):
                             print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
                         return False
                     return True
                 def setup(self,
                             path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=None,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True):
                     if path == None:
                         raise ValueError, "The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if online:
                         print "Searching files in online mode..."
                         for nTries in range( self.nTries ):
                             fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
                             if fullpath:
                                 break
                             print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
                             time.sleep( self.delay )
                         if not(fullpath):
                             print "There 'isn't valied files in %s" % path
                             return None
                         self.year = year
                         self.doy  = doy
                         self.set  = set - 1
                         self.path = path
                         self.foldercounter = foldercounter
                     else:
                         print "Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
                             print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime())
                             sys.exit(-1)
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                     self.online = online
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "No files"
                         sys.exit(-1)
             #        self.updateDataHeader()
                     return self.dataOut
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds
                     self.dataOut.flagTimeBlock = self.flagTimeBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                 def getFirstHeader(self):
                     raise ValueError, "This method has not been implemented"
                 def getData():
                     raise ValueError, "This method has not been implemented"
                 def hasNotDataInBuffer():
                     raise ValueError, "This method has not been implemented"
                 def readBlock():
                     raise ValueError, "This method has not been implemented"
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print "Number of read blocks per file %04d" %self.nReadBlocks
                 def printTotalBlocks(self):
                     print "Number of read blocks %04d" %self.nTotalBlocks
                 def printNumberOfBlock(self):
                     if self.flagIsNewBlock:
                         print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
                 def printInfo(self):
                     if self.__printInfo == False:
                         return
                     self.basicHeaderObj.printInfo()
                     self.systemHeaderObj.printInfo()
                     self.radarControllerHeaderObj.printInfo()
                     self.processingHeaderObj.printInfo()
                     self.__printInfo = False
                 def run(self, **kwargs):
                     if not(self.isConfig):
             #            self.dataOut = dataOut
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             class JRODataWriter(JRODataIO, Operation):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 blockIndex = 0
                 path = None
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 def __init__(self, dataOut=None):
                     raise ValueError, "Not implemented"
                 def hasAllDataInBuffer(self):
                     raise ValueError, "Not implemented"
                 def setBlockDimension(self):
                     raise ValueError, "Not implemented"
                 def writeBlock(self):
                     raise ValueError, "No implemented"
                 def putData(self):
                     raise ValueError, "No implemented"
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize #bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond  = (self.dataOut.utctime - utc)* 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
                     raise ValueError, "No implemented"
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                     self.dtype = self.dataOut.dtype
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def setNextFile(self):
                     """
                     Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, file )
                     fp = open( filename,'wb' )
                     self.blockIndex = 0
                     #guardando atributos
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.setFirstHeader()
                     print 'Writing the file: %s'%self.filename
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path      :    el path destino en el cual se escribiran los files a crear
                         format    :    formato en el cual sera salvado un file
                         set       :    el setebo del file
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     ext = ext.lower()
                     self.ext = ext
                     self.path = path
                     self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     if not(self.setNextFile()):
                         print "There isn't a next file"
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class VoltageReader(JRODataReader):
                 """
                 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
                 perfiles*alturas*canales) son almacenados en la variable "buffer".
                                          perfiles * alturas * canales
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = VoltageReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         #to get one profile
                         profile =  readerObj.getData()
                         #print the profile
                         print profile
                         #If you want to see all datablock
                         print readerObj.datablock
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 ext = ".r"
                 optchar = "D"
                 dataOut = None
                 def __init__(self):
                     """
                     Inicializador de la clase VoltageReader para la lectura de datos de voltage.
                     Input:
                         dataOut    :    Objeto de la clase Voltage. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Variables afectadas:
                         self.dataOut
                     Return:
                         None
                     """
                     self.isConfig = False
                     self.datablock = None
                     self.utc = 0
                     self.ext = ".r"
                     self.optchar = "D"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.online = 0
                     self.fp = None
                     self.idFile = None
                     self.dtype = None
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.filename = None
                     self.fileSize = None
                     self.firstHeaderSize = 0
                     self.basicHeaderSize = 24
                     self.pathList = []
                     self.filenameList = []
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
                     self.flagNoMoreFiles = 0
                     self.set = 0
                     self.path = None
                     self.profileIndex = 2**32-1
                     self.delay  = 3   #seconds
                     self.nTries  = 3  #quantity tries
                     self.nFiles = 3   #number of files for searching
                     self.nReadBlocks = 0
                     self.flagIsNewFile = 1
                     self.__isFirstTimeOnline = 1
                     self.ippSeconds = 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     self.nTotalBlocks = 0
                     self.blocksize = 0
                     self.dataOut = self.createObjByDefault()
                 def createObjByDefault(self):
                     dataObj = Voltage()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
                         return 1
                     return 0
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.blocksize
                     Return:
                         None
                     """
                     pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
                     self.blocksize = pts2read
                 def readBlock(self):
                     """
                     readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Inputs:
                         None
                     Return:
                         None
                     Affected:
                         self.profileIndex
                         self.datablock
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
                     current_pointer_location = self.fp.tell()
                     junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
                     try:
                         junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
                     except:
                         #print "The read block (%3d) has not enough data" %self.nReadBlocks
                         if self.waitDataBlock(pointer_location=current_pointer_location):
                             junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
                             junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
             #             return 0
                     junk = numpy.transpose(junk, (2,0,1))
                     self.datablock = junk['real'] + junk['imag']*1j
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.dataOut.dtype = self.dtype
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
                     self.dataOut.ippSeconds = self.ippSeconds
                     self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.flagShiftFFT = False
                     if self.radarControllerHeaderObj.code != None:
                         self.dataOut.nCode = self.radarControllerHeaderObj.nCode
                         self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
                         self.dataOut.code = self.radarControllerHeaderObj.code
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip
                     self.dataOut.flagShiftFFT = False
                 def getData(self):
                     """
                     getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Ademas incrementa el contador del buffer en 1.
                     Return:
                         data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                      buffer. Si no hay mas archivos a leer retorna None.
                     Variables afectadas:
                         self.dataOut
                         self.profileIndex
                     Affected:
                         self.dataOut
                         self.profileIndex
                         self.flagTimeBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             return 0
                         self.getFirstHeader()
                     if self.datablock == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.dataOut.data = self.datablock[:,self.profileIndex,:]
                     self.dataOut.flagNoData = False
                     self.getBasicHeader()
                     self.profileIndex += 1
                     self.dataOut.realtime = self.online
                     return self.dataOut.data
             class VoltageWriter(JRODataWriter):
                 """
                 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".r"
                 optchar = "D"
                 shapeBuffer = None
                 def __init__(self):
                     """
                     Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                     Return: None
                     """
                     self.nTotalBlocks = 0
                     self.profileIndex = 0
                     self.isConfig = False
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.nTotalBlocks = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
                         return 1
                     return 0
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
                                         self.processingHeaderObj.nHeights,
                                         self.systemHeaderObj.nChannels)
                     self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
                                                  self.processingHeaderObj.profilesPerBlock,
                                                  self.processingHeaderObj.nHeights),
                                                  dtype=numpy.dtype('complex64'))
                 def writeBlock(self):
                     """
                     Escribe el buffer en el file designado
                     Affected:
                         self.profileIndex
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.blockIndex
                     Return: None
                     """
                     data = numpy.zeros( self.shapeBuffer, self.dtype )
                     junk = numpy.transpose(self.datablock, (1,2,0))
                     data['real'] = junk.real
                     data['imag'] = junk.imag
                     data = data.reshape( (-1) )
                     data.tofile( self.fp )
                     self.datablock.fill(0)
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.flagIsNewBlock
                         self.profileIndex
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagTimeBlock:
                         self.datablock.fill(0)
                         self.profileIndex = 0
                         self.setNextFile()
                     if self.profileIndex == 0:
                         self.setBasicHeader()
                     self.datablock[:,self.profileIndex,:] = self.dataOut.data
                     self.profileIndex += 1
                     if self.hasAllDataInBuffer():
                         #if self.flagIsNewFile:
                         self.writeNextBlock()
             #            self.setFirstHeader()
                     return 1
                 def __getProcessFlags(self):
                     processFlags = 0
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList =  [PROCFLAG.DATATYPE_CHAR,
                                        PROCFLAG.DATATYPE_SHORT,
                                        PROCFLAG.DATATYPE_LONG,
                                        PROCFLAG.DATATYPE_INT64,
                                        PROCFLAG.DATATYPE_FLOAT,
                                        PROCFLAG.DATATYPE_DOUBLE]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             dtypeValue = datatypeValueList[index]
                             break
                     processFlags += dtypeValue
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code != None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nCohInt > 1:
                         processFlags += PROCFLAG.COHERENT_INTEGRATION
                     return processFlags
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
                     '''
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList = [1,2,4,8,4,8]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             datatypeValue = datatypeValueList[index]
                             break
                     blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * datatypeValue * 2)
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.setBasicHeader()
                     processingHeaderSize = 40 # bytes
                     self.processingHeaderObj.dtype = 0 # Voltage
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.processFlags = self.__getProcessFlags()
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
                     self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
                     self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
             #         if self.dataOut.code != None:
             #             self.processingHeaderObj.code = self.dataOut.code
             #             self.processingHeaderObj.nCode = self.dataOut.nCode
             #             self.processingHeaderObj.nBaud = self.dataOut.nBaud
             #             codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)
             #             processingHeaderSize += codesize
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                         processingHeaderSize += 12
                     self.processingHeaderObj.size = processingHeaderSize
             class SpectraReader(JRODataReader):
                 """
                 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
                 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
                                     paresCanalesIguales    * alturas * perfiles  (Self Spectra)
                                     paresCanalesDiferentes * alturas * perfiles  (Cross Spectra)
                                     canales * alturas                            (DC Channels)
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = SpectraReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         readerObj.getData()
                         print readerObj.data_spc
                         print readerObj.data_cspc
                         print readerObj.data_dc
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 pts2read_SelfSpectra = 0
                 pts2read_CrossSpectra = 0
                 pts2read_DCchannels = 0
                 ext = ".pdata"
                 optchar = "P"
                 dataOut = None
                 nRdChannels = None
                 nRdPairs = None
                 rdPairList = []
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraReader para la lectura de datos de espectros.
                     Inputs:
                         dataOut    :    Objeto de la clase Spectra. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Affected:
                         self.dataOut
                     Return      : None
                     """
                     self.isConfig = False
                     self.pts2read_SelfSpectra = 0
                     self.pts2read_CrossSpectra = 0
                     self.pts2read_DCchannels = 0
                     self.datablock = None
                     self.utc = None
                     self.ext = ".pdata"
                     self.optchar = "P"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.online = 0
                     self.fp = None
                     self.idFile = None
                     self.dtype = None
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.filename = None
                     self.fileSize = None
                     self.firstHeaderSize = 0
                     self.basicHeaderSize = 24
                     self.pathList = []
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
                     self.flagNoMoreFiles = 0
                     self.set = 0
                     self.path = None
                     self.delay  = 60   #seconds
                     self.nTries = 3  #quantity tries
                     self.nFiles = 3   #number of files for searching
                     self.nReadBlocks = 0
                     self.flagIsNewFile = 1
                     self.__isFirstTimeOnline = 1
                     self.ippSeconds = 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     self.nTotalBlocks = 0
                     self.blocksize = 0
                     self.dataOut = self.createObjByDefault()
                     self.profileIndex = 1 #Always
                 def createObjByDefault(self):
                     dataObj = Spectra()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     return 1
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.nRdChannels
                         self.nRdPairs
                         self.pts2read_SelfSpectra
                         self.pts2read_CrossSpectra
                         self.pts2read_DCchannels
                         self.blocksize
                         self.dataOut.nChannels
                         self.dataOut.nPairs
                     Return:
                         None
                     """
                     self.nRdChannels = 0
                     self.nRdPairs = 0
                     self.rdPairList = []
                     for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
                         if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
                             self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
                         else:
                             self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
                             self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
                     pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
                     self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
                     self.blocksize = self.pts2read_SelfSpectra
                     if self.processingHeaderObj.flag_cspc:
                         self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
                         self.blocksize += self.pts2read_CrossSpectra
                     if self.processingHeaderObj.flag_dc:
                         self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
                         self.blocksize += self.pts2read_DCchannels
             #        self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
                 def readBlock(self):
                     """
                     Lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Return: None
                     Variables afectadas:
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
                     blockOk_flag = False
                     fpointer = self.fp.tell()
                     spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
                     spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
                         cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_dc:
                         dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
                         dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
                     if not(self.processingHeaderObj.shif_fft):
                         #desplaza a la derecha en el eje 2 determinadas posiciones
                         shift = int(self.processingHeaderObj.profilesPerBlock/2)
                         spc = numpy.roll( spc, shift , axis=2 )
                         if self.processingHeaderObj.flag_cspc:
                             #desplaza a la derecha en el eje 2 determinadas posiciones
                             cspc = numpy.roll( cspc, shift, axis=2 )
             #            self.processingHeaderObj.shif_fft = True
                     spc = numpy.transpose( spc, (0,2,1) )
                     self.data_spc = spc
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.transpose( cspc, (0,2,1) )
                         self.data_cspc = cspc['real'] + cspc['imag']*1j
                     else:
                         self.data_cspc = None
                     if self.processingHeaderObj.flag_dc:
                         self.data_dc = dc['real'] + dc['imag']*1j
                     else:
                         self.data_dc = None
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.dataOut.dtype = self.dtype
                     self.dataOut.nPairs = self.nRdPairs
                     self.dataOut.pairsList = self.rdPairList
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
                     self.dataOut.ippSeconds = self.ippSeconds
                     self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
                     self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip
                     if self.processingHeaderObj.code != None:
                         self.dataOut.nCode = self.processingHeaderObj.nCode
                         self.dataOut.nBaud = self.processingHeaderObj.nBaud
                         self.dataOut.code = self.processingHeaderObj.code
                         self.dataOut.flagDecodeData = True
                 def getData(self):
                     """
                     Copia el buffer de lectura a la clase "Spectra",
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Return:
 :    Si no hay mas archivos disponibles
 :    Si hizo una buena copia del buffer
                     Affected:
                         self.dataOut
                         self.flagTimeBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             self.dataOut.flagNoData = True
                             return 0
                     #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
                     if self.data_dc == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.getBasicHeader()
                     self.getFirstHeader()
                     self.dataOut.data_spc = self.data_spc
                     self.dataOut.data_cspc = self.data_cspc
                     self.dataOut.data_dc = self.data_dc
                     self.dataOut.flagNoData = False
                     self.dataOut.realtime = self.online
                     return self.dataOut.data_spc
             class SpectraWriter(JRODataWriter):
                 """
                 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".pdata"
                 optchar = "P"
                 shape_spc_Buffer = None
                 shape_cspc_Buffer = None
                 shape_dc_Buffer = None
                 data_spc = None
                 data_cspc = None
                 data_dc = None
             #    dataOut = None
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj
                     Return: None
                     """
                     self.isConfig = False
                     self.nTotalBlocks = 0
                     self.data_spc = None
                     self.data_cspc = None
                     self.data_dc = None
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.nTotalBlocks = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.noMoreFiles = 0
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     return 1
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shape_spc_Buffer = (self.dataOut.nChannels,
                                              self.processingHeaderObj.nHeights,
                                              self.processingHeaderObj.profilesPerBlock)
                     self.shape_cspc_Buffer = (self.dataOut.nPairs,
                                               self.processingHeaderObj.nHeights,
                                               self.processingHeaderObj.profilesPerBlock)
                     self.shape_dc_Buffer = (self.dataOut.nChannels,
                                             self.processingHeaderObj.nHeights)
                 def writeBlock(self):
                     """
                     Escribe el buffer en el file designado
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.nWriteBlocks
                     Return: None
                     """
                     spc = numpy.transpose( self.data_spc, (0,2,1) )
                     if not( self.processingHeaderObj.shif_fft ):
                         spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                     data = spc.reshape((-1))
                     data = data.astype(self.dtype[0])
                     data.tofile(self.fp)
                     if self.data_cspc != None:
                         data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
                         cspc = numpy.transpose( self.data_cspc, (0,2,1) )
                         if not( self.processingHeaderObj.shif_fft ):
                             cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                         data['real'] = cspc.real
                         data['imag'] = cspc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     if self.data_dc != None:
                         data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
                         dc = self.data_dc
                         data['real'] = dc.real
                         data['imag'] = dc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     self.data_spc.fill(0)
                     if self.data_dc != None:
                         self.data_dc.fill(0)
                     if self.data_cspc != None:
                         self.data_cspc.fill(0)
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nWriteBlocks += 1
                     self.blockIndex += 1
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagTimeBlock:
                         self.data_spc.fill(0)
                         self.data_cspc.fill(0)
                         self.data_dc.fill(0)
                         self.setNextFile()
                     if self.flagIsNewFile == 0:
                         self.setBasicHeader()
                     self.data_spc = self.dataOut.data_spc.copy()
                     if self.dataOut.data_cspc != None:
                         self.data_cspc = self.dataOut.data_cspc.copy()
                     self.data_dc = self.dataOut.data_dc.copy()
                     # #self.processingHeaderObj.dataBlocksPerFile)
                     if self.hasAllDataInBuffer():
             #            self.setFirstHeader()
                         self.writeNextBlock()
                     return 1
                 def __getProcessFlags(self):
                     processFlags = 0
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList =  [PROCFLAG.DATATYPE_CHAR,
                                        PROCFLAG.DATATYPE_SHORT,
                                        PROCFLAG.DATATYPE_LONG,
                                        PROCFLAG.DATATYPE_INT64,
                                        PROCFLAG.DATATYPE_FLOAT,
                                        PROCFLAG.DATATYPE_DOUBLE]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             dtypeValue = datatypeValueList[index]
                             break
                     processFlags += dtypeValue
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code != None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nIncohInt > 1:
                         processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                     if self.dataOut.data_dc != None:
                         processFlags += PROCFLAG.SAVE_CHANNELS_DC
                     return processFlags
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
                     '''
                     dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
                     dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
                     dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
                     dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
                     dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
                     dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
                     dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
                     datatypeValueList = [1,2,4,8,4,8]
                     for index in range(len(dtypeList)):
                         if self.dataOut.dtype == dtypeList[index]:
                             datatypeValue = datatypeValueList[index]
                             break
                     pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
                     pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
                     blocksize = (pts2write_SelfSpectra*datatypeValue)
                     if self.dataOut.data_cspc != None:
                         pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
                         blocksize += (pts2write_CrossSpectra*datatypeValue*2)
                     if self.dataOut.data_dc != None:
                         pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
                         blocksize += (pts2write_DCchannels*datatypeValue*2)
                     blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.setBasicHeader()
                     processingHeaderSize = 40 # bytes
                     self.processingHeaderObj.dtype = 1 # Spectra
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.processFlags = self.__getProcessFlags()
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
                     self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
                     self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
                     self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
                     if self.processingHeaderObj.totalSpectra > 0:
                         channelList = []
                         for channel in range(self.dataOut.nChannels):
                             channelList.append(channel)
                             channelList.append(channel)
                         pairsList = []
                         if self.dataOut.nPairs > 0:
                             for pair in self.dataOut.pairsList:
                                 pairsList.append(pair[0])
                                 pairsList.append(pair[1])
                         spectraComb = channelList + pairsList
                         spectraComb = numpy.array(spectraComb,dtype="u1")
                         self.processingHeaderObj.spectraComb = spectraComb
                         sizeOfSpcComb = len(spectraComb)
                         processingHeaderSize += sizeOfSpcComb
             #        The processing header should not have information about code
             #        if self.dataOut.code != None:
             #            self.processingHeaderObj.code = self.dataOut.code
             #            self.processingHeaderObj.nCode = self.dataOut.nCode
             #            self.processingHeaderObj.nBaud = self.dataOut.nBaud
             #            nCodeSize = 4 # bytes
             #            nBaudSize = 4 # bytes
             #            codeSize = 4 # bytes
             #            sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)
             #            processingHeaderSize += sizeOfCode
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                         sizeOfFirstHeight = 4
                         sizeOfdeltaHeight = 4
                         sizeOfnHeights = 4
                         sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows
                         processingHeaderSize += sizeOfWindows
                     self.processingHeaderObj.size = processingHeaderSize
             class SpectraHeisWriter(Operation):
             #    set = None
                 setFile = None
                 idblock = None
                 doypath = None
                 subfolder = None
                 def __init__(self):
                     self.wrObj = FITS()
             #        self.dataOut = dataOut
                     self.nTotalBlocks=0
             #        self.set = None
                     self.setFile = None
                     self.idblock = 0
                     self.wrpath = None
                     self.doypath = None
                     self.subfolder = None
                     self.isConfig = False
                 def isNumber(str):
                     """
                     Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                     Excepciones:
                     Si un determinado string no puede ser convertido a numero
                     Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                     Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                     """
                     try:
                         float( str )
                         return True
                     except:
                         return False
                 def setup(self, dataOut, wrpath):
                     if not(os.path.exists(wrpath)):
                         os.mkdir(wrpath)
                     self.wrpath = wrpath
             #        self.setFile = 0
                     self.dataOut = dataOut
                 def putData(self):
                     name= time.localtime( self.dataOut.utctime)
                     ext=".fits"
                     if self.doypath == None:
                        self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
                        self.doypath = os.path.join( self.wrpath, self.subfolder )
                        os.mkdir(self.doypath)
                     if self.setFile == None:
             #           self.set = self.dataOut.set
                        self.setFile = 0
             #        if self.set != self.dataOut.set:
             ##            self.set = self.dataOut.set
             #            self.setFile = 0
                     #make the filename
                     file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
                     filename = os.path.join(self.wrpath,self.subfolder, file)
                     idblock = numpy.array([self.idblock],dtype="int64")
                     header=self.wrObj.cFImage(idblock=idblock,
                                             year=time.gmtime(self.dataOut.utctime).tm_year,
                                             month=time.gmtime(self.dataOut.utctime).tm_mon,
                                             day=time.gmtime(self.dataOut.utctime).tm_mday,
                                             hour=time.gmtime(self.dataOut.utctime).tm_hour,
                                             minute=time.gmtime(self.dataOut.utctime).tm_min,
                                             second=time.gmtime(self.dataOut.utctime).tm_sec)
                     c=3E8
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
                     colList = []
                     colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
                     colList.append(colFreq)
                     nchannel=self.dataOut.nChannels
                     for i in range(nchannel):
                         col = self.wrObj.writeData(name="PCh"+str(i+1),
                                                      format=str(self.dataOut.nFFTPoints)+'E',
                                                       data=10*numpy.log10(self.dataOut.data_spc[i,:]))
                         colList.append(col)
                     data=self.wrObj.Ctable(colList=colList)
                     self.wrObj.CFile(header,data)
                     self.wrObj.wFile(filename)
                     #update the setFile
                     self.setFile += 1
                     self.idblock += 1
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class ParameterConf:
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.name = ''
                     self.value = ''
                 def readXml(self, parmElement):
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                 def getElementName(self):
                     return self.ELEMENTNAME
             class Metadata:
                 def __init__(self, filename):
                     self.parmConfObjList = []
                     self.readXml(filename)
                 def readXml(self, filename):
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     self.projectElement = ElementTree().parse(filename)
                     self.project = self.projectElement.tag
                     parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         self.parmConfObjList.append(parmConfObj)
             class FitsWriter(Operation):
                 def __init__(self):
                     self.isConfig = False
                     self.dataBlocksPerFile = None
                     self.blockIndex = 0
                     self.flagIsNewFile = 1
                     self.fitsObj = None
                     self.optchar = 'P'
                     self.ext = '.fits'
                     self.setFile = 0
                 def setFitsHeader(self, dataOut, metadatafile):
                     header_data = pyfits.PrimaryHDU()
                     metadata4fits = Metadata(metadatafile)
                     for parameter in metadata4fits.parmConfObjList:
                         parm_name = parameter.name
                         parm_value = parameter.value
             #             if parm_value == 'fromdatadatetime':
             #                 value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
             #             elif parm_value == 'fromdataheights':
             #                 value = dataOut.nHeights
             #             elif parm_value == 'fromdatachannel':
             #                 value = dataOut.nChannels
             #             elif parm_value == 'fromdatasamples':
             #                 value = dataOut.nFFTPoints
             #             else:
             #                 value = parm_value
                         header_data.header[parm_name] = parm_value
                     header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
                     header_data.header['CHANNELLIST'] = str(dataOut.channelList)
                     header_data.header['NCHANNELS'] = dataOut.nChannels
                     #header_data.header['HEIGHTS'] = dataOut.heightList
                     header_data.header['NHEIGHTS'] = dataOut.nHeights
                     header_data.header['IPPSECONDS'] = dataOut.ippSeconds
                     header_data.header['NCOHINT'] = dataOut.nCohInt
                     header_data.header['NINCOHINT'] = dataOut.nIncohInt
                     header_data.header['TIMEZONE'] = dataOut.timeZone
                     header_data.header['NBLOCK'] = self.blockIndex
                     header_data.writeto(self.filename)
                     self.addExtension(dataOut.heightList,'HEIGHTLIST')
                 def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):
                     self.path = path
                     self.dataOut = dataOut
                     self.metadatafile = metadatafile
                     self.dataBlocksPerFile = dataBlocksPerFile
                 def open(self):
                     self.fitsObj = pyfits.open(self.filename, mode='update')
                 def addExtension(self, data, tagname):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=tagname)
                     #extension.header['TAG'] = tagname
                     self.fitsObj.append(extension)
                     self.write()
                 def addData(self, data):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
                     extension.header['UTCTIME'] = self.dataOut.utctime
                     self.fitsObj.append(extension)
                     self.blockIndex += 1
                     self.fitsObj[0].header['NBLOCK'] = self.blockIndex
                     self.write()
                 def write(self):
                     self.fitsObj.flush(verbose=True)
                     self.fitsObj.close()
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, file )
                     self.blockIndex = 0
                     self.filename = filename
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     print 'Writing the file: %s'%self.filename
                     self.setFitsHeader(self.dataOut, self.metadatafile)
                     return 1
                 def writeBlock(self):
                     self.addData(self.dataOut.data_spc)
                     self.flagIsNewFile = 0
                 def __setNewBlock(self):
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.dataBlocksPerFile:
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def putData(self):
                     if self.flagIsNewFile:
                         self.setNextFile()
                     self.writeNextBlock()
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class FitsReader(ProcessingUnit):
             #     __TIMEZONE = time.timezone
                 expName = None
                 datetimestr = None
                 utc = None
                 nChannels = None
                 nSamples = None
                 dataBlocksPerFile = None
                 comments = None
                 lastUTTime = None
                 header_dict = None
                 data = None
                 data_header_dict = None
                 def __init__(self):
                     self.isConfig = False
                     self.ext = '.fits'
                     self.setFile = 0
                     self.flagNoMoreFiles = 0
                     self.flagIsNewFile = 1
                     self.flagTimeBlock = None
                     self.fileIndex = None
                     self.filename = None
                     self.fileSize = None
                     self.fitsObj = None
                     self.timeZone = None
                     self.nReadBlocks = 0
                     self.nTotalBlocks = 0
                     self.dataOut = self.createObjByDefault()
                     self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
                     self.blockIndex = 1
                 def createObjByDefault(self):
                     dataObj = Fits()
                     return dataObj
                 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
                     try:
                         fitsObj = pyfits.open(filename,'readonly')
                     except:
                         raise IOError, "The file %s can't be opened" %(filename)
                     header = fitsObj[0].header
                     struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
                     utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
                     ltc = utc
                     if useLocalTime:
                         ltc -= time.timezone
                     thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
                     thisTime = thisDatetime.time()
                     if not ((startTime <= thisTime) and (endTime > thisTime)):
                         return None
                     return thisDatetime
                 def __setNextFileOnline(self):
                     raise ValueError, "No implemented"
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print "No more Files"
                             return 0
                         filename = self.filenameList[idFile]
             #            if not(self.__verifyFile(filename)):
             #                continue
                         fileSize = os.path.getsize(filename)
                         fitsObj = pyfits.open(filename,'readonly')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fitsObj = fitsObj
                     self.blockIndex = 0
                     print "Setting the file: %s"%self.filename
                     return 1
                 def readHeader(self):
                     headerObj = self.fitsObj[0]
                     self.header_dict = headerObj.header
                     if 'EXPNAME' in headerObj.header.keys():
                         self.expName = headerObj.header['EXPNAME']
                     if 'DATATYPE' in headerObj.header.keys():
                         self.dataType = headerObj.header['DATATYPE']
                     self.datetimestr = headerObj.header['DATETIME']
                     channelList = headerObj.header['CHANNELLIST']
                     channelList = channelList.split('[')
                     channelList = channelList[1].split(']')
                     channelList = channelList[0].split(',')
                     channelList = [int(ch) for ch in channelList]
                     self.channelList = channelList
                     self.nChannels = headerObj.header['NCHANNELS']
                     self.nHeights = headerObj.header['NHEIGHTS']
                     self.ippSeconds = headerObj.header['IPPSECONDS']
                     self.nCohInt = headerObj.header['NCOHINT']
                     self.nIncohInt = headerObj.header['NINCOHINT']
                     self.dataBlocksPerFile = headerObj.header['NBLOCK']
                     self.timeZone = headerObj.header['TIMEZONE']
                     self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     if 'COMMENT' in headerObj.header.keys():
                         self.comments = headerObj.header['COMMENT']
                     self.readHeightList()
                 def readHeightList(self):
                     self.blockIndex = self.blockIndex + 1
                     obj = self.fitsObj[self.blockIndex]
                     self.heightList = obj.data
                     self.blockIndex = self.blockIndex + 1
                 def readExtension(self):
                     obj = self.fitsObj[self.blockIndex]
                     self.heightList = obj.data
                     self.blockIndex = self.blockIndex + 1
                 def setNextFile(self):
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         return 0
                     self.readHeader()
                     self.nReadBlocks = 0
             #         self.blockIndex = 1
                     return 1
                 def __searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.fits',
                                         walk=True):
                     pathList = []
                     if not walk:
                         pathList.append(path)
                     else:
                         dirList = []
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isDoyFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None
                         thisDate = startDate
                         while(thisDate <= endDate):
                             year = thisDate.timetuple().tm_year
                             doy = thisDate.timetuple().tm_yday
                             matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
                             if len(matchlist) == 0:
                                 thisDate += datetime.timedelta(1)
                                 continue
                             for match in matchlist:
                                 pathList.append(os.path.join(path,match,expLabel))
                             thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
                         return None, None
                     print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
                     filenameList = []
                     datetimeList = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "Any file was found for the time range %s - %s" %(startTime, endTime)
                         return None, None
                     print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def setup(self, path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=0,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True):
                     if path == None:
                         raise ValueError, "The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if not(online):
                         print "Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
                             print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime())
                             sys.exit(-1)
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                     self.online = online
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "No files"
                         sys.exit(-1)
                 def readBlock(self):
                     dataObj = self.fitsObj[self.blockIndex]
                     self.data = dataObj.data
                     self.data_header_dict = dataObj.header
                     self.utc = self.data_header_dict['UTCTIME']
                     self.flagIsNewFile = 0
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def __jumpToLastBlock(self):
                     raise ValueError, "No implemented"
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.__rdBasicHeader()
                             return 1
                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         time.sleep( self.delay )
                     return 0
                 def __setNewBlock(self):
                     if self.online:
                         self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         return 1
                     self.lastUTTime = self.utc
                     if self.online:
                         if self.__waitNewBlock():
                             return 1
                     if self.nReadBlocks < self.dataBlocksPerFile:
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.utc - self.lastUTTime
                     self.flagTimeBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagTimeBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagTimeBlock = 0
                     self.flagIsNewBlock = 0
                     if not(self.readNextBlock()):
                         return 0
                     if self.data == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.dataOut.data = self.data
                     self.dataOut.data_header = self.data_header_dict
                     self.dataOut.utctime = self.utc
                     self.dataOut.header = self.header_dict
                     self.dataOut.expName = self.expName
                     self.dataOut.nChannels = self.nChannels
                     self.dataOut.timeZone = self.timeZone
                     self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
                     self.dataOut.comments = self.comments
                     self.dataOut.timeInterval = self.timeInterval
                     self.dataOut.channelList = self.channelList
                     self.dataOut.heightList = self.heightList
                     self.dataOut.flagNoData = False
                     return self.dataOut.data
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()

1

'''

1

'''

2

3

$Author: murco $

3

$Author: murco $

4

$Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $

4

$Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $

5

'''

5

'''

6

7

import os, sys

7

import os, sys

8

import glob

8

import glob

9

import time

9

import time

10

import numpy

10

import numpy

11

import fnmatch

11

import fnmatch

12

import time, datetime

12

import time, datetime

13

from xml.etree.ElementTree import Element, SubElement, ElementTree

13

from xml.etree.ElementTree import Element, SubElement, ElementTree

14

try:

14

try:

15

import pyfits

15

import pyfits

16

except:

16

except:

17

print "pyfits module has not been imported, it should be installed to save files in fits format"

17

print "pyfits module has not been imported, it should be installed to save files in fits format"

18

19

from jrodata import *

19

from jrodata import *

20

from jroheaderIO import *

20

from jroheaderIO import *

21

from jroprocessing import *

21

from jroprocessing import *

22

23

LOCALTIME = True #-18000

23

LOCALTIME = True #-18000

24

25

def isNumber(str):

25

def isNumber(str):

26

"""

26

"""

27

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

27

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

28

29

Excepciones:

29

Excepciones:

30

Si un determinado string no puede ser convertido a numero

30

Si un determinado string no puede ser convertido a numero

31

Input:

31

Input:

32

str, string al cual se le analiza para determinar si convertible a un numero o no

32

str, string al cual se le analiza para determinar si convertible a un numero o no

33

34

Return:

34

Return:

35

True : si el string es uno numerico

35

True : si el string es uno numerico

36

False : no es un string numerico

36

False : no es un string numerico

37

"""

37

"""

38

try:

38

try:

39

float( str )

39

float( str )

40

return True

40

return True

41

except:

41

except:

42

return False

42

return False

43

44

def isThisFileinRange(filename, startUTSeconds, endUTSeconds):

44

def isThisFileinRange(filename, startUTSeconds, endUTSeconds):

45

"""

45

"""

46

Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.

46

Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.

47

48

Inputs:

48

Inputs:

49

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

49

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

50

51

startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en

51

startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en

52

segundos contados desde 01/01/1970.

52

segundos contados desde 01/01/1970.

53

endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en

53

endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en

54

segundos contados desde 01/01/1970.

54

segundos contados desde 01/01/1970.

55

56

Return:

56

Return:

57

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

57

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

58

fecha especificado, de lo contrario retorna False.

58

fecha especificado, de lo contrario retorna False.

59

60

Excepciones:

60

Excepciones:

61

Si el archivo no existe o no puede ser abierto

61

Si el archivo no existe o no puede ser abierto

62

Si la cabecera no puede ser leida.

62

Si la cabecera no puede ser leida.

63

64

"""

64

"""

65

basicHeaderObj = BasicHeader(LOCALTIME)

65

basicHeaderObj = BasicHeader(LOCALTIME)

66

67

try:

67

try:

68

fp = open(filename,'rb')

68

fp = open(filename,'rb')

69

except:

69

except:

70

raise IOError, "The file %s can't be opened" %(filename)

70

raise IOError, "The file %s can't be opened" %(filename)

71

72

sts = basicHeaderObj.read(fp)

72

sts = basicHeaderObj.read(fp)

73

fp.close()

73

fp.close()

74

75

if not(sts):

75

if not(sts):

76

print "Skipping the file %s because it has not a valid header" %(filename)

76

print "Skipping the file %s because it has not a valid header" %(filename)

77

return 0

77

return 0

78

79

if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):

79

if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):

80

return 0

80

return 0

81

82

return 1

82

return 1

83

84

def isFileinThisTime(filename, startTime, endTime):

84

def isFileinThisTime(filename, startTime, endTime):

85

"""

85

"""

86

Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.

86

Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.

87

88

Inputs:

88

Inputs:

89

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

89

filename : nombre completo del archivo de datos en formato Jicamarca (.r)

90

91

startTime : tiempo inicial del rango seleccionado en formato datetime.time

91

startTime : tiempo inicial del rango seleccionado en formato datetime.time

92

93

endTime : tiempo final del rango seleccionado en formato datetime.time

93

endTime : tiempo final del rango seleccionado en formato datetime.time

94

95

Return:

95

Return:

96

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

96

Boolean : Retorna True si el archivo de datos contiene datos en el rango de

97

fecha especificado, de lo contrario retorna False.

97

fecha especificado, de lo contrario retorna False.

98

99

Excepciones:

99

Excepciones:

100

Si el archivo no existe o no puede ser abierto

100

Si el archivo no existe o no puede ser abierto

101

Si la cabecera no puede ser leida.

101

Si la cabecera no puede ser leida.

102

103

"""

103

"""

104

105

106

try:

106

try:

107

fp = open(filename,'rb')

107

fp = open(filename,'rb')

108

except:

108

except:

109

raise IOError, "The file %s can't be opened" %(filename)

109

raise IOError, "The file %s can't be opened" %(filename)

110

111

basicHeaderObj = BasicHeader(LOCALTIME)

111

basicHeaderObj = BasicHeader(LOCALTIME)

112

sts = basicHeaderObj.read(fp)

112

sts = basicHeaderObj.read(fp)

113

fp.close()

113

fp.close()

114

115

thisDatetime = basicHeaderObj.datatime

115

thisDatetime = basicHeaderObj.datatime

116

thisTime = basicHeaderObj.datatime.time()

116

thisTime = basicHeaderObj.datatime.time()

117

118

if not(sts):

118

if not(sts):

119

print "Skipping the file %s because it has not a valid header" %(filename)

119

print "Skipping the file %s because it has not a valid header" %(filename)

120

return None

120

return None

121

122

if not ((startTime <= thisTime) and (endTime > thisTime)):

122

if not ((startTime <= thisTime) and (endTime > thisTime)):

123

return None

123

return None

124

125

return thisDatetime

125

return thisDatetime

126

127

def getFileFromSet(path,ext,set):

127

def getFileFromSet(path,ext,set):

128

validFilelist = []

128

validFilelist = []

129

fileList = os.listdir(path)

129

fileList = os.listdir(path)

130

131

# 0 1234 567 89A BCDE

131

# 0 1234 567 89A BCDE

132

# H YYYY DDD SSS .ext

132

# H YYYY DDD SSS .ext

133

134

for file in fileList:

134

for file in fileList:

135

try:

135

try:

136

year = int(file[1:5])

136

year = int(file[1:5])

137

doy = int(file[5:8])

137

doy = int(file[5:8])

138

139

140

except:

140

except:

141

continue

141

continue

142

143

if (os.path.splitext(file)[-1].lower() != ext.lower()):

143

if (os.path.splitext(file)[-1].lower() != ext.lower()):

144

continue

144

continue

145

146

validFilelist.append(file)

146

validFilelist.append(file)

147

148

myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))

148

myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))

149

150

if len(myfile)!= 0:

150

if len(myfile)!= 0:

151

return myfile[0]

151

return myfile[0]

152

else:

152

else:

153

filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())

153

filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())

154

print 'the filename %s does not exist'%filename

154

print 'the filename %s does not exist'%filename

155

print '...going to the last file: '

155

print '...going to the last file: '

156

157

if validFilelist:

157

if validFilelist:

158

validFilelist = sorted( validFilelist, key=str.lower )

158

validFilelist = sorted( validFilelist, key=str.lower )

159

return validFilelist[-1]

159

return validFilelist[-1]

160

161

return None

161

return None

162

163

164

def getlastFileFromPath(path, ext):

164

def getlastFileFromPath(path, ext):

165

"""

165

"""

166

Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"

166

Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"

167

al final de la depuracion devuelve el ultimo file de la lista que quedo.

167

al final de la depuracion devuelve el ultimo file de la lista que quedo.

168

169

Input:

169

Input:

170

fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta

170

fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta

171

ext : extension de los files contenidos en una carpeta

171

ext : extension de los files contenidos en una carpeta

172

173

Return:

173

Return:

174

El ultimo file de una determinada carpeta, no se considera el path.

174

El ultimo file de una determinada carpeta, no se considera el path.

175

"""

175

"""

176

validFilelist = []

176

validFilelist = []

177

fileList = os.listdir(path)

177

fileList = os.listdir(path)

178

179

# 0 1234 567 89A BCDE

179

# 0 1234 567 89A BCDE

180

# H YYYY DDD SSS .ext

180

# H YYYY DDD SSS .ext

181

182

for file in fileList:

182

for file in fileList:

183

try:

183

try:

184

year = int(file[1:5])

184

year = int(file[1:5])

185

doy = int(file[5:8])

185

doy = int(file[5:8])

186

187

188

except:

188

except:

189

continue

189

continue

190

191

if (os.path.splitext(file)[-1].lower() != ext.lower()):

191

if (os.path.splitext(file)[-1].lower() != ext.lower()):

192

continue

192

continue

193

194

validFilelist.append(file)

194

validFilelist.append(file)

195

196

if validFilelist:

196

if validFilelist:

197

validFilelist = sorted( validFilelist, key=str.lower )

197

validFilelist = sorted( validFilelist, key=str.lower )

198

return validFilelist[-1]

198

return validFilelist[-1]

199

200

return None

200

return None

201

202

def checkForRealPath(path, foldercounter, year, doy, set, ext):

202

def checkForRealPath(path, foldercounter, year, doy, set, ext):

203

"""

203

"""

204

Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,

204

Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,

205

Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar

205

Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar

206

el path exacto de un determinado file.

206

el path exacto de un determinado file.

207

208

Example :

208

Example :

209

nombre correcto del file es .../.../D2009307/P2009307367.ext

209

nombre correcto del file es .../.../D2009307/P2009307367.ext

210

211

Entonces la funcion prueba con las siguientes combinaciones

211

Entonces la funcion prueba con las siguientes combinaciones

212

.../.../y2009307367.ext

212

.../.../y2009307367.ext

213

.../.../Y2009307367.ext

213

.../.../Y2009307367.ext

214

.../.../x2009307/y2009307367.ext

214

.../.../x2009307/y2009307367.ext

215

.../.../x2009307/Y2009307367.ext

215

.../.../x2009307/Y2009307367.ext

216

.../.../X2009307/y2009307367.ext

216

.../.../X2009307/y2009307367.ext

217

.../.../X2009307/Y2009307367.ext

217

.../.../X2009307/Y2009307367.ext

218

siendo para este caso, la ultima combinacion de letras, identica al file buscado

218

siendo para este caso, la ultima combinacion de letras, identica al file buscado

219

220

Return:

220

Return:

221

Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file

221

Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file

222

caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas

222

caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas

223

para el filename

223

para el filename

224

"""

224

"""

225

fullfilename = None

225

fullfilename = None

226

find_flag = False

226

find_flag = False

227

filename = None

227

filename = None

228

229

prefixDirList = [None,'d','D']

229

prefixDirList = [None,'d','D']

230

if ext.lower() == ".r": #voltage

230

if ext.lower() == ".r": #voltage

231

prefixFileList = ['d','D']

231

prefixFileList = ['d','D']

232

elif ext.lower() == ".pdata": #spectra

232

elif ext.lower() == ".pdata": #spectra

233

prefixFileList = ['p','P']

233

prefixFileList = ['p','P']

234

else:

234

else:

235

return None, filename

235

return None, filename

236

237

#barrido por las combinaciones posibles

237

#barrido por las combinaciones posibles

238

for prefixDir in prefixDirList:

238

for prefixDir in prefixDirList:

239

thispath = path

239

thispath = path

240

if prefixDir != None:

240

if prefixDir != None:

241

#formo el nombre del directorio xYYYYDDD (x=d o x=D)

241

#formo el nombre del directorio xYYYYDDD (x=d o x=D)

242

if foldercounter == 0:

242

if foldercounter == 0:

243

thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))

243

thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))

244

else:

244

else:

245

thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))

245

thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))

246

for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"

246

for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"

247

filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext

247

filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext

248

fullfilename = os.path.join( thispath, filename ) #formo el path completo

248

fullfilename = os.path.join( thispath, filename ) #formo el path completo

249

250

if os.path.exists( fullfilename ): #verifico que exista

250

if os.path.exists( fullfilename ): #verifico que exista

251

find_flag = True

251

find_flag = True

252

break

252

break

253

if find_flag:

253

if find_flag:

254

break

254

break

255

256

if not(find_flag):

256

if not(find_flag):

257

return None, filename

257

return None, filename

258

259

return fullfilename, filename

259

return fullfilename, filename

260

261

def isDoyFolder(folder):

261

def isDoyFolder(folder):

262

try:

262

try:

263

year = int(folder[1:5])

263

year = int(folder[1:5])

264

except:

264

except:

265

return 0

265

return 0

266

267

try:

267

try:

268

doy = int(folder[5:8])

268

doy = int(folder[5:8])

269

except:

269

except:

270

return 0

270

return 0

271

272

return 1

272

return 1

273

274

class JRODataIO:

274

class JRODataIO:

275

276

c = 3E8

276

c = 3E8

277

278

isConfig = False

278

isConfig = False

279

280

basicHeaderObj = BasicHeader(LOCALTIME)

280

basicHeaderObj = BasicHeader(LOCALTIME)

281

282

systemHeaderObj = SystemHeader()

282

systemHeaderObj = SystemHeader()

283

284

radarControllerHeaderObj = RadarControllerHeader()

284

radarControllerHeaderObj = RadarControllerHeader()

285

286

processingHeaderObj = ProcessingHeader()

286

processingHeaderObj = ProcessingHeader()

287

288

online = 0

288

online = 0

289

290

dtype = None

290

dtype = None

291

292

pathList = []

292

pathList = []

293

294

filenameList = []

294

filenameList = []

295

296

filename = None

296

filename = None

297

298

ext = None

298

ext = None

299

300

flagIsNewFile = 1

300

flagIsNewFile = 1

301

302

flagTimeBlock = 0

302

flagTimeBlock = 0

303

304

flagIsNewBlock = 0

304

flagIsNewBlock = 0

305

306

fp = None

306

fp = None

307

308

firstHeaderSize = 0

308

firstHeaderSize = 0

309

310

basicHeaderSize = 24

310

basicHeaderSize = 24

311

312

versionFile = 1103

312

versionFile = 1103

313

314

fileSize = None

314

fileSize = None

315

316

ippSeconds = None

316

ippSeconds = None

317

318

fileSizeByHeader = None

318

fileSizeByHeader = None

319

320

fileIndex = None

320

fileIndex = None

321

322

profileIndex = None

322

profileIndex = None

323

324

blockIndex = None

324

blockIndex = None

325

326

nTotalBlocks = None

326

nTotalBlocks = None

327

328

maxTimeStep = 30

328

maxTimeStep = 30

329

330

lastUTTime = None

330

lastUTTime = None

331

332

datablock = None

332

datablock = None

333

334

dataOut = None

334

dataOut = None

335

336

blocksize = None

336

blocksize = None

337

338

def __init__(self):

338

def __init__(self):

339

340

raise ValueError, "Not implemented"

340

raise ValueError, "Not implemented"

341

342

def run(self):

342

def run(self):

343

344

raise ValueError, "Not implemented"

344

raise ValueError, "Not implemented"

345

346

def getOutput(self):

346

def getOutput(self):

347

348

return self.dataOut

348

return self.dataOut

349

350

class JRODataReader(JRODataIO, ProcessingUnit):

350

class JRODataReader(JRODataIO, ProcessingUnit):

351

352

nReadBlocks = 0

352

nReadBlocks = 0

353

354

delay = 10 #number of seconds waiting a new file

354

delay = 10 #number of seconds waiting a new file

355

356

nTries = 3 #quantity tries

356

nTries = 3 #quantity tries

357

358

nFiles = 3 #number of files for searching

358

nFiles = 3 #number of files for searching

359

360

path = None

360

path = None

361

362

foldercounter = 0

362

foldercounter = 0

363

364

flagNoMoreFiles = 0

364

flagNoMoreFiles = 0

365

366

datetimeList = []

366

datetimeList = []

367

368

__isFirstTimeOnline = 1

368

__isFirstTimeOnline = 1

369

370

__printInfo = True

370

__printInfo = True

371

372

profileIndex = None

372

profileIndex = None

373

374

def __init__(self):

374

def __init__(self):

375

376

"""

376

"""

377

378

"""

378

"""

379

380

raise ValueError, "This method has not been implemented"

380

raise ValueError, "This method has not been implemented"

381

382

383

def createObjByDefault(self):

383

def createObjByDefault(self):

384

"""

384

"""

385

386

"""

386

"""

387

raise ValueError, "This method has not been implemented"

387

raise ValueError, "This method has not been implemented"

388

389

def getBlockDimension(self):

389

def getBlockDimension(self):

390

391

raise ValueError, "No implemented"

391

raise ValueError, "No implemented"

392

393

def __searchFilesOffLine(self,

393

def __searchFilesOffLine(self,

394

path,

394

path,

395

startDate,

395

startDate,

396

endDate,

396

endDate,

397

startTime=datetime.time(0,0,0),

397

startTime=datetime.time(0,0,0),

398

endTime=datetime.time(23,59,59),

398

endTime=datetime.time(23,59,59),

399

set=None,

399

set=None,

400

expLabel='',

400

expLabel='',

401

ext='.r',

401

ext='.r',

402

walk=True):

402

walk=True):

403

404

pathList = []

404

pathList = []

405

406

if not walk:

406

if not walk:

407

pathList.append(path)

407

#pathList.append(path)

408

multi_path = path.split(',')

409

for single_path in multi_path:

410

pathList.append(single_path)

408

411

409

else:

412

else:

410

dirList = []

413

#dirList = []

411

for thisPath in os.listdir(path):

414

multi_path = path.split(',')

412

if not os.path.isdir(os.path.join(path,thisPath)):

415

for single_path in multi_path:

413

~~continue~~

416

dirList = []

414

if not isDoyFolder(thisPath):

417

for thisPath in os.listdir(single_path):

415

continue

418

if not os.path.isdir(os.path.join(single_path,thisPath)):

419

continue

420

if not isDoyFolder(thisPath):

421

continue

416

422

417

dirList.append(thisPath)

423

dirList.append(thisPath)

418

424

419

if not(dirList):

425

if not(dirList):

420

return None, None

426

return None, None

421

427

422

thisDate = startDate

428

thisDate = startDate

423

429

424

while(thisDate <= endDate):

430

while(thisDate <= endDate):

425

year = thisDate.timetuple().tm_year

431

year = thisDate.timetuple().tm_year

426

doy = thisDate.timetuple().tm_yday

432

doy = thisDate.timetuple().tm_yday

427

433

428

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

434

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

429

if len(matchlist) == 0:

435

if len(matchlist) == 0:

436

thisDate += datetime.timedelta(1)

437

continue

438

for match in matchlist:

439

pathList.append(os.path.join(single_path,match,expLabel))

440

430

thisDate += datetime.timedelta(1)

441

thisDate += datetime.timedelta(1)

431

continue

432

for match in matchlist:

433

pathList.append(os.path.join(path,match,expLabel))

434

435

thisDate += datetime.timedelta(1)

436

442

437

if pathList == []:

443

if pathList == []:

438

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

444

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

439

return None, None

445

return None, None

440

446

441

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

447

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

442

448

443

filenameList = []

449

filenameList = []

444

datetimeList = []

450

datetimeList = []

451

pathDict = {}

452

filenameList_to_sort = []

445

453

446

for i in range(len(pathList)):

454

for i in range(len(pathList)):

447

455

448

thisPath = pathList[i]

456

thisPath = pathList[i]

449

457

450

fileList = glob.glob1(thisPath, "*%s" %ext)

458

fileList = glob.glob1(thisPath, "*%s" %ext)

451

fileList.sort()

459

fileList.sort()

460

pathDict.setdefault(fileList[0])

461

pathDict[fileList[0]] = i

462

filenameList_to_sort.append(fileList[0])

463

464

filenameList_to_sort.sort()

465

466

for file in filenameList_to_sort:

467

thisPath = pathList[pathDict[file]]

452

468

469

fileList = glob.glob1(thisPath, "*%s" %ext)

470

fileList.sort()

471

453

for file in fileList:

472

for file in fileList:

454

473

455

filename = os.path.join(thisPath,file)

474

filename = os.path.join(thisPath,file)

456

thisDatetime = isFileinThisTime(filename, startTime, endTime)

475

thisDatetime = isFileinThisTime(filename, startTime, endTime)

457

476

458

if not(thisDatetime):

477

if not(thisDatetime):

459

continue

478

continue

460

479

461

filenameList.append(filename)

480

filenameList.append(filename)

462

datetimeList.append(thisDatetime)

481

datetimeList.append(thisDatetime)

463

482

464

if not(filenameList):

483

if not(filenameList):

465

print "Any file was found for the time range %s - %s" %(startTime, endTime)

484

print "Any file was found for the time range %s - %s" %(startTime, endTime)

466

return None, None

485

return None, None

467

486

468

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

487

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

469

print

488

print

470

489

471

for i in range(len(filenameList)):

490

for i in range(len(filenameList)):

472

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

491

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

473

492

474

self.filenameList = filenameList

493

self.filenameList = filenameList

475

self.datetimeList = datetimeList

494

self.datetimeList = datetimeList

476

495

477

return pathList, filenameList

496

return pathList, filenameList

478

497

479

def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):

498

def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):

480

499

481

"""

500

"""

482

Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y

501

Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y

483

devuelve el archivo encontrado ademas de otros datos.

502

devuelve el archivo encontrado ademas de otros datos.

484

503

485

Input:

504

Input:

486

path : carpeta donde estan contenidos los files que contiene data

505

path : carpeta donde estan contenidos los files que contiene data

487

506

488

expLabel : Nombre del subexperimento (subfolder)

507

expLabel : Nombre del subexperimento (subfolder)

489

508

490

ext : extension de los files

509

ext : extension de los files

491

510

492

walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)

511

walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)

493

512

494

Return:

513

Return:

495

directory : eL directorio donde esta el file encontrado

514

directory : eL directorio donde esta el file encontrado

496

filename : el ultimo file de una determinada carpeta

515

filename : el ultimo file de una determinada carpeta

497

year : el anho

516

year : el anho

498

doy : el numero de dia del anho

517

doy : el numero de dia del anho

499

set : el set del archivo

518

set : el set del archivo

500

519

501

520

502

"""

521

"""

503

dirList = []

522

dirList = []

504

523

505

if not walk:

524

if not walk:

506

fullpath = path

525

fullpath = path

507

foldercounter = 0

526

foldercounter = 0

508

else:

527

else:

509

#Filtra solo los directorios

528

#Filtra solo los directorios

510

for thisPath in os.listdir(path):

529

for thisPath in os.listdir(path):

511

if not os.path.isdir(os.path.join(path,thisPath)):

530

if not os.path.isdir(os.path.join(path,thisPath)):

512

continue

531

continue

513

if not isDoyFolder(thisPath):

532

if not isDoyFolder(thisPath):

514

continue

533

continue

515

534

516

dirList.append(thisPath)

535

dirList.append(thisPath)

517

536

518

if not(dirList):

537

if not(dirList):

519

return None, None, None, None, None, None

538

return None, None, None, None, None, None

520

539

521

dirList = sorted( dirList, key=str.lower )

540

dirList = sorted( dirList, key=str.lower )

522

541

523

doypath = dirList[-1]

542

doypath = dirList[-1]

524

foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0

543

foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0

525

fullpath = os.path.join(path, doypath, expLabel)

544

fullpath = os.path.join(path, doypath, expLabel)

526

545

527

546

528

print "%s folder was found: " %(fullpath )

547

print "%s folder was found: " %(fullpath )

529

548

530

if set == None:

549

if set == None:

531

filename = getlastFileFromPath(fullpath, ext)

550

filename = getlastFileFromPath(fullpath, ext)

532

else:

551

else:

533

filename = getFileFromSet(fullpath, ext, set)

552

filename = getFileFromSet(fullpath, ext, set)

534

553

535

if not(filename):

554

if not(filename):

536

return None, None, None, None, None, None

555

return None, None, None, None, None, None

537

556

538

print "%s file was found" %(filename)

557

print "%s file was found" %(filename)

539

558

540

if not(self.__verifyFile(os.path.join(fullpath, filename))):

559

if not(self.__verifyFile(os.path.join(fullpath, filename))):

541

return None, None, None, None, None, None

560

return None, None, None, None, None, None

542

561

543

year = int( filename[1:5] )

562

year = int( filename[1:5] )

544

doy = int( filename[5:8] )

563

doy = int( filename[5:8] )

545

set = int( filename[8:11] )

564

set = int( filename[8:11] )

546

565

547

return fullpath, foldercounter, filename, year, doy, set

566

return fullpath, foldercounter, filename, year, doy, set

548

567

549

def __setNextFileOffline(self):

568

def __setNextFileOffline(self):

550

569

551

idFile = self.fileIndex

570

idFile = self.fileIndex

552

571

553

while (True):

572

while (True):

554

idFile += 1

573

idFile += 1

555

if not(idFile < len(self.filenameList)):

574

if not(idFile < len(self.filenameList)):

556

self.flagNoMoreFiles = 1

575

self.flagNoMoreFiles = 1

557

print "No more Files"

576

print "No more Files"

558

return 0

577

return 0

559

578

560

filename = self.filenameList[idFile]

579

filename = self.filenameList[idFile]

561

580

562

if not(self.__verifyFile(filename)):

581

if not(self.__verifyFile(filename)):

563

continue

582

continue

564

583

565

fileSize = os.path.getsize(filename)

584

fileSize = os.path.getsize(filename)

566

fp = open(filename,'rb')

585

fp = open(filename,'rb')

567

break

586

break

568

587

569

self.flagIsNewFile = 1

588

self.flagIsNewFile = 1

570

self.fileIndex = idFile

589

self.fileIndex = idFile

571

self.filename = filename

590

self.filename = filename

572

self.fileSize = fileSize

591

self.fileSize = fileSize

573

self.fp = fp

592

self.fp = fp

574

593

575

print "Setting the file: %s"%self.filename

594

print "Setting the file: %s"%self.filename

576

595

577

return 1

596

return 1

578

597

579

def __setNextFileOnline(self):

598

def __setNextFileOnline(self):

580

"""

599

"""

581

Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si

600

Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si

582

no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files

601

no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files

583

siguientes.

602

siguientes.

584

603

585

Affected:

604

Affected:

586

self.flagIsNewFile

605

self.flagIsNewFile

587

self.filename

606

self.filename

588

self.fileSize

607

self.fileSize

589

self.fp

608

self.fp

590

self.set

609

self.set

591

self.flagNoMoreFiles

610

self.flagNoMoreFiles

592

611

593

Return:

612

Return:

594

0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado

613

0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado

595

1 : si el file fue abierto con exito y esta listo a ser leido

614

1 : si el file fue abierto con exito y esta listo a ser leido

596

615

597

Excepciones:

616

Excepciones:

598

Si un determinado file no puede ser abierto

617

Si un determinado file no puede ser abierto

599

"""

618

"""

600

nFiles = 0

619

nFiles = 0

601

fileOk_flag = False

620

fileOk_flag = False

602

firstTime_flag = True

621

firstTime_flag = True

603

622

604

self.set += 1

623

self.set += 1

605

624

606

if self.set > 999:

625

if self.set > 999:

607

self.set = 0

626

self.set = 0

608

self.foldercounter += 1

627

self.foldercounter += 1

609

628

610

#busca el 1er file disponible

629

#busca el 1er file disponible

611

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

630

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

612

if fullfilename:

631

if fullfilename:

613

if self.__verifyFile(fullfilename, False):

632

if self.__verifyFile(fullfilename, False):

614

fileOk_flag = True

633

fileOk_flag = True

615

634

616

#si no encuentra un file entonces espera y vuelve a buscar

635

#si no encuentra un file entonces espera y vuelve a buscar

617

if not(fileOk_flag):

636

if not(fileOk_flag):

618

for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles

637

for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles

619

638

620

if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces

639

if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces

621

tries = self.nTries

640

tries = self.nTries

622

else:

641

else:

623

tries = 1 #si no es la 1era vez entonces solo lo hace una vez

642

tries = 1 #si no es la 1era vez entonces solo lo hace una vez

624

643

625

for nTries in range( tries ):

644

for nTries in range( tries ):

626

if firstTime_flag:

645

if firstTime_flag:

627

print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )

646

print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )

628

time.sleep( self.delay )

647

time.sleep( self.delay )

629

else:

648

else:

630

print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)

649

print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)

631

650

632

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

651

fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )

633

if fullfilename:

652

if fullfilename:

634

if self.__verifyFile(fullfilename):

653

if self.__verifyFile(fullfilename):

635

fileOk_flag = True

654

fileOk_flag = True

636

break

655

break

637

656

638

if fileOk_flag:

657

if fileOk_flag:

639

break

658

break

640

659

641

firstTime_flag = False

660

firstTime_flag = False

642

661

643

print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename

662

print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename

644

self.set += 1

663

self.set += 1

645

664

646

if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta

665

if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta

647

self.set = 0

666

self.set = 0

648

self.doy += 1

667

self.doy += 1

649

self.foldercounter = 0

668

self.foldercounter = 0

650

669

651

if fileOk_flag:

670

if fileOk_flag:

652

self.fileSize = os.path.getsize( fullfilename )

671

self.fileSize = os.path.getsize( fullfilename )

653

self.filename = fullfilename

672

self.filename = fullfilename

654

self.flagIsNewFile = 1

673

self.flagIsNewFile = 1

655

if self.fp != None: self.fp.close()

674

if self.fp != None: self.fp.close()

656

self.fp = open(fullfilename, 'rb')

675

self.fp = open(fullfilename, 'rb')

657

self.flagNoMoreFiles = 0

676

self.flagNoMoreFiles = 0

658

print 'Setting the file: %s' % fullfilename

677

print 'Setting the file: %s' % fullfilename

659

else:

678

else:

660

self.fileSize = 0

679

self.fileSize = 0

661

self.filename = None

680

self.filename = None

662

self.flagIsNewFile = 0

681

self.flagIsNewFile = 0

663

self.fp = None

682

self.fp = None

664

self.flagNoMoreFiles = 1

683

self.flagNoMoreFiles = 1

665

print 'No more Files'

684

print 'No more Files'

666

685

667

return fileOk_flag

686

return fileOk_flag

668

687

669

688

670

def setNextFile(self):

689

def setNextFile(self):

671

if self.fp != None:

690

if self.fp != None:

672

self.fp.close()

691

self.fp.close()

673

692

674

if self.online:

693

if self.online:

675

newFile = self.__setNextFileOnline()

694

newFile = self.__setNextFileOnline()

676

else:

695

else:

677

newFile = self.__setNextFileOffline()

696

newFile = self.__setNextFileOffline()

678

697

679

if not(newFile):

698

if not(newFile):

680

return 0

699

return 0

681

700

682

self.__readFirstHeader()

701

self.__readFirstHeader()

683

self.nReadBlocks = 0

702

self.nReadBlocks = 0

684

return 1

703

return 1

685

704

686

def __waitNewBlock(self):

705

def __waitNewBlock(self):

687

"""

706

"""

688

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

707

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

689

708

690

Si el modo de lectura es OffLine siempre retorn 0

709

Si el modo de lectura es OffLine siempre retorn 0

691

"""

710

"""

692

if not self.online:

711

if not self.online:

693

return 0

712

return 0

694

713

695

if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):

714

if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):

696

return 0

715

return 0

697

716

698

currentPointer = self.fp.tell()

717

currentPointer = self.fp.tell()

699

718

700

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

719

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

701

720

702

for nTries in range( self.nTries ):

721

for nTries in range( self.nTries ):

703

722

704

self.fp.close()

723

self.fp.close()

705

self.fp = open( self.filename, 'rb' )

724

self.fp = open( self.filename, 'rb' )

706

self.fp.seek( currentPointer )

725

self.fp.seek( currentPointer )

707

726

708

self.fileSize = os.path.getsize( self.filename )

727

self.fileSize = os.path.getsize( self.filename )

709

currentSize = self.fileSize - currentPointer

728

currentSize = self.fileSize - currentPointer

710

729

711

if ( currentSize >= neededSize ):

730

if ( currentSize >= neededSize ):

712

self.__rdBasicHeader()

731

self.__rdBasicHeader()

713

return 1

732

return 1

714

733

715

if self.fileSize == self.fileSizeByHeader:

734

if self.fileSize == self.fileSizeByHeader:

716

# self.flagEoF = True

735

# self.flagEoF = True

717

return 0

736

return 0

718

737

719

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

738

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

720

time.sleep( self.delay )

739

time.sleep( self.delay )

721

740

722

741

723

return 0

742

return 0

724

743

725

def waitDataBlock(self,pointer_location):

744

def waitDataBlock(self,pointer_location):

726

745

727

currentPointer = pointer_location

746

currentPointer = pointer_location

728

747

729

neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize

748

neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize

730

749

731

for nTries in range( self.nTries ):

750

for nTries in range( self.nTries ):

732

self.fp.close()

751

self.fp.close()

733

self.fp = open( self.filename, 'rb' )

752

self.fp = open( self.filename, 'rb' )

734

self.fp.seek( currentPointer )

753

self.fp.seek( currentPointer )

735

754

736

self.fileSize = os.path.getsize( self.filename )

755

self.fileSize = os.path.getsize( self.filename )

737

currentSize = self.fileSize - currentPointer

756

currentSize = self.fileSize - currentPointer

738

757

739

if ( currentSize >= neededSize ):

758

if ( currentSize >= neededSize ):

740

return 1

759

return 1

741

760

742

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

761

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

743

time.sleep( self.delay )

762

time.sleep( self.delay )

744

763

745

return 0

764

return 0

746

765

747

766

748

def __jumpToLastBlock(self):

767

def __jumpToLastBlock(self):

749

768

750

if not(self.__isFirstTimeOnline):

769

if not(self.__isFirstTimeOnline):

751

return

770

return

752

771

753

csize = self.fileSize - self.fp.tell()

772

csize = self.fileSize - self.fp.tell()

754

blocksize = self.processingHeaderObj.blockSize

773

blocksize = self.processingHeaderObj.blockSize

755

774

756

#salta el primer bloque de datos

775

#salta el primer bloque de datos

757

if csize > self.processingHeaderObj.blockSize:

776

if csize > self.processingHeaderObj.blockSize:

758

self.fp.seek(self.fp.tell() + blocksize)

777

self.fp.seek(self.fp.tell() + blocksize)

759

else:

778

else:

760

return

779

return

761

780

762

csize = self.fileSize - self.fp.tell()

781

csize = self.fileSize - self.fp.tell()

763

neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

782

neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

764

while True:

783

while True:

765

784

766

if self.fp.tell()<self.fileSize:

785

if self.fp.tell()<self.fileSize:

767

self.fp.seek(self.fp.tell() + neededsize)

786

self.fp.seek(self.fp.tell() + neededsize)

768

else:

787

else:

769

self.fp.seek(self.fp.tell() - neededsize)

788

self.fp.seek(self.fp.tell() - neededsize)

770

break

789

break

771

790

772

# csize = self.fileSize - self.fp.tell()

791

# csize = self.fileSize - self.fp.tell()

773

# neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

792

# neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize

774

# factor = int(csize/neededsize)

793

# factor = int(csize/neededsize)

775

# if factor > 0:

794

# if factor > 0:

776

# self.fp.seek(self.fp.tell() + factor*neededsize)

795

# self.fp.seek(self.fp.tell() + factor*neededsize)

777

796

778

self.flagIsNewFile = 0

797

self.flagIsNewFile = 0

779

self.__isFirstTimeOnline = 0

798

self.__isFirstTimeOnline = 0

780

799

781

800

782

def __setNewBlock(self):

801

def __setNewBlock(self):

783

802

784

if self.fp == None:

803

if self.fp == None:

785

return 0

804

return 0

786

805

787

if self.online:

806

if self.online:

788

self.__jumpToLastBlock()

807

self.__jumpToLastBlock()

789

808

790

if self.flagIsNewFile:

809

if self.flagIsNewFile:

791

return 1

810

return 1

792

811

793

self.lastUTTime = self.basicHeaderObj.utc

812

self.lastUTTime = self.basicHeaderObj.utc

794

currentSize = self.fileSize - self.fp.tell()

813

currentSize = self.fileSize - self.fp.tell()

795

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

814

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

796

815

797

if (currentSize >= neededSize):

816

if (currentSize >= neededSize):

798

self.__rdBasicHeader()

817

self.__rdBasicHeader()

799

return 1

818

return 1

800

819

801

if self.__waitNewBlock():

820

if self.__waitNewBlock():

802

return 1

821

return 1

803

822

804

if not(self.setNextFile()):

823

if not(self.setNextFile()):

805

return 0

824

return 0

806

825

807

deltaTime = self.basicHeaderObj.utc - self.lastUTTime #

826

deltaTime = self.basicHeaderObj.utc - self.lastUTTime #

808

827

809

self.flagTimeBlock = 0

828

self.flagTimeBlock = 0

810

829

811

if deltaTime > self.maxTimeStep:

830

if deltaTime > self.maxTimeStep:

812

self.flagTimeBlock = 1

831

self.flagTimeBlock = 1

813

832

814

return 1

833

return 1

815

834

816

835

817

def readNextBlock(self):

836

def readNextBlock(self):

818

if not(self.__setNewBlock()):

837

if not(self.__setNewBlock()):

819

return 0

838

return 0

820

839

821

if not(self.readBlock()):

840

if not(self.readBlock()):

822

return 0

841

return 0

823

842

824

return 1

843

return 1

825

844

826

def __rdProcessingHeader(self, fp=None):

845

def __rdProcessingHeader(self, fp=None):

827

if fp == None:

846

if fp == None:

828

fp = self.fp

847

fp = self.fp

829

848

830

self.processingHeaderObj.read(fp)

849

self.processingHeaderObj.read(fp)

831

850

832

def __rdRadarControllerHeader(self, fp=None):

851

def __rdRadarControllerHeader(self, fp=None):

833

if fp == None:

852

if fp == None:

834

fp = self.fp

853

fp = self.fp

835

854

836

self.radarControllerHeaderObj.read(fp)

855

self.radarControllerHeaderObj.read(fp)

837

856

838

def __rdSystemHeader(self, fp=None):

857

def __rdSystemHeader(self, fp=None):

839

if fp == None:

858

if fp == None:

840

fp = self.fp

859

fp = self.fp

841

860

842

self.systemHeaderObj.read(fp)

861

self.systemHeaderObj.read(fp)

843

862

844

def __rdBasicHeader(self, fp=None):

863

def __rdBasicHeader(self, fp=None):

845

if fp == None:

864

if fp == None:

846

fp = self.fp

865

fp = self.fp

847

866

848

self.basicHeaderObj.read(fp)

867

self.basicHeaderObj.read(fp)

849

868

850

869

851

def __readFirstHeader(self):

870

def __readFirstHeader(self):

852

self.__rdBasicHeader()

871

self.__rdBasicHeader()

853

self.__rdSystemHeader()

872

self.__rdSystemHeader()

854

self.__rdRadarControllerHeader()

873

self.__rdRadarControllerHeader()

855

self.__rdProcessingHeader()

874

self.__rdProcessingHeader()

856

875

857

self.firstHeaderSize = self.basicHeaderObj.size

876

self.firstHeaderSize = self.basicHeaderObj.size

858

877

859

datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

878

datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

860

if datatype == 0:

879

if datatype == 0:

861

datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])

880

datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])

862

elif datatype == 1:

881

elif datatype == 1:

863

datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])

882

datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])

864

elif datatype == 2:

883

elif datatype == 2:

865

datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])

884

datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])

866

elif datatype == 3:

885

elif datatype == 3:

867

datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])

886

datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])

868

elif datatype == 4:

887

elif datatype == 4:

869

datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])

888

datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])

870

elif datatype == 5:

889

elif datatype == 5:

871

datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])

890

datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])

872

else:

891

else:

873

raise ValueError, 'Data type was not defined'

892

raise ValueError, 'Data type was not defined'

874

893

875

self.dtype = datatype_str

894

self.dtype = datatype_str

876

self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c

895

self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c

877

self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)

896

self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)

878

# self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)

897

# self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)

879

# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)

898

# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)

880

self.getBlockDimension()

899

self.getBlockDimension()

881

900

882

901

883

def __verifyFile(self, filename, msgFlag=True):

902

def __verifyFile(self, filename, msgFlag=True):

884

msg = None

903

msg = None

885

try:

904

try:

886

fp = open(filename, 'rb')

905

fp = open(filename, 'rb')

887

currentPosition = fp.tell()

906

currentPosition = fp.tell()

888

except:

907

except:

889

if msgFlag:

908

if msgFlag:

890

print "The file %s can't be opened" % (filename)

909

print "The file %s can't be opened" % (filename)

891

return False

910

return False

892

911

893

neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize

912

neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize

894

913

895

if neededSize == 0:

914

if neededSize == 0:

896

basicHeaderObj = BasicHeader(LOCALTIME)

915

basicHeaderObj = BasicHeader(LOCALTIME)

897

systemHeaderObj = SystemHeader()

916

systemHeaderObj = SystemHeader()

898

radarControllerHeaderObj = RadarControllerHeader()

917

radarControllerHeaderObj = RadarControllerHeader()

899

processingHeaderObj = ProcessingHeader()

918

processingHeaderObj = ProcessingHeader()

900

919

901

try:

920

try:

902

if not( basicHeaderObj.read(fp) ): raise IOError

921

if not( basicHeaderObj.read(fp) ): raise IOError

903

if not( systemHeaderObj.read(fp) ): raise IOError

922

if not( systemHeaderObj.read(fp) ): raise IOError

904

if not( radarControllerHeaderObj.read(fp) ): raise IOError

923

if not( radarControllerHeaderObj.read(fp) ): raise IOError

905

if not( processingHeaderObj.read(fp) ): raise IOError

924

if not( processingHeaderObj.read(fp) ): raise IOError

906

data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

925

data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))

907

926

908

neededSize = processingHeaderObj.blockSize + basicHeaderObj.size

927

neededSize = processingHeaderObj.blockSize + basicHeaderObj.size

909

928

910

except:

929

except:

911

if msgFlag:

930

if msgFlag:

912

print "\tThe file %s is empty or it hasn't enough data" % filename

931

print "\tThe file %s is empty or it hasn't enough data" % filename

913

932

914

fp.close()

933

fp.close()

915

return False

934

return False

916

else:

935

else:

917

msg = "\tSkipping the file %s due to it hasn't enough data" %filename

936

msg = "\tSkipping the file %s due to it hasn't enough data" %filename

918

937

919

fp.close()

938

fp.close()

920

fileSize = os.path.getsize(filename)

939

fileSize = os.path.getsize(filename)

921

currentSize = fileSize - currentPosition

940

currentSize = fileSize - currentPosition

922

if currentSize < neededSize:

941

if currentSize < neededSize:

923

if msgFlag and (msg != None):

942

if msgFlag and (msg != None):

924

print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename

943

print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename

925

return False

944

return False

926

945

927

return True

946

return True

928

947

929

def setup(self,

948

def setup(self,

930

path=None,

949

path=None,

931

startDate=None,

950

startDate=None,

932

endDate=None,

951

endDate=None,

933

startTime=datetime.time(0,0,0),

952

startTime=datetime.time(0,0,0),

934

endTime=datetime.time(23,59,59),

953

endTime=datetime.time(23,59,59),

935

set=None,

954

set=None,

936

expLabel = "",

955

expLabel = "",

937

ext = None,

956

ext = None,

938

online = False,

957

online = False,

939

delay = 60,

958

delay = 60,

940

walk = True):

959

walk = True):

941

960

942

if path == None:

961

if path == None:

943

raise ValueError, "The path is not valid"

962

raise ValueError, "The path is not valid"

944

963

945

if ext == None:

964

if ext == None:

946

ext = self.ext

965

ext = self.ext

947

966

948

if online:

967

if online:

949

print "Searching files in online mode..."

968

print "Searching files in online mode..."

950

969

951

for nTries in range( self.nTries ):

970

for nTries in range( self.nTries ):

952

fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)

971

fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)

953

972

954

if fullpath:

973

if fullpath:

955

break

974

break

956

975

957

print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)

976

print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)

958

time.sleep( self.delay )

977

time.sleep( self.delay )

959

978

960

if not(fullpath):

979

if not(fullpath):

961

print "There 'isn't valied files in %s" % path

980

print "There 'isn't valied files in %s" % path

962

return None

981

return None

963

982

964

self.year = year

983

self.year = year

965

self.doy = doy

984

self.doy = doy

966

self.set = set - 1

985

self.set = set - 1

967

self.path = path

986

self.path = path

968

self.foldercounter = foldercounter

987

self.foldercounter = foldercounter

969

988

970

else:

989

else:

971

print "Searching files in offline mode ..."

990

print "Searching files in offline mode ..."

972

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

991

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

973

startTime=startTime, endTime=endTime,

992

startTime=startTime, endTime=endTime,

974

set=set, expLabel=expLabel, ext=ext,

993

set=set, expLabel=expLabel, ext=ext,

975

walk=walk)

994

walk=walk)

976

995

977

if not(pathList):

996

if not(pathList):

978

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

997

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

979

datetime.datetime.combine(startDate,startTime).ctime(),

998

datetime.datetime.combine(startDate,startTime).ctime(),

980

datetime.datetime.combine(endDate,endTime).ctime())

999

datetime.datetime.combine(endDate,endTime).ctime())

981

1000

982

sys.exit(-1)

1001

sys.exit(-1)

983

1002

984

1003

985

self.fileIndex = -1

1004

self.fileIndex = -1

986

self.pathList = pathList

1005

self.pathList = pathList

987

self.filenameList = filenameList

1006

self.filenameList = filenameList

988

1007

989

self.online = online

1008

self.online = online

990

self.delay = delay

1009

self.delay = delay

991

ext = ext.lower()

1010

ext = ext.lower()

992

self.ext = ext

1011

self.ext = ext

993

1012

994

if not(self.setNextFile()):

1013

if not(self.setNextFile()):

995

if (startDate!=None) and (endDate!=None):

1014

if (startDate!=None) and (endDate!=None):

996

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

1015

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

997

elif startDate != None:

1016

elif startDate != None:

998

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

1017

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

999

else:

1018

else:

1000

print "No files"

1019

print "No files"

1001

1020

1002

sys.exit(-1)

1021

sys.exit(-1)

1003

1022

1004

# self.updateDataHeader()

1023

# self.updateDataHeader()

1005

1024

1006

return self.dataOut

1025

return self.dataOut

1007

1026

1008

def getBasicHeader(self):

1027

def getBasicHeader(self):

1009

1028

1010

self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds

1029

self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds

1011

1030

1012

self.dataOut.flagTimeBlock = self.flagTimeBlock

1031

self.dataOut.flagTimeBlock = self.flagTimeBlock

1013

1032

1014

self.dataOut.timeZone = self.basicHeaderObj.timeZone

1033

self.dataOut.timeZone = self.basicHeaderObj.timeZone

1015

1034

1016

self.dataOut.dstFlag = self.basicHeaderObj.dstFlag

1035

self.dataOut.dstFlag = self.basicHeaderObj.dstFlag

1017

1036

1018

self.dataOut.errorCount = self.basicHeaderObj.errorCount

1037

self.dataOut.errorCount = self.basicHeaderObj.errorCount

1019

1038

1020

self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime

1039

self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime

1021

1040

1022

def getFirstHeader(self):

1041

def getFirstHeader(self):

1023

1042

1024

raise ValueError, "This method has not been implemented"

1043

raise ValueError, "This method has not been implemented"

1025

1044

1026

def getData():

1045

def getData():

1027

1046

1028

raise ValueError, "This method has not been implemented"

1047

raise ValueError, "This method has not been implemented"

1029

1048

1030

def hasNotDataInBuffer():

1049

def hasNotDataInBuffer():

1031

1050

1032

raise ValueError, "This method has not been implemented"

1051

raise ValueError, "This method has not been implemented"

1033

1052

1034

def readBlock():

1053

def readBlock():

1035

1054

1036

raise ValueError, "This method has not been implemented"

1055

raise ValueError, "This method has not been implemented"

1037

1056

1038

def isEndProcess(self):

1057

def isEndProcess(self):

1039

1058

1040

return self.flagNoMoreFiles

1059

return self.flagNoMoreFiles

1041

1060

1042

def printReadBlocks(self):

1061

def printReadBlocks(self):

1043

1062

1044

print "Number of read blocks per file %04d" %self.nReadBlocks

1063

print "Number of read blocks per file %04d" %self.nReadBlocks

1045

1064

1046

def printTotalBlocks(self):

1065

def printTotalBlocks(self):

1047

1066

1048

print "Number of read blocks %04d" %self.nTotalBlocks

1067

print "Number of read blocks %04d" %self.nTotalBlocks

1049

1068

1050

def printNumberOfBlock(self):

1069

def printNumberOfBlock(self):

1051

1070

1052

if self.flagIsNewBlock:

1071

if self.flagIsNewBlock:

1053

print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())

1072

print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())

1054

1073

1055

def printInfo(self):

1074

def printInfo(self):

1056

1075

1057

if self.__printInfo == False:

1076

if self.__printInfo == False:

1058

return

1077

return

1059

1078

1060

self.basicHeaderObj.printInfo()

1079

self.basicHeaderObj.printInfo()

1061

self.systemHeaderObj.printInfo()

1080

self.systemHeaderObj.printInfo()

1062

self.radarControllerHeaderObj.printInfo()

1081

self.radarControllerHeaderObj.printInfo()

1063

self.processingHeaderObj.printInfo()

1082

self.processingHeaderObj.printInfo()

1064

1083

1065

self.__printInfo = False

1084

self.__printInfo = False

1066

1085

1067

1086

1068

def run(self, **kwargs):

1087

def run(self, **kwargs):

1069

1088

1070

if not(self.isConfig):

1089

if not(self.isConfig):

1071

1090

1072

# self.dataOut = dataOut

1091

# self.dataOut = dataOut

1073

self.setup(**kwargs)

1092

self.setup(**kwargs)

1074

self.isConfig = True

1093

self.isConfig = True

1075

1094

1076

self.getData()

1095

self.getData()

1077

1096

1078

class JRODataWriter(JRODataIO, Operation):

1097

class JRODataWriter(JRODataIO, Operation):

1079

1098

1080

"""

1099

"""

1081

Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura

1100

Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura

1082

de los datos siempre se realiza por bloques.

1101

de los datos siempre se realiza por bloques.

1083

"""

1102

"""

1084

1103

1085

blockIndex = 0

1104

blockIndex = 0

1086

1105

1087

path = None

1106

path = None

1088

1107

1089

setFile = None

1108

setFile = None

1090

1109

1091

profilesPerBlock = None

1110

profilesPerBlock = None

1092

1111

1093

blocksPerFile = None

1112

blocksPerFile = None

1094

1113

1095

nWriteBlocks = 0

1114

nWriteBlocks = 0

1096

1115

1097

def __init__(self, dataOut=None):

1116

def __init__(self, dataOut=None):

1098

raise ValueError, "Not implemented"

1117

raise ValueError, "Not implemented"

1099

1118

1100

1119

1101

def hasAllDataInBuffer(self):

1120

def hasAllDataInBuffer(self):

1102

raise ValueError, "Not implemented"

1121

raise ValueError, "Not implemented"

1103

1122

1104

1123

1105

def setBlockDimension(self):

1124

def setBlockDimension(self):

1106

raise ValueError, "Not implemented"

1125

raise ValueError, "Not implemented"

1107

1126

1108

1127

1109

def writeBlock(self):

1128

def writeBlock(self):

1110

raise ValueError, "No implemented"

1129

raise ValueError, "No implemented"

1111

1130

1112

1131

1113

def putData(self):

1132

def putData(self):

1114

raise ValueError, "No implemented"

1133

raise ValueError, "No implemented"

1115

1134

1116

1135

1117

def setBasicHeader(self):

1136

def setBasicHeader(self):

1118

1137

1119

self.basicHeaderObj.size = self.basicHeaderSize #bytes

1138

self.basicHeaderObj.size = self.basicHeaderSize #bytes

1120

self.basicHeaderObj.version = self.versionFile

1139

self.basicHeaderObj.version = self.versionFile

1121

self.basicHeaderObj.dataBlock = self.nTotalBlocks

1140

self.basicHeaderObj.dataBlock = self.nTotalBlocks

1122

1141

1123

utc = numpy.floor(self.dataOut.utctime)

1142

utc = numpy.floor(self.dataOut.utctime)

1124

milisecond = (self.dataOut.utctime - utc)* 1000.0

1143

milisecond = (self.dataOut.utctime - utc)* 1000.0

1125

1144

1126

self.basicHeaderObj.utc = utc

1145

self.basicHeaderObj.utc = utc

1127

self.basicHeaderObj.miliSecond = milisecond

1146

self.basicHeaderObj.miliSecond = milisecond

1128

self.basicHeaderObj.timeZone = self.dataOut.timeZone

1147

self.basicHeaderObj.timeZone = self.dataOut.timeZone

1129

self.basicHeaderObj.dstFlag = self.dataOut.dstFlag

1148

self.basicHeaderObj.dstFlag = self.dataOut.dstFlag

1130

self.basicHeaderObj.errorCount = self.dataOut.errorCount

1149

self.basicHeaderObj.errorCount = self.dataOut.errorCount

1131

1150

1132

def setFirstHeader(self):

1151

def setFirstHeader(self):

1133

"""

1152

"""

1134

Obtiene una copia del First Header

1153

Obtiene una copia del First Header

1135

1154

1136

Affected:

1155

Affected:

1137

1156

1138

self.basicHeaderObj

1157

self.basicHeaderObj

1139

self.systemHeaderObj

1158

self.systemHeaderObj

1140

self.radarControllerHeaderObj

1159

self.radarControllerHeaderObj

1141

self.processingHeaderObj self.

1160

self.processingHeaderObj self.

1142

1161

1143

Return:

1162

Return:

1144

None

1163

None

1145

"""

1164

"""

1146

1165

1147

raise ValueError, "No implemented"

1166

raise ValueError, "No implemented"

1148

1167

1149

def __writeFirstHeader(self):

1168

def __writeFirstHeader(self):

1150

"""

1169

"""

1151

Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)

1170

Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)

1152

1171

1153

Affected:

1172

Affected:

1154

__dataType

1173

__dataType

1155

1174

1156

Return:

1175

Return:

1157

None

1176

None

1158

"""

1177

"""

1159

1178

1160

# CALCULAR PARAMETROS

1179

# CALCULAR PARAMETROS

1161

1180

1162

sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size

1181

sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size

1163

self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader

1182

self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader

1164

1183

1165

self.basicHeaderObj.write(self.fp)

1184

self.basicHeaderObj.write(self.fp)

1166

self.systemHeaderObj.write(self.fp)

1185

self.systemHeaderObj.write(self.fp)

1167

self.radarControllerHeaderObj.write(self.fp)

1186

self.radarControllerHeaderObj.write(self.fp)

1168

self.processingHeaderObj.write(self.fp)

1187

self.processingHeaderObj.write(self.fp)

1169

1188

1170

self.dtype = self.dataOut.dtype

1189

self.dtype = self.dataOut.dtype

1171

1190

1172

def __setNewBlock(self):

1191

def __setNewBlock(self):

1173

"""

1192

"""

1174

Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header

1193

Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header

1175

1194

1176

Return:

1195

Return:

1177

0 : si no pudo escribir nada

1196

0 : si no pudo escribir nada

1178

1 : Si escribio el Basic el First Header

1197

1 : Si escribio el Basic el First Header

1179

"""

1198

"""

1180

if self.fp == None:

1199

if self.fp == None:

1181

self.setNextFile()

1200

self.setNextFile()

1182

1201

1183

if self.flagIsNewFile:

1202

if self.flagIsNewFile:

1184

return 1

1203

return 1

1185

1204

1186

if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:

1205

if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:

1187

self.basicHeaderObj.write(self.fp)

1206

self.basicHeaderObj.write(self.fp)

1188

return 1

1207

return 1

1189

1208

1190

if not( self.setNextFile() ):

1209

if not( self.setNextFile() ):

1191

return 0

1210

return 0

1192

1211

1193

return 1

1212

return 1

1194

1213

1195

1214

1196

def writeNextBlock(self):

1215

def writeNextBlock(self):

1197

"""

1216

"""

1198

Selecciona el bloque siguiente de datos y los escribe en un file

1217

Selecciona el bloque siguiente de datos y los escribe en un file

1199

1218

1200

Return:

1219

Return:

1201

0 : Si no hizo pudo escribir el bloque de datos

1220

0 : Si no hizo pudo escribir el bloque de datos

1202

1 : Si no pudo escribir el bloque de datos

1221

1 : Si no pudo escribir el bloque de datos

1203

"""

1222

"""

1204

if not( self.__setNewBlock() ):

1223

if not( self.__setNewBlock() ):

1205

return 0

1224

return 0

1206

1225

1207

self.writeBlock()

1226

self.writeBlock()

1208

1227

1209

return 1

1228

return 1

1210

1229

1211

def setNextFile(self):

1230

def setNextFile(self):

1212

"""

1231

"""

1213

Determina el siguiente file que sera escrito

1232

Determina el siguiente file que sera escrito

1214

1233

1215

Affected:

1234

Affected:

1216

self.filename

1235

self.filename

1217

self.subfolder

1236

self.subfolder

1218

self.fp

1237

self.fp

1219

self.setFile

1238

self.setFile

1220

self.flagIsNewFile

1239

self.flagIsNewFile

1221

1240

1222

Return:

1241

Return:

1223

0 : Si el archivo no puede ser escrito

1242

0 : Si el archivo no puede ser escrito

1224

1 : Si el archivo esta listo para ser escrito

1243

1 : Si el archivo esta listo para ser escrito

1225

"""

1244

"""

1226

ext = self.ext

1245

ext = self.ext

1227

path = self.path

1246

path = self.path

1228

1247

1229

if self.fp != None:

1248

if self.fp != None:

1230

self.fp.close()

1249

self.fp.close()

1231

1250

1232

timeTuple = time.localtime( self.dataOut.utctime)

1251

timeTuple = time.localtime( self.dataOut.utctime)

1233

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

1252

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

1234

1253

1235

fullpath = os.path.join( path, subfolder )

1254

fullpath = os.path.join( path, subfolder )

1236

if not( os.path.exists(fullpath) ):

1255

if not( os.path.exists(fullpath) ):

1237

os.mkdir(fullpath)

1256

os.mkdir(fullpath)

1238

self.setFile = -1 #inicializo mi contador de seteo

1257

self.setFile = -1 #inicializo mi contador de seteo

1239

else:

1258

else:

1240

filesList = os.listdir( fullpath )

1259

filesList = os.listdir( fullpath )

1241

if len( filesList ) > 0:

1260

if len( filesList ) > 0:

1242

filesList = sorted( filesList, key=str.lower )

1261

filesList = sorted( filesList, key=str.lower )

1243

filen = filesList[-1]

1262

filen = filesList[-1]

1244

# el filename debera tener el siguiente formato

1263

# el filename debera tener el siguiente formato

1245

# 0 1234 567 89A BCDE (hex)

1264

# 0 1234 567 89A BCDE (hex)

1246

# x YYYY DDD SSS .ext

1265

# x YYYY DDD SSS .ext

1247

if isNumber( filen[8:11] ):

1266

if isNumber( filen[8:11] ):

1248

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

1267

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

1249

else:

1268

else:

1250

self.setFile = -1

1269

self.setFile = -1

1251

else:

1270

else:

1252

self.setFile = -1 #inicializo mi contador de seteo

1271

self.setFile = -1 #inicializo mi contador de seteo

1253

1272

1254

setFile = self.setFile

1273

setFile = self.setFile

1255

setFile += 1

1274

setFile += 1

1256

1275

1257

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

1276

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

1258

timeTuple.tm_year,

1277

timeTuple.tm_year,

1259

timeTuple.tm_yday,

1278

timeTuple.tm_yday,

1260

setFile,

1279

setFile,

1261

ext )

1280

ext )

1262

1281

1263

filename = os.path.join( path, subfolder, file )

1282

filename = os.path.join( path, subfolder, file )

1264

1283

1265

fp = open( filename,'wb' )

1284

fp = open( filename,'wb' )

1266

1285

1267

self.blockIndex = 0

1286

self.blockIndex = 0

1268

1287

1269

#guardando atributos

1288

#guardando atributos

1270

self.filename = filename

1289

self.filename = filename

1271

self.subfolder = subfolder

1290

self.subfolder = subfolder

1272

self.fp = fp

1291

self.fp = fp

1273

self.setFile = setFile

1292

self.setFile = setFile

1274

self.flagIsNewFile = 1

1293

self.flagIsNewFile = 1

1275

1294

1276

self.setFirstHeader()

1295

self.setFirstHeader()

1277

1296

1278

print 'Writing the file: %s'%self.filename

1297

print 'Writing the file: %s'%self.filename

1279

1298

1280

self.__writeFirstHeader()

1299

self.__writeFirstHeader()

1281

1300

1282

return 1

1301

return 1

1283

1302

1284

def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):

1303

def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):

1285

"""

1304

"""

1286

Setea el tipo de formato en la cual sera guardada la data y escribe el First Header

1305

Setea el tipo de formato en la cual sera guardada la data y escribe el First Header

1287

1306

1288

Inputs:

1307

Inputs:

1289

path : el path destino en el cual se escribiran los files a crear

1308

path : el path destino en el cual se escribiran los files a crear

1290

format : formato en el cual sera salvado un file

1309

format : formato en el cual sera salvado un file

1291

set : el setebo del file

1310

set : el setebo del file

1292

1311

1293

Return:

1312

Return:

1294

0 : Si no realizo un buen seteo

1313

0 : Si no realizo un buen seteo

1295

1 : Si realizo un buen seteo

1314

1 : Si realizo un buen seteo

1296

"""

1315

"""

1297

1316

1298

if ext == None:

1317

if ext == None:

1299

ext = self.ext

1318

ext = self.ext

1300

1319

1301

ext = ext.lower()

1320

ext = ext.lower()

1302

1321

1303

self.ext = ext

1322

self.ext = ext

1304

1323

1305

self.path = path

1324

self.path = path

1306

1325

1307

self.setFile = set - 1

1326

self.setFile = set - 1

1308

1327

1309

self.blocksPerFile = blocksPerFile

1328

self.blocksPerFile = blocksPerFile

1310

1329

1311

self.profilesPerBlock = profilesPerBlock

1330

self.profilesPerBlock = profilesPerBlock

1312

1331

1313

self.dataOut = dataOut

1332

self.dataOut = dataOut

1314

1333

1315

if not(self.setNextFile()):

1334

if not(self.setNextFile()):

1316

print "There isn't a next file"

1335

print "There isn't a next file"

1317

return 0

1336

return 0

1318

1337

1319

self.setBlockDimension()

1338

self.setBlockDimension()

1320

1339

1321

return 1

1340

return 1

1322

1341

1323

def run(self, dataOut, **kwargs):

1342

def run(self, dataOut, **kwargs):

1324

1343

1325

if not(self.isConfig):

1344

if not(self.isConfig):

1326

1345

1327

self.setup(dataOut, **kwargs)

1346

self.setup(dataOut, **kwargs)

1328

self.isConfig = True

1347

self.isConfig = True

1329

1348

1330

self.putData()

1349

self.putData()

1331

1350

1332

class VoltageReader(JRODataReader):

1351

class VoltageReader(JRODataReader):

1333

"""

1352

"""

1334

Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura

1353

Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura

1335

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:

1354

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:

1336

perfiles*alturas*canales) son almacenados en la variable "buffer".

1355

perfiles*alturas*canales) son almacenados en la variable "buffer".

1337

1356

1338

perfiles * alturas * canales

1357

perfiles * alturas * canales

1339

1358

1340

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1359

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1341

RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la

1360

RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la

1342

cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de

1361

cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de

1343

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1362

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1344

1363

1345

Example:

1364

Example:

1346

1365

1347

dpath = "/home/myuser/data"

1366

dpath = "/home/myuser/data"

1348

1367

1349

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1368

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1350

1369

1351

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1370

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1352

1371

1353

readerObj = VoltageReader()

1372

readerObj = VoltageReader()

1354

1373

1355

readerObj.setup(dpath, startTime, endTime)

1374

readerObj.setup(dpath, startTime, endTime)

1356

1375

1357

while(True):

1376

while(True):

1358

1377

1359

#to get one profile

1378

#to get one profile

1360

profile = readerObj.getData()

1379

profile = readerObj.getData()

1361

1380

1362

#print the profile

1381

#print the profile

1363

print profile

1382

print profile

1364

1383

1365

#If you want to see all datablock

1384

#If you want to see all datablock

1366

print readerObj.datablock

1385

print readerObj.datablock

1367

1386

1368

if readerObj.flagNoMoreFiles:

1387

if readerObj.flagNoMoreFiles:

1369

break

1388

break

1370

1389

1371

"""

1390

"""

1372

1391

1373

ext = ".r"

1392

ext = ".r"

1374

1393

1375

optchar = "D"

1394

optchar = "D"

1376

dataOut = None

1395

dataOut = None

1377

1396

1378

1397

1379

def __init__(self):

1398

def __init__(self):

1380

"""

1399

"""

1381

Inicializador de la clase VoltageReader para la lectura de datos de voltage.

1400

Inicializador de la clase VoltageReader para la lectura de datos de voltage.

1382

1401

1383

Input:

1402

Input:

1384

dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para

1403

dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para

1385

almacenar un perfil de datos cada vez que se haga un requerimiento

1404

almacenar un perfil de datos cada vez que se haga un requerimiento

1386

(getData). El perfil sera obtenido a partir del buffer de datos,

1405

(getData). El perfil sera obtenido a partir del buffer de datos,

1387

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1406

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1388

bloque de datos.

1407

bloque de datos.

1389

Si este parametro no es pasado se creara uno internamente.

1408

Si este parametro no es pasado se creara uno internamente.

1390

1409

1391

Variables afectadas:

1410

Variables afectadas:

1392

self.dataOut

1411

self.dataOut

1393

1412

1394

Return:

1413

Return:

1395

None

1414

None

1396

"""

1415

"""

1397

1416

1398

self.isConfig = False

1417

self.isConfig = False

1399

1418

1400

self.datablock = None

1419

self.datablock = None

1401

1420

1402

self.utc = 0

1421

self.utc = 0

1403

1422

1404

self.ext = ".r"

1423

self.ext = ".r"

1405

1424

1406

self.optchar = "D"

1425

self.optchar = "D"

1407

1426

1408

self.basicHeaderObj = BasicHeader(LOCALTIME)

1427

self.basicHeaderObj = BasicHeader(LOCALTIME)

1409

1428

1410

self.systemHeaderObj = SystemHeader()

1429

self.systemHeaderObj = SystemHeader()

1411

1430

1412

self.radarControllerHeaderObj = RadarControllerHeader()

1431

self.radarControllerHeaderObj = RadarControllerHeader()

1413

1432

1414

self.processingHeaderObj = ProcessingHeader()

1433

self.processingHeaderObj = ProcessingHeader()

1415

1434

1416

self.online = 0

1435

self.online = 0

1417

1436

1418

self.fp = None

1437

self.fp = None

1419

1438

1420

self.idFile = None

1439

self.idFile = None

1421

1440

1422

self.dtype = None

1441

self.dtype = None

1423

1442

1424

self.fileSizeByHeader = None

1443

self.fileSizeByHeader = None

1425

1444

1426

self.filenameList = []

1445

self.filenameList = []

1427

1446

1428

self.filename = None

1447

self.filename = None

1429

1448

1430

self.fileSize = None

1449

self.fileSize = None

1431

1450

1432

self.firstHeaderSize = 0

1451

self.firstHeaderSize = 0

1433

1452

1434

self.basicHeaderSize = 24

1453

self.basicHeaderSize = 24

1435

1454

1436

self.pathList = []

1455

self.pathList = []

1437

1456

1438

self.filenameList = []

1457

self.filenameList = []

1439

1458

1440

self.lastUTTime = 0

1459

self.lastUTTime = 0

1441

1460

1442

self.maxTimeStep = 30

1461

self.maxTimeStep = 30

1443

1462

1444

self.flagNoMoreFiles = 0

1463

self.flagNoMoreFiles = 0

1445

1464

1446

self.set = 0

1465

self.set = 0

1447

1466

1448

self.path = None

1467

self.path = None

1449

1468

1450

self.profileIndex = 2**32-1

1469

self.profileIndex = 2**32-1

1451

1470

1452

self.delay = 3 #seconds

1471

self.delay = 3 #seconds

1453

1472

1454

self.nTries = 3 #quantity tries

1473

self.nTries = 3 #quantity tries

1455

1474

1456

self.nFiles = 3 #number of files for searching

1475

self.nFiles = 3 #number of files for searching

1457

1476

1458

self.nReadBlocks = 0

1477

self.nReadBlocks = 0

1459

1478

1460

self.flagIsNewFile = 1

1479

self.flagIsNewFile = 1

1461

1480

1462

self.__isFirstTimeOnline = 1

1481

self.__isFirstTimeOnline = 1

1463

1482

1464

self.ippSeconds = 0

1483

self.ippSeconds = 0

1465

1484

1466

self.flagTimeBlock = 0

1485

self.flagTimeBlock = 0

1467

1486

1468

self.flagIsNewBlock = 0

1487

self.flagIsNewBlock = 0

1469

1488

1470

self.nTotalBlocks = 0

1489

self.nTotalBlocks = 0

1471

1490

1472

self.blocksize = 0

1491

self.blocksize = 0

1473

1492

1474

self.dataOut = self.createObjByDefault()

1493

self.dataOut = self.createObjByDefault()

1475

1494

1476

def createObjByDefault(self):

1495

def createObjByDefault(self):

1477

1496

1478

dataObj = Voltage()

1497

dataObj = Voltage()

1479

1498

1480

return dataObj

1499

return dataObj

1481

1500

1482

def __hasNotDataInBuffer(self):

1501

def __hasNotDataInBuffer(self):

1483

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1502

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1484

return 1

1503

return 1

1485

return 0

1504

return 0

1486

1505

1487

1506

1488

def getBlockDimension(self):

1507

def getBlockDimension(self):

1489

"""

1508

"""

1490

Obtiene la cantidad de puntos a leer por cada bloque de datos

1509

Obtiene la cantidad de puntos a leer por cada bloque de datos

1491

1510

1492

Affected:

1511

Affected:

1493

self.blocksize

1512

self.blocksize

1494

1513

1495

Return:

1514

Return:

1496

None

1515

None

1497

"""

1516

"""

1498

pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels

1517

pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels

1499

self.blocksize = pts2read

1518

self.blocksize = pts2read

1500

1519

1501

1520

1502

def readBlock(self):

1521

def readBlock(self):

1503

"""

1522

"""

1504

readBlock lee el bloque de datos desde la posicion actual del puntero del archivo

1523

readBlock lee el bloque de datos desde la posicion actual del puntero del archivo

1505

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

1524

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

1506

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

1525

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

1507

es seteado a 0

1526

es seteado a 0

1508

1527

1509

Inputs:

1528

Inputs:

1510

None

1529

None

1511

1530

1512

Return:

1531

Return:

1513

None

1532

None

1514

1533

1515

Affected:

1534

Affected:

1516

self.profileIndex

1535

self.profileIndex

1517

self.datablock

1536

self.datablock

1518

self.flagIsNewFile

1537

self.flagIsNewFile

1519

self.flagIsNewBlock

1538

self.flagIsNewBlock

1520

self.nTotalBlocks

1539

self.nTotalBlocks

1521

1540

1522

Exceptions:

1541

Exceptions:

1523

Si un bloque leido no es un bloque valido

1542

Si un bloque leido no es un bloque valido

1524

"""

1543

"""

1525

current_pointer_location = self.fp.tell()

1544

current_pointer_location = self.fp.tell()

1526

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1545

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1527

1546

1528

try:

1547

try:

1529

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1548

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1530

except:

1549

except:

1531

#print "The read block (%3d) has not enough data" %self.nReadBlocks

1550

#print "The read block (%3d) has not enough data" %self.nReadBlocks

1532

1551

1533

if self.waitDataBlock(pointer_location=current_pointer_location):

1552

if self.waitDataBlock(pointer_location=current_pointer_location):

1534

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1553

junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )

1535

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1554

junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )

1536

# return 0

1555

# return 0

1537

1556

1538

junk = numpy.transpose(junk, (2,0,1))

1557

junk = numpy.transpose(junk, (2,0,1))

1539

self.datablock = junk['real'] + junk['imag']*1j

1558

self.datablock = junk['real'] + junk['imag']*1j

1540

1559

1541

self.profileIndex = 0

1560

self.profileIndex = 0

1542

1561

1543

self.flagIsNewFile = 0

1562

self.flagIsNewFile = 0

1544

self.flagIsNewBlock = 1

1563

self.flagIsNewBlock = 1

1545

1564

1546

self.nTotalBlocks += 1

1565

self.nTotalBlocks += 1

1547

self.nReadBlocks += 1

1566

self.nReadBlocks += 1

1548

1567

1549

return 1

1568

return 1

1550

1569

1551

def getFirstHeader(self):

1570

def getFirstHeader(self):

1552

1571

1553

self.dataOut.dtype = self.dtype

1572

self.dataOut.dtype = self.dtype

1554

1573

1555

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

1574

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

1556

1575

1557

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

1576

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

1558

1577

1559

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

1578

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

1560

1579

1561

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

1580

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

1562

1581

1563

self.dataOut.ippSeconds = self.ippSeconds

1582

self.dataOut.ippSeconds = self.ippSeconds

1564

1583

1565

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt

1584

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt

1566

1585

1567

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

1586

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

1568

1587

1569

self.dataOut.flagShiftFFT = False

1588

self.dataOut.flagShiftFFT = False

1570

1589

1571

if self.radarControllerHeaderObj.code != None:

1590

if self.radarControllerHeaderObj.code != None:

1572

1591

1573

self.dataOut.nCode = self.radarControllerHeaderObj.nCode

1592

self.dataOut.nCode = self.radarControllerHeaderObj.nCode

1574

1593

1575

self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud

1594

self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud

1576

1595

1577

self.dataOut.code = self.radarControllerHeaderObj.code

1596

self.dataOut.code = self.radarControllerHeaderObj.code

1578

1597

1579

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

1598

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

1580

1599

1581

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

1600

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

1582

1601

1583

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

1602

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

1584

1603

1585

self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip

1604

self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip

1586

1605

1587

self.dataOut.flagShiftFFT = False

1606

self.dataOut.flagShiftFFT = False

1588

1607

1589

def getData(self):

1608

def getData(self):

1590

"""

1609

"""

1591

getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"

1610

getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"

1592

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

1611

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

1593

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

1612

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

1594

1613

1595

Ademas incrementa el contador del buffer en 1.

1614

Ademas incrementa el contador del buffer en 1.

1596

1615

1597

Return:

1616

Return:

1598

data : retorna un perfil de voltages (alturas * canales) copiados desde el

1617

data : retorna un perfil de voltages (alturas * canales) copiados desde el

1599

buffer. Si no hay mas archivos a leer retorna None.

1618

buffer. Si no hay mas archivos a leer retorna None.

1600

1619

1601

Variables afectadas:

1620

Variables afectadas:

1602

self.dataOut

1621

self.dataOut

1603

self.profileIndex

1622

self.profileIndex

1604

1623

1605

Affected:

1624

Affected:

1606

self.dataOut

1625

self.dataOut

1607

self.profileIndex

1626

self.profileIndex

1608

self.flagTimeBlock

1627

self.flagTimeBlock

1609

self.flagIsNewBlock

1628

self.flagIsNewBlock

1610

"""

1629

"""

1611

1630

1612

if self.flagNoMoreFiles:

1631

if self.flagNoMoreFiles:

1613

self.dataOut.flagNoData = True

1632

self.dataOut.flagNoData = True

1614

print 'Process finished'

1633

print 'Process finished'

1615

return 0

1634

return 0

1616

1635

1617

self.flagTimeBlock = 0

1636

self.flagTimeBlock = 0

1618

self.flagIsNewBlock = 0

1637

self.flagIsNewBlock = 0

1619

1638

1620

if self.__hasNotDataInBuffer():

1639

if self.__hasNotDataInBuffer():

1621

1640

1622

if not( self.readNextBlock() ):

1641

if not( self.readNextBlock() ):

1623

return 0

1642

return 0

1624

1643

1625

self.getFirstHeader()

1644

self.getFirstHeader()

1626

1645

1627

if self.datablock == None:

1646

if self.datablock == None:

1628

self.dataOut.flagNoData = True

1647

self.dataOut.flagNoData = True

1629

return 0

1648

return 0

1630

1649

1631

self.dataOut.data = self.datablock[:,self.profileIndex,:]

1650

self.dataOut.data = self.datablock[:,self.profileIndex,:]

1632

1651

1633

self.dataOut.flagNoData = False

1652

self.dataOut.flagNoData = False

1634

1653

1635

self.getBasicHeader()

1654

self.getBasicHeader()

1636

1655

1637

self.profileIndex += 1

1656

self.profileIndex += 1

1638

1657

1639

self.dataOut.realtime = self.online

1658

self.dataOut.realtime = self.online

1640

1659

1641

return self.dataOut.data

1660

return self.dataOut.data

1642

1661

1643

1662

1644

class VoltageWriter(JRODataWriter):

1663

class VoltageWriter(JRODataWriter):

1645

"""

1664

"""

1646

Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura

1665

Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura

1647

de los datos siempre se realiza por bloques.

1666

de los datos siempre se realiza por bloques.

1648

"""

1667

"""

1649

1668

1650

ext = ".r"

1669

ext = ".r"

1651

1670

1652

optchar = "D"

1671

optchar = "D"

1653

1672

1654

shapeBuffer = None

1673

shapeBuffer = None

1655

1674

1656

1675

1657

def __init__(self):

1676

def __init__(self):

1658

"""

1677

"""

1659

Inicializador de la clase VoltageWriter para la escritura de datos de espectros.

1678

Inicializador de la clase VoltageWriter para la escritura de datos de espectros.

1660

1679

1661

Affected:

1680

Affected:

1662

self.dataOut

1681

self.dataOut

1663

1682

1664

Return: None

1683

Return: None

1665

"""

1684

"""

1666

1685

1667

self.nTotalBlocks = 0

1686

self.nTotalBlocks = 0

1668

1687

1669

self.profileIndex = 0

1688

self.profileIndex = 0

1670

1689

1671

self.isConfig = False

1690

self.isConfig = False

1672

1691

1673

self.fp = None

1692

self.fp = None

1674

1693

1675

self.flagIsNewFile = 1

1694

self.flagIsNewFile = 1

1676

1695

1677

self.nTotalBlocks = 0

1696

self.nTotalBlocks = 0

1678

1697

1679

self.flagIsNewBlock = 0

1698

self.flagIsNewBlock = 0

1680

1699

1681

self.setFile = None

1700

self.setFile = None

1682

1701

1683

self.dtype = None

1702

self.dtype = None

1684

1703

1685

self.path = None

1704

self.path = None

1686

1705

1687

self.filename = None

1706

self.filename = None

1688

1707

1689

self.basicHeaderObj = BasicHeader(LOCALTIME)

1708

self.basicHeaderObj = BasicHeader(LOCALTIME)

1690

1709

1691

self.systemHeaderObj = SystemHeader()

1710

self.systemHeaderObj = SystemHeader()

1692

1711

1693

self.radarControllerHeaderObj = RadarControllerHeader()

1712

self.radarControllerHeaderObj = RadarControllerHeader()

1694

1713

1695

self.processingHeaderObj = ProcessingHeader()

1714

self.processingHeaderObj = ProcessingHeader()

1696

1715

1697

def hasAllDataInBuffer(self):

1716

def hasAllDataInBuffer(self):

1698

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1717

if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:

1699

return 1

1718

return 1

1700

return 0

1719

return 0

1701

1720

1702

1721

1703

def setBlockDimension(self):

1722

def setBlockDimension(self):

1704

"""

1723

"""

1705

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

1724

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

1706

1725

1707

Affected:

1726

Affected:

1708

self.shape_spc_Buffer

1727

self.shape_spc_Buffer

1709

self.shape_cspc_Buffer

1728

self.shape_cspc_Buffer

1710

self.shape_dc_Buffer

1729

self.shape_dc_Buffer

1711

1730

1712

Return: None

1731

Return: None

1713

"""

1732

"""

1714

self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,

1733

self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,

1715

self.processingHeaderObj.nHeights,

1734

self.processingHeaderObj.nHeights,

1716

self.systemHeaderObj.nChannels)

1735

self.systemHeaderObj.nChannels)

1717

1736

1718

self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,

1737

self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,

1719

self.processingHeaderObj.profilesPerBlock,

1738

self.processingHeaderObj.profilesPerBlock,

1720

self.processingHeaderObj.nHeights),

1739

self.processingHeaderObj.nHeights),

1721

dtype=numpy.dtype('complex64'))

1740

dtype=numpy.dtype('complex64'))

1722

1741

1723

1742

1724

def writeBlock(self):

1743

def writeBlock(self):

1725

"""

1744

"""

1726

Escribe el buffer en el file designado

1745

Escribe el buffer en el file designado

1727

1746

1728

Affected:

1747

Affected:

1729

self.profileIndex

1748

self.profileIndex

1730

self.flagIsNewFile

1749

self.flagIsNewFile

1731

self.flagIsNewBlock

1750

self.flagIsNewBlock

1732

self.nTotalBlocks

1751

self.nTotalBlocks

1733

self.blockIndex

1752

self.blockIndex

1734

1753

1735

Return: None

1754

Return: None

1736

"""

1755

"""

1737

data = numpy.zeros( self.shapeBuffer, self.dtype )

1756

data = numpy.zeros( self.shapeBuffer, self.dtype )

1738

1757

1739

junk = numpy.transpose(self.datablock, (1,2,0))

1758

junk = numpy.transpose(self.datablock, (1,2,0))

1740

1759

1741

data['real'] = junk.real

1760

data['real'] = junk.real

1742

data['imag'] = junk.imag

1761

data['imag'] = junk.imag

1743

1762

1744

data = data.reshape( (-1) )

1763

data = data.reshape( (-1) )

1745

1764

1746

data.tofile( self.fp )

1765

data.tofile( self.fp )

1747

1766

1748

self.datablock.fill(0)

1767

self.datablock.fill(0)

1749

1768

1750

self.profileIndex = 0

1769

self.profileIndex = 0

1751

self.flagIsNewFile = 0

1770

self.flagIsNewFile = 0

1752

self.flagIsNewBlock = 1

1771

self.flagIsNewBlock = 1

1753

1772

1754

self.blockIndex += 1

1773

self.blockIndex += 1

1755

self.nTotalBlocks += 1

1774

self.nTotalBlocks += 1

1756

1775

1757

def putData(self):

1776

def putData(self):

1758

"""

1777

"""

1759

Setea un bloque de datos y luego los escribe en un file

1778

Setea un bloque de datos y luego los escribe en un file

1760

1779

1761

Affected:

1780

Affected:

1762

self.flagIsNewBlock

1781

self.flagIsNewBlock

1763

self.profileIndex

1782

self.profileIndex

1764

1783

1765

Return:

1784

Return:

1766

0 : Si no hay data o no hay mas files que puedan escribirse

1785

0 : Si no hay data o no hay mas files que puedan escribirse

1767

1 : Si se escribio la data de un bloque en un file

1786

1 : Si se escribio la data de un bloque en un file

1768

"""

1787

"""

1769

if self.dataOut.flagNoData:

1788

if self.dataOut.flagNoData:

1770

return 0

1789

return 0

1771

1790

1772

self.flagIsNewBlock = 0

1791

self.flagIsNewBlock = 0

1773

1792

1774

if self.dataOut.flagTimeBlock:

1793

if self.dataOut.flagTimeBlock:

1775

1794

1776

self.datablock.fill(0)

1795

self.datablock.fill(0)

1777

self.profileIndex = 0

1796

self.profileIndex = 0

1778

self.setNextFile()

1797

self.setNextFile()

1779

1798

1780

if self.profileIndex == 0:

1799

if self.profileIndex == 0:

1781

self.setBasicHeader()

1800

self.setBasicHeader()

1782

1801

1783

self.datablock[:,self.profileIndex,:] = self.dataOut.data

1802

self.datablock[:,self.profileIndex,:] = self.dataOut.data

1784

1803

1785

self.profileIndex += 1

1804

self.profileIndex += 1

1786

1805

1787

if self.hasAllDataInBuffer():

1806

if self.hasAllDataInBuffer():

1788

#if self.flagIsNewFile:

1807

#if self.flagIsNewFile:

1789

self.writeNextBlock()

1808

self.writeNextBlock()

1790

# self.setFirstHeader()

1809

# self.setFirstHeader()

1791

1810

1792

return 1

1811

return 1

1793

1812

1794

def __getProcessFlags(self):

1813

def __getProcessFlags(self):

1795

1814

1796

processFlags = 0

1815

processFlags = 0

1797

1816

1798

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1817

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1799

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1818

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1800

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1819

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1801

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1820

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1802

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1821

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1803

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1822

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1804

1823

1805

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1824

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1806

1825

1807

1826

1808

1827

1809

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

1828

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

1810

PROCFLAG.DATATYPE_SHORT,

1829

PROCFLAG.DATATYPE_SHORT,

1811

PROCFLAG.DATATYPE_LONG,

1830

PROCFLAG.DATATYPE_LONG,

1812

PROCFLAG.DATATYPE_INT64,

1831

PROCFLAG.DATATYPE_INT64,

1813

PROCFLAG.DATATYPE_FLOAT,

1832

PROCFLAG.DATATYPE_FLOAT,

1814

PROCFLAG.DATATYPE_DOUBLE]

1833

PROCFLAG.DATATYPE_DOUBLE]

1815

1834

1816

1835

1817

for index in range(len(dtypeList)):

1836

for index in range(len(dtypeList)):

1818

if self.dataOut.dtype == dtypeList[index]:

1837

if self.dataOut.dtype == dtypeList[index]:

1819

dtypeValue = datatypeValueList[index]

1838

dtypeValue = datatypeValueList[index]

1820

break

1839

break

1821

1840

1822

processFlags += dtypeValue

1841

processFlags += dtypeValue

1823

1842

1824

if self.dataOut.flagDecodeData:

1843

if self.dataOut.flagDecodeData:

1825

processFlags += PROCFLAG.DECODE_DATA

1844

processFlags += PROCFLAG.DECODE_DATA

1826

1845

1827

if self.dataOut.flagDeflipData:

1846

if self.dataOut.flagDeflipData:

1828

processFlags += PROCFLAG.DEFLIP_DATA

1847

processFlags += PROCFLAG.DEFLIP_DATA

1829

1848

1830

if self.dataOut.code != None:

1849

if self.dataOut.code != None:

1831

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

1850

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

1832

1851

1833

if self.dataOut.nCohInt > 1:

1852

if self.dataOut.nCohInt > 1:

1834

processFlags += PROCFLAG.COHERENT_INTEGRATION

1853

processFlags += PROCFLAG.COHERENT_INTEGRATION

1835

1854

1836

return processFlags

1855

return processFlags

1837

1856

1838

1857

1839

def __getBlockSize(self):

1858

def __getBlockSize(self):

1840

'''

1859

'''

1841

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage

1860

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage

1842

'''

1861

'''

1843

1862

1844

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1863

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

1845

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1864

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

1846

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1865

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

1847

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1866

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

1848

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1867

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

1849

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1868

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

1850

1869

1851

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1870

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

1852

datatypeValueList = [1,2,4,8,4,8]

1871

datatypeValueList = [1,2,4,8,4,8]

1853

for index in range(len(dtypeList)):

1872

for index in range(len(dtypeList)):

1854

if self.dataOut.dtype == dtypeList[index]:

1873

if self.dataOut.dtype == dtypeList[index]:

1855

datatypeValue = datatypeValueList[index]

1874

datatypeValue = datatypeValueList[index]

1856

break

1875

break

1857

1876

1858

blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * datatypeValue * 2)

1877

blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * datatypeValue * 2)

1859

1878

1860

return blocksize

1879

return blocksize

1861

1880

1862

def setFirstHeader(self):

1881

def setFirstHeader(self):

1863

1882

1864

"""

1883

"""

1865

Obtiene una copia del First Header

1884

Obtiene una copia del First Header

1866

1885

1867

Affected:

1886

Affected:

1868

self.systemHeaderObj

1887

self.systemHeaderObj

1869

self.radarControllerHeaderObj

1888

self.radarControllerHeaderObj

1870

self.dtype

1889

self.dtype

1871

1890

1872

Return:

1891

Return:

1873

None

1892

None

1874

"""

1893

"""

1875

1894

1876

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

1895

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

1877

self.systemHeaderObj.nChannels = self.dataOut.nChannels

1896

self.systemHeaderObj.nChannels = self.dataOut.nChannels

1878

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

1897

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

1879

1898

1880

self.setBasicHeader()

1899

self.setBasicHeader()

1881

1900

1882

processingHeaderSize = 40 # bytes

1901

processingHeaderSize = 40 # bytes

1883

self.processingHeaderObj.dtype = 0 # Voltage

1902

self.processingHeaderObj.dtype = 0 # Voltage

1884

self.processingHeaderObj.blockSize = self.__getBlockSize()

1903

self.processingHeaderObj.blockSize = self.__getBlockSize()

1885

self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock

1904

self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock

1886

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

1905

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

1887

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

1906

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

1888

self.processingHeaderObj.processFlags = self.__getProcessFlags()

1907

self.processingHeaderObj.processFlags = self.__getProcessFlags()

1889

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt

1908

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt

1890

self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage

1909

self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage

1891

self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage

1910

self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage

1892

1911

1893

# if self.dataOut.code != None:

1912

# if self.dataOut.code != None:

1894

# self.processingHeaderObj.code = self.dataOut.code

1913

# self.processingHeaderObj.code = self.dataOut.code

1895

# self.processingHeaderObj.nCode = self.dataOut.nCode

1914

# self.processingHeaderObj.nCode = self.dataOut.nCode

1896

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

1915

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

1897

# codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)

1916

# codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)

1898

# processingHeaderSize += codesize

1917

# processingHeaderSize += codesize

1899

1918

1900

if self.processingHeaderObj.nWindows != 0:

1919

if self.processingHeaderObj.nWindows != 0:

1901

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

1920

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

1902

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

1921

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

1903

self.processingHeaderObj.nHeights = self.dataOut.nHeights

1922

self.processingHeaderObj.nHeights = self.dataOut.nHeights

1904

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

1923

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

1905

processingHeaderSize += 12

1924

processingHeaderSize += 12

1906

1925

1907

self.processingHeaderObj.size = processingHeaderSize

1926

self.processingHeaderObj.size = processingHeaderSize

1908

1927

1909

class SpectraReader(JRODataReader):

1928

class SpectraReader(JRODataReader):

1910

"""

1929

"""

1911

Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura

1930

Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura

1912

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)

1931

de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)

1913

son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.

1932

son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.

1914

1933

1915

paresCanalesIguales * alturas * perfiles (Self Spectra)

1934

paresCanalesIguales * alturas * perfiles (Self Spectra)

1916

paresCanalesDiferentes * alturas * perfiles (Cross Spectra)

1935

paresCanalesDiferentes * alturas * perfiles (Cross Spectra)

1917

canales * alturas (DC Channels)

1936

canales * alturas (DC Channels)

1918

1937

1919

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1938

Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,

1920

RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la

1939

RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la

1921

cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de

1940

cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de

1922

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1941

datos desde el "buffer" cada vez que se ejecute el metodo "getData".

1923

1942

1924

Example:

1943

Example:

1925

dpath = "/home/myuser/data"

1944

dpath = "/home/myuser/data"

1926

1945

1927

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1946

startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)

1928

1947

1929

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1948

endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)

1930

1949

1931

readerObj = SpectraReader()

1950

readerObj = SpectraReader()

1932

1951

1933

readerObj.setup(dpath, startTime, endTime)

1952

readerObj.setup(dpath, startTime, endTime)

1934

1953

1935

while(True):

1954

while(True):

1936

1955

1937

readerObj.getData()

1956

readerObj.getData()

1938

1957

1939

print readerObj.data_spc

1958

print readerObj.data_spc

1940

1959

1941

print readerObj.data_cspc

1960

print readerObj.data_cspc

1942

1961

1943

print readerObj.data_dc

1962

print readerObj.data_dc

1944

1963

1945

if readerObj.flagNoMoreFiles:

1964

if readerObj.flagNoMoreFiles:

1946

break

1965

break

1947

1966

1948

"""

1967

"""

1949

1968

1950

pts2read_SelfSpectra = 0

1969

pts2read_SelfSpectra = 0

1951

1970

1952

pts2read_CrossSpectra = 0

1971

pts2read_CrossSpectra = 0

1953

1972

1954

pts2read_DCchannels = 0

1973

pts2read_DCchannels = 0

1955

1974

1956

ext = ".pdata"

1975

ext = ".pdata"

1957

1976

1958

optchar = "P"

1977

optchar = "P"

1959

1978

1960

dataOut = None

1979

dataOut = None

1961

1980

1962

nRdChannels = None

1981

nRdChannels = None

1963

1982

1964

nRdPairs = None

1983

nRdPairs = None

1965

1984

1966

rdPairList = []

1985

rdPairList = []

1967

1986

1968

def __init__(self):

1987

def __init__(self):

1969

"""

1988

"""

1970

Inicializador de la clase SpectraReader para la lectura de datos de espectros.

1989

Inicializador de la clase SpectraReader para la lectura de datos de espectros.

1971

1990

1972

Inputs:

1991

Inputs:

1973

dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para

1992

dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para

1974

almacenar un perfil de datos cada vez que se haga un requerimiento

1993

almacenar un perfil de datos cada vez que se haga un requerimiento

1975

(getData). El perfil sera obtenido a partir del buffer de datos,

1994

(getData). El perfil sera obtenido a partir del buffer de datos,

1976

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1995

si el buffer esta vacio se hara un nuevo proceso de lectura de un

1977

bloque de datos.

1996

bloque de datos.

1978

Si este parametro no es pasado se creara uno internamente.

1997

Si este parametro no es pasado se creara uno internamente.

1979

1998

1980

Affected:

1999

Affected:

1981

self.dataOut

2000

self.dataOut

1982

2001

1983

Return : None

2002

Return : None

1984

"""

2003

"""

1985

2004

1986

self.isConfig = False

2005

self.isConfig = False

1987

2006

1988

self.pts2read_SelfSpectra = 0

2007

self.pts2read_SelfSpectra = 0

1989

2008

1990

self.pts2read_CrossSpectra = 0

2009

self.pts2read_CrossSpectra = 0

1991

2010

1992

self.pts2read_DCchannels = 0

2011

self.pts2read_DCchannels = 0

1993

2012

1994

self.datablock = None

2013

self.datablock = None

1995

2014

1996

self.utc = None

2015

self.utc = None

1997

2016

1998

self.ext = ".pdata"

2017

self.ext = ".pdata"

1999

2018

2000

self.optchar = "P"

2019

self.optchar = "P"

2001

2020

2002

self.basicHeaderObj = BasicHeader(LOCALTIME)

2021

self.basicHeaderObj = BasicHeader(LOCALTIME)

2003

2022

2004

self.systemHeaderObj = SystemHeader()

2023

self.systemHeaderObj = SystemHeader()

2005

2024

2006

self.radarControllerHeaderObj = RadarControllerHeader()

2025

self.radarControllerHeaderObj = RadarControllerHeader()

2007

2026

2008

self.processingHeaderObj = ProcessingHeader()

2027

self.processingHeaderObj = ProcessingHeader()

2009

2028

2010

self.online = 0

2029

self.online = 0

2011

2030

2012

self.fp = None

2031

self.fp = None

2013

2032

2014

self.idFile = None

2033

self.idFile = None

2015

2034

2016

self.dtype = None

2035

self.dtype = None

2017

2036

2018

self.fileSizeByHeader = None

2037

self.fileSizeByHeader = None

2019

2038

2020

self.filenameList = []

2039

self.filenameList = []

2021

2040

2022

self.filename = None

2041

self.filename = None

2023

2042

2024

self.fileSize = None

2043

self.fileSize = None

2025

2044

2026

self.firstHeaderSize = 0

2045

self.firstHeaderSize = 0

2027

2046

2028

self.basicHeaderSize = 24

2047

self.basicHeaderSize = 24

2029

2048

2030

self.pathList = []

2049

self.pathList = []

2031

2050

2032

self.lastUTTime = 0

2051

self.lastUTTime = 0

2033

2052

2034

self.maxTimeStep = 30

2053

self.maxTimeStep = 30

2035

2054

2036

self.flagNoMoreFiles = 0

2055

self.flagNoMoreFiles = 0

2037

2056

2038

self.set = 0

2057

self.set = 0

2039

2058

2040

self.path = None

2059

self.path = None

2041

2060

2042

self.delay = 60 #seconds

2061

self.delay = 60 #seconds

2043

2062

2044

self.nTries = 3 #quantity tries

2063

self.nTries = 3 #quantity tries

2045

2064

2046

self.nFiles = 3 #number of files for searching

2065

self.nFiles = 3 #number of files for searching

2047

2066

2048

self.nReadBlocks = 0

2067

self.nReadBlocks = 0

2049

2068

2050

self.flagIsNewFile = 1

2069

self.flagIsNewFile = 1

2051

2070

2052

self.__isFirstTimeOnline = 1

2071

self.__isFirstTimeOnline = 1

2053

2072

2054

self.ippSeconds = 0

2073

self.ippSeconds = 0

2055

2074

2056

self.flagTimeBlock = 0

2075

self.flagTimeBlock = 0

2057

2076

2058

self.flagIsNewBlock = 0

2077

self.flagIsNewBlock = 0

2059

2078

2060

self.nTotalBlocks = 0

2079

self.nTotalBlocks = 0

2061

2080

2062

self.blocksize = 0

2081

self.blocksize = 0

2063

2082

2064

self.dataOut = self.createObjByDefault()

2083

self.dataOut = self.createObjByDefault()

2065

2084

2066

self.profileIndex = 1 #Always

2085

self.profileIndex = 1 #Always

2067

2086

2068

2087

2069

def createObjByDefault(self):

2088

def createObjByDefault(self):

2070

2089

2071

dataObj = Spectra()

2090

dataObj = Spectra()

2072

2091

2073

return dataObj

2092

return dataObj

2074

2093

2075

def __hasNotDataInBuffer(self):

2094

def __hasNotDataInBuffer(self):

2076

return 1

2095

return 1

2077

2096

2078

2097

2079

def getBlockDimension(self):

2098

def getBlockDimension(self):

2080

"""

2099

"""

2081

Obtiene la cantidad de puntos a leer por cada bloque de datos

2100

Obtiene la cantidad de puntos a leer por cada bloque de datos

2082

2101

2083

Affected:

2102

Affected:

2084

self.nRdChannels

2103

self.nRdChannels

2085

self.nRdPairs

2104

self.nRdPairs

2086

self.pts2read_SelfSpectra

2105

self.pts2read_SelfSpectra

2087

self.pts2read_CrossSpectra

2106

self.pts2read_CrossSpectra

2088

self.pts2read_DCchannels

2107

self.pts2read_DCchannels

2089

self.blocksize

2108

self.blocksize

2090

self.dataOut.nChannels

2109

self.dataOut.nChannels

2091

self.dataOut.nPairs

2110

self.dataOut.nPairs

2092

2111

2093

Return:

2112

Return:

2094

None

2113

None

2095

"""

2114

"""

2096

self.nRdChannels = 0

2115

self.nRdChannels = 0

2097

self.nRdPairs = 0

2116

self.nRdPairs = 0

2098

self.rdPairList = []

2117

self.rdPairList = []

2099

2118

2100

for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):

2119

for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):

2101

if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:

2120

if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:

2102

self.nRdChannels = self.nRdChannels + 1 #par de canales iguales

2121

self.nRdChannels = self.nRdChannels + 1 #par de canales iguales

2103

else:

2122

else:

2104

self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes

2123

self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes

2105

self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))

2124

self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))

2106

2125

2107

pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock

2126

pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock

2108

2127

2109

self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)

2128

self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)

2110

self.blocksize = self.pts2read_SelfSpectra

2129

self.blocksize = self.pts2read_SelfSpectra

2111

2130

2112

if self.processingHeaderObj.flag_cspc:

2131

if self.processingHeaderObj.flag_cspc:

2113

self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)

2132

self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)

2114

self.blocksize += self.pts2read_CrossSpectra

2133

self.blocksize += self.pts2read_CrossSpectra

2115

2134

2116

if self.processingHeaderObj.flag_dc:

2135

if self.processingHeaderObj.flag_dc:

2117

self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)

2136

self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)

2118

self.blocksize += self.pts2read_DCchannels

2137

self.blocksize += self.pts2read_DCchannels

2119

2138

2120

# self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels

2139

# self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels

2121

2140

2122

2141

2123

def readBlock(self):

2142

def readBlock(self):

2124

"""

2143

"""

2125

Lee el bloque de datos desde la posicion actual del puntero del archivo

2144

Lee el bloque de datos desde la posicion actual del puntero del archivo

2126

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

2145

(self.fp) y actualiza todos los parametros relacionados al bloque de datos

2127

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

2146

(metadata + data). La data leida es almacenada en el buffer y el contador del buffer

2128

es seteado a 0

2147

es seteado a 0

2129

2148

2130

Return: None

2149

Return: None

2131

2150

2132

Variables afectadas:

2151

Variables afectadas:

2133

2152

2134

self.flagIsNewFile

2153

self.flagIsNewFile

2135

self.flagIsNewBlock

2154

self.flagIsNewBlock

2136

self.nTotalBlocks

2155

self.nTotalBlocks

2137

self.data_spc

2156

self.data_spc

2138

self.data_cspc

2157

self.data_cspc

2139

self.data_dc

2158

self.data_dc

2140

2159

2141

Exceptions:

2160

Exceptions:

2142

Si un bloque leido no es un bloque valido

2161

Si un bloque leido no es un bloque valido

2143

"""

2162

"""

2144

blockOk_flag = False

2163

blockOk_flag = False

2145

fpointer = self.fp.tell()

2164

fpointer = self.fp.tell()

2146

2165

2147

spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )

2166

spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )

2148

spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2167

spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2149

2168

2150

if self.processingHeaderObj.flag_cspc:

2169

if self.processingHeaderObj.flag_cspc:

2151

cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )

2170

cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )

2152

cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2171

cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D

2153

2172

2154

if self.processingHeaderObj.flag_dc:

2173

if self.processingHeaderObj.flag_dc:

2155

dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )

2174

dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )

2156

dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D

2175

dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D

2157

2176

2158

2177

2159

if not(self.processingHeaderObj.shif_fft):

2178

if not(self.processingHeaderObj.shif_fft):

2160

#desplaza a la derecha en el eje 2 determinadas posiciones

2179

#desplaza a la derecha en el eje 2 determinadas posiciones

2161

shift = int(self.processingHeaderObj.profilesPerBlock/2)

2180

shift = int(self.processingHeaderObj.profilesPerBlock/2)

2162

spc = numpy.roll( spc, shift , axis=2 )

2181

spc = numpy.roll( spc, shift , axis=2 )

2163

2182

2164

if self.processingHeaderObj.flag_cspc:

2183

if self.processingHeaderObj.flag_cspc:

2165

#desplaza a la derecha en el eje 2 determinadas posiciones

2184

#desplaza a la derecha en el eje 2 determinadas posiciones

2166

cspc = numpy.roll( cspc, shift, axis=2 )

2185

cspc = numpy.roll( cspc, shift, axis=2 )

2167

2186

2168

# self.processingHeaderObj.shif_fft = True

2187

# self.processingHeaderObj.shif_fft = True

2169

2188

2170

spc = numpy.transpose( spc, (0,2,1) )

2189

spc = numpy.transpose( spc, (0,2,1) )

2171

self.data_spc = spc

2190

self.data_spc = spc

2172

2191

2173

if self.processingHeaderObj.flag_cspc:

2192

if self.processingHeaderObj.flag_cspc:

2174

cspc = numpy.transpose( cspc, (0,2,1) )

2193

cspc = numpy.transpose( cspc, (0,2,1) )

2175

self.data_cspc = cspc['real'] + cspc['imag']*1j

2194

self.data_cspc = cspc['real'] + cspc['imag']*1j

2176

else:

2195

else:

2177

self.data_cspc = None

2196

self.data_cspc = None

2178

2197

2179

if self.processingHeaderObj.flag_dc:

2198

if self.processingHeaderObj.flag_dc:

2180

self.data_dc = dc['real'] + dc['imag']*1j

2199

self.data_dc = dc['real'] + dc['imag']*1j

2181

else:

2200

else:

2182

self.data_dc = None

2201

self.data_dc = None

2183

2202

2184

self.flagIsNewFile = 0

2203

self.flagIsNewFile = 0

2185

self.flagIsNewBlock = 1

2204

self.flagIsNewBlock = 1

2186

2205

2187

self.nTotalBlocks += 1

2206

self.nTotalBlocks += 1

2188

self.nReadBlocks += 1

2207

self.nReadBlocks += 1

2189

2208

2190

return 1

2209

return 1

2191

2210

2192

def getFirstHeader(self):

2211

def getFirstHeader(self):

2193

2212

2194

self.dataOut.dtype = self.dtype

2213

self.dataOut.dtype = self.dtype

2195

2214

2196

self.dataOut.nPairs = self.nRdPairs

2215

self.dataOut.nPairs = self.nRdPairs

2197

2216

2198

self.dataOut.pairsList = self.rdPairList

2217

self.dataOut.pairsList = self.rdPairList

2199

2218

2200

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

2219

self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock

2201

2220

2202

self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock

2221

self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock

2203

2222

2204

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

2223

self.dataOut.nCohInt = self.processingHeaderObj.nCohInt

2205

2224

2206

self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt

2225

self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt

2207

2226

2208

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

2227

xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight

2209

2228

2210

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

2229

self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)

2211

2230

2212

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

2231

self.dataOut.channelList = range(self.systemHeaderObj.nChannels)

2213

2232

2214

self.dataOut.ippSeconds = self.ippSeconds

2233

self.dataOut.ippSeconds = self.ippSeconds

2215

2234

2216

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints

2235

self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints

2217

2236

2218

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

2237

self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()

2219

2238

2220

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

2239

self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()

2221

2240

2222

self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft

2241

self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft

2223

2242

2224

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

2243

self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada

2225

2244

2226

self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip

2245

self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip

2227

2246

2228

if self.processingHeaderObj.code != None:

2247

if self.processingHeaderObj.code != None:

2229

2248

2230

self.dataOut.nCode = self.processingHeaderObj.nCode

2249

self.dataOut.nCode = self.processingHeaderObj.nCode

2231

2250

2232

self.dataOut.nBaud = self.processingHeaderObj.nBaud

2251

self.dataOut.nBaud = self.processingHeaderObj.nBaud

2233

2252

2234

self.dataOut.code = self.processingHeaderObj.code

2253

self.dataOut.code = self.processingHeaderObj.code

2235

2254

2236

self.dataOut.flagDecodeData = True

2255

self.dataOut.flagDecodeData = True

2237

2256

2238

def getData(self):

2257

def getData(self):

2239

"""

2258

"""

2240

Copia el buffer de lectura a la clase "Spectra",

2259

Copia el buffer de lectura a la clase "Spectra",

2241

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

2260

con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de

2242

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

2261

lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"

2243

2262

2244

Return:

2263

Return:

2245

0 : Si no hay mas archivos disponibles

2264

0 : Si no hay mas archivos disponibles

2246

1 : Si hizo una buena copia del buffer

2265

1 : Si hizo una buena copia del buffer

2247

2266

2248

Affected:

2267

Affected:

2249

self.dataOut

2268

self.dataOut

2250

2269

2251

self.flagTimeBlock

2270

self.flagTimeBlock

2252

self.flagIsNewBlock

2271

self.flagIsNewBlock

2253

"""

2272

"""

2254

2273

2255

if self.flagNoMoreFiles:

2274

if self.flagNoMoreFiles:

2256

self.dataOut.flagNoData = True

2275

self.dataOut.flagNoData = True

2257

print 'Process finished'

2276

print 'Process finished'

2258

return 0

2277

return 0

2259

2278

2260

self.flagTimeBlock = 0

2279

self.flagTimeBlock = 0

2261

self.flagIsNewBlock = 0

2280

self.flagIsNewBlock = 0

2262

2281

2263

if self.__hasNotDataInBuffer():

2282

if self.__hasNotDataInBuffer():

2264

2283

2265

if not( self.readNextBlock() ):

2284

if not( self.readNextBlock() ):

2266

self.dataOut.flagNoData = True

2285

self.dataOut.flagNoData = True

2267

return 0

2286

return 0

2268

2287

2269

#data es un numpy array de 3 dmensiones (perfiles, alturas y canales)

2288

#data es un numpy array de 3 dmensiones (perfiles, alturas y canales)

2270

2289

2271

if self.data_dc == None:

2290

if self.data_dc == None:

2272

self.dataOut.flagNoData = True

2291

self.dataOut.flagNoData = True

2273

return 0

2292

return 0

2274

2293

2275

self.getBasicHeader()

2294

self.getBasicHeader()

2276

2295

2277

self.getFirstHeader()

2296

self.getFirstHeader()

2278

2297

2279

self.dataOut.data_spc = self.data_spc

2298

self.dataOut.data_spc = self.data_spc

2280

2299

2281

self.dataOut.data_cspc = self.data_cspc

2300

self.dataOut.data_cspc = self.data_cspc

2282

2301

2283

self.dataOut.data_dc = self.data_dc

2302

self.dataOut.data_dc = self.data_dc

2284

2303

2285

self.dataOut.flagNoData = False

2304

self.dataOut.flagNoData = False

2286

2305

2287

self.dataOut.realtime = self.online

2306

self.dataOut.realtime = self.online

2288

2307

2289

return self.dataOut.data_spc

2308

return self.dataOut.data_spc

2290

2309

2291

2310

2292

class SpectraWriter(JRODataWriter):

2311

class SpectraWriter(JRODataWriter):

2293

2312

2294

"""

2313

"""

2295

Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura

2314

Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura

2296

de los datos siempre se realiza por bloques.

2315

de los datos siempre se realiza por bloques.

2297

"""

2316

"""

2298

2317

2299

ext = ".pdata"

2318

ext = ".pdata"

2300

2319

2301

optchar = "P"

2320

optchar = "P"

2302

2321

2303

shape_spc_Buffer = None

2322

shape_spc_Buffer = None

2304

2323

2305

shape_cspc_Buffer = None

2324

shape_cspc_Buffer = None

2306

2325

2307

shape_dc_Buffer = None

2326

shape_dc_Buffer = None

2308

2327

2309

data_spc = None

2328

data_spc = None

2310

2329

2311

data_cspc = None

2330

data_cspc = None

2312

2331

2313

data_dc = None

2332

data_dc = None

2314

2333

2315

# dataOut = None

2334

# dataOut = None

2316

2335

2317

def __init__(self):

2336

def __init__(self):

2318

"""

2337

"""

2319

Inicializador de la clase SpectraWriter para la escritura de datos de espectros.

2338

Inicializador de la clase SpectraWriter para la escritura de datos de espectros.

2320

2339

2321

Affected:

2340

Affected:

2322

self.dataOut

2341

self.dataOut

2323

self.basicHeaderObj

2342

self.basicHeaderObj

2324

self.systemHeaderObj

2343

self.systemHeaderObj

2325

self.radarControllerHeaderObj

2344

self.radarControllerHeaderObj

2326

self.processingHeaderObj

2345

self.processingHeaderObj

2327

2346

2328

Return: None

2347

Return: None

2329

"""

2348

"""

2330

2349

2331

self.isConfig = False

2350

self.isConfig = False

2332

2351

2333

self.nTotalBlocks = 0

2352

self.nTotalBlocks = 0

2334

2353

2335

self.data_spc = None

2354

self.data_spc = None

2336

2355

2337

self.data_cspc = None

2356

self.data_cspc = None

2338

2357

2339

self.data_dc = None

2358

self.data_dc = None

2340

2359

2341

self.fp = None

2360

self.fp = None

2342

2361

2343

self.flagIsNewFile = 1

2362

self.flagIsNewFile = 1

2344

2363

2345

self.nTotalBlocks = 0

2364

self.nTotalBlocks = 0

2346

2365

2347

self.flagIsNewBlock = 0

2366

self.flagIsNewBlock = 0

2348

2367

2349

self.setFile = None

2368

self.setFile = None

2350

2369

2351

self.dtype = None

2370

self.dtype = None

2352

2371

2353

self.path = None

2372

self.path = None

2354

2373

2355

self.noMoreFiles = 0

2374

self.noMoreFiles = 0

2356

2375

2357

self.filename = None

2376

self.filename = None

2358

2377

2359

self.basicHeaderObj = BasicHeader(LOCALTIME)

2378

self.basicHeaderObj = BasicHeader(LOCALTIME)

2360

2379

2361

self.systemHeaderObj = SystemHeader()

2380

self.systemHeaderObj = SystemHeader()

2362

2381

2363

self.radarControllerHeaderObj = RadarControllerHeader()

2382

self.radarControllerHeaderObj = RadarControllerHeader()

2364

2383

2365

self.processingHeaderObj = ProcessingHeader()

2384

self.processingHeaderObj = ProcessingHeader()

2366

2385

2367

2386

2368

def hasAllDataInBuffer(self):

2387

def hasAllDataInBuffer(self):

2369

return 1

2388

return 1

2370

2389

2371

2390

2372

def setBlockDimension(self):

2391

def setBlockDimension(self):

2373

"""

2392

"""

2374

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

2393

Obtiene las formas dimensionales del los subbloques de datos que componen un bloque

2375

2394

2376

Affected:

2395

Affected:

2377

self.shape_spc_Buffer

2396

self.shape_spc_Buffer

2378

self.shape_cspc_Buffer

2397

self.shape_cspc_Buffer

2379

self.shape_dc_Buffer

2398

self.shape_dc_Buffer

2380

2399

2381

Return: None

2400

Return: None

2382

"""

2401

"""

2383

self.shape_spc_Buffer = (self.dataOut.nChannels,

2402

self.shape_spc_Buffer = (self.dataOut.nChannels,

2384

self.processingHeaderObj.nHeights,

2403

self.processingHeaderObj.nHeights,

2385

self.processingHeaderObj.profilesPerBlock)

2404

self.processingHeaderObj.profilesPerBlock)

2386

2405

2387

self.shape_cspc_Buffer = (self.dataOut.nPairs,

2406

self.shape_cspc_Buffer = (self.dataOut.nPairs,

2388

self.processingHeaderObj.nHeights,

2407

self.processingHeaderObj.nHeights,

2389

self.processingHeaderObj.profilesPerBlock)

2408

self.processingHeaderObj.profilesPerBlock)

2390

2409

2391

self.shape_dc_Buffer = (self.dataOut.nChannels,

2410

self.shape_dc_Buffer = (self.dataOut.nChannels,

2392

self.processingHeaderObj.nHeights)

2411

self.processingHeaderObj.nHeights)

2393

2412

2394

2413

2395

def writeBlock(self):

2414

def writeBlock(self):

2396

"""

2415

"""

2397

Escribe el buffer en el file designado

2416

Escribe el buffer en el file designado

2398

2417

2399

Affected:

2418

Affected:

2400

self.data_spc

2419

self.data_spc

2401

self.data_cspc

2420

self.data_cspc

2402

self.data_dc

2421

self.data_dc

2403

self.flagIsNewFile

2422

self.flagIsNewFile

2404

self.flagIsNewBlock

2423

self.flagIsNewBlock

2405

self.nTotalBlocks

2424

self.nTotalBlocks

2406

self.nWriteBlocks

2425

self.nWriteBlocks

2407

2426

2408

Return: None

2427

Return: None

2409

"""

2428

"""

2410

2429

2411

spc = numpy.transpose( self.data_spc, (0,2,1) )

2430

spc = numpy.transpose( self.data_spc, (0,2,1) )

2412

if not( self.processingHeaderObj.shif_fft ):

2431

if not( self.processingHeaderObj.shif_fft ):

2413

spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2432

spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2414

data = spc.reshape((-1))

2433

data = spc.reshape((-1))

2415

data = data.astype(self.dtype[0])

2434

data = data.astype(self.dtype[0])

2416

data.tofile(self.fp)

2435

data.tofile(self.fp)

2417

2436

2418

if self.data_cspc != None:

2437

if self.data_cspc != None:

2419

data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )

2438

data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )

2420

cspc = numpy.transpose( self.data_cspc, (0,2,1) )

2439

cspc = numpy.transpose( self.data_cspc, (0,2,1) )

2421

if not( self.processingHeaderObj.shif_fft ):

2440

if not( self.processingHeaderObj.shif_fft ):

2422

cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2441

cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones

2423

data['real'] = cspc.real

2442

data['real'] = cspc.real

2424

data['imag'] = cspc.imag

2443

data['imag'] = cspc.imag

2425

data = data.reshape((-1))

2444

data = data.reshape((-1))

2426

data.tofile(self.fp)

2445

data.tofile(self.fp)

2427

2446

2428

if self.data_dc != None:

2447

if self.data_dc != None:

2429

data = numpy.zeros( self.shape_dc_Buffer, self.dtype )

2448

data = numpy.zeros( self.shape_dc_Buffer, self.dtype )

2430

dc = self.data_dc

2449

dc = self.data_dc

2431

data['real'] = dc.real

2450

data['real'] = dc.real

2432

data['imag'] = dc.imag

2451

data['imag'] = dc.imag

2433

data = data.reshape((-1))

2452

data = data.reshape((-1))

2434

data.tofile(self.fp)

2453

data.tofile(self.fp)

2435

2454

2436

self.data_spc.fill(0)

2455

self.data_spc.fill(0)

2437

2456

2438

if self.data_dc != None:

2457

if self.data_dc != None:

2439

self.data_dc.fill(0)

2458

self.data_dc.fill(0)

2440

2459

2441

if self.data_cspc != None:

2460

if self.data_cspc != None:

2442

self.data_cspc.fill(0)

2461

self.data_cspc.fill(0)

2443

2462

2444

self.flagIsNewFile = 0

2463

self.flagIsNewFile = 0

2445

self.flagIsNewBlock = 1

2464

self.flagIsNewBlock = 1

2446

self.nTotalBlocks += 1

2465

self.nTotalBlocks += 1

2447

self.nWriteBlocks += 1

2466

self.nWriteBlocks += 1

2448

self.blockIndex += 1

2467

self.blockIndex += 1

2449

2468

2450

2469

2451

def putData(self):

2470

def putData(self):

2452

"""

2471

"""

2453

Setea un bloque de datos y luego los escribe en un file

2472

Setea un bloque de datos y luego los escribe en un file

2454

2473

2455

Affected:

2474

Affected:

2456

self.data_spc

2475

self.data_spc

2457

self.data_cspc

2476

self.data_cspc

2458

self.data_dc

2477

self.data_dc

2459

2478

2460

Return:

2479

Return:

2461

0 : Si no hay data o no hay mas files que puedan escribirse

2480

0 : Si no hay data o no hay mas files que puedan escribirse

2462

1 : Si se escribio la data de un bloque en un file

2481

1 : Si se escribio la data de un bloque en un file

2463

"""

2482

"""

2464

2483

2465

if self.dataOut.flagNoData:

2484

if self.dataOut.flagNoData:

2466

return 0

2485

return 0

2467

2486

2468

self.flagIsNewBlock = 0

2487

self.flagIsNewBlock = 0

2469

2488

2470

if self.dataOut.flagTimeBlock:

2489

if self.dataOut.flagTimeBlock:

2471

self.data_spc.fill(0)

2490

self.data_spc.fill(0)

2472

self.data_cspc.fill(0)

2491

self.data_cspc.fill(0)

2473

self.data_dc.fill(0)

2492

self.data_dc.fill(0)

2474

self.setNextFile()

2493

self.setNextFile()

2475

2494

2476

if self.flagIsNewFile == 0:

2495

if self.flagIsNewFile == 0:

2477

self.setBasicHeader()

2496

self.setBasicHeader()

2478

2497

2479

self.data_spc = self.dataOut.data_spc.copy()

2498

self.data_spc = self.dataOut.data_spc.copy()

2480

if self.dataOut.data_cspc != None:

2499

if self.dataOut.data_cspc != None:

2481

self.data_cspc = self.dataOut.data_cspc.copy()

2500

self.data_cspc = self.dataOut.data_cspc.copy()

2482

self.data_dc = self.dataOut.data_dc.copy()

2501

self.data_dc = self.dataOut.data_dc.copy()

2483

2502

2484

# #self.processingHeaderObj.dataBlocksPerFile)

2503

# #self.processingHeaderObj.dataBlocksPerFile)

2485

if self.hasAllDataInBuffer():

2504

if self.hasAllDataInBuffer():

2486

# self.setFirstHeader()

2505

# self.setFirstHeader()

2487

self.writeNextBlock()

2506

self.writeNextBlock()

2488

2507

2489

return 1

2508

return 1

2490

2509

2491

2510

2492

def __getProcessFlags(self):

2511

def __getProcessFlags(self):

2493

2512

2494

processFlags = 0

2513

processFlags = 0

2495

2514

2496

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2515

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2497

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2516

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2498

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2517

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2499

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2518

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2500

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2519

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2501

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2520

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2502

2521

2503

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2522

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2504

2523

2505

2524

2506

2525

2507

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

2526

datatypeValueList = [PROCFLAG.DATATYPE_CHAR,

2508

PROCFLAG.DATATYPE_SHORT,

2527

PROCFLAG.DATATYPE_SHORT,

2509

PROCFLAG.DATATYPE_LONG,

2528

PROCFLAG.DATATYPE_LONG,

2510

PROCFLAG.DATATYPE_INT64,

2529

PROCFLAG.DATATYPE_INT64,

2511

PROCFLAG.DATATYPE_FLOAT,

2530

PROCFLAG.DATATYPE_FLOAT,

2512

PROCFLAG.DATATYPE_DOUBLE]

2531

PROCFLAG.DATATYPE_DOUBLE]

2513

2532

2514

2533

2515

for index in range(len(dtypeList)):

2534

for index in range(len(dtypeList)):

2516

if self.dataOut.dtype == dtypeList[index]:

2535

if self.dataOut.dtype == dtypeList[index]:

2517

dtypeValue = datatypeValueList[index]

2536

dtypeValue = datatypeValueList[index]

2518

break

2537

break

2519

2538

2520

processFlags += dtypeValue

2539

processFlags += dtypeValue

2521

2540

2522

if self.dataOut.flagDecodeData:

2541

if self.dataOut.flagDecodeData:

2523

processFlags += PROCFLAG.DECODE_DATA

2542

processFlags += PROCFLAG.DECODE_DATA

2524

2543

2525

if self.dataOut.flagDeflipData:

2544

if self.dataOut.flagDeflipData:

2526

processFlags += PROCFLAG.DEFLIP_DATA

2545

processFlags += PROCFLAG.DEFLIP_DATA

2527

2546

2528

if self.dataOut.code != None:

2547

if self.dataOut.code != None:

2529

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

2548

processFlags += PROCFLAG.DEFINE_PROCESS_CODE

2530

2549

2531

if self.dataOut.nIncohInt > 1:

2550

if self.dataOut.nIncohInt > 1:

2532

processFlags += PROCFLAG.INCOHERENT_INTEGRATION

2551

processFlags += PROCFLAG.INCOHERENT_INTEGRATION

2533

2552

2534

if self.dataOut.data_dc != None:

2553

if self.dataOut.data_dc != None:

2535

processFlags += PROCFLAG.SAVE_CHANNELS_DC

2554

processFlags += PROCFLAG.SAVE_CHANNELS_DC

2536

2555

2537

return processFlags

2556

return processFlags

2538

2557

2539

2558

2540

def __getBlockSize(self):

2559

def __getBlockSize(self):

2541

'''

2560

'''

2542

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra

2561

Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra

2543

'''

2562

'''

2544

2563

2545

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2564

dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])

2546

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2565

dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])

2547

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2566

dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])

2548

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2567

dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])

2549

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2568

dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])

2550

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2569

dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])

2551

2570

2552

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2571

dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]

2553

datatypeValueList = [1,2,4,8,4,8]

2572

datatypeValueList = [1,2,4,8,4,8]

2554

for index in range(len(dtypeList)):

2573

for index in range(len(dtypeList)):

2555

if self.dataOut.dtype == dtypeList[index]:

2574

if self.dataOut.dtype == dtypeList[index]:

2556

datatypeValue = datatypeValueList[index]

2575

datatypeValue = datatypeValueList[index]

2557

break

2576

break

2558

2577

2559

2578

2560

pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints

2579

pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints

2561

2580

2562

pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)

2581

pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)

2563

blocksize = (pts2write_SelfSpectra*datatypeValue)

2582

blocksize = (pts2write_SelfSpectra*datatypeValue)

2564

2583

2565

if self.dataOut.data_cspc != None:

2584

if self.dataOut.data_cspc != None:

2566

pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)

2585

pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)

2567

blocksize += (pts2write_CrossSpectra*datatypeValue*2)

2586

blocksize += (pts2write_CrossSpectra*datatypeValue*2)

2568

2587

2569

if self.dataOut.data_dc != None:

2588

if self.dataOut.data_dc != None:

2570

pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)

2589

pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)

2571

blocksize += (pts2write_DCchannels*datatypeValue*2)

2590

blocksize += (pts2write_DCchannels*datatypeValue*2)

2572

2591

2573

blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO

2592

blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO

2574

2593

2575

return blocksize

2594

return blocksize

2576

2595

2577

def setFirstHeader(self):

2596

def setFirstHeader(self):

2578

2597

2579

"""

2598

"""

2580

Obtiene una copia del First Header

2599

Obtiene una copia del First Header

2581

2600

2582

Affected:

2601

Affected:

2583

self.systemHeaderObj

2602

self.systemHeaderObj

2584

self.radarControllerHeaderObj

2603

self.radarControllerHeaderObj

2585

self.dtype

2604

self.dtype

2586

2605

2587

Return:

2606

Return:

2588

None

2607

None

2589

"""

2608

"""

2590

2609

2591

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

2610

self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()

2592

self.systemHeaderObj.nChannels = self.dataOut.nChannels

2611

self.systemHeaderObj.nChannels = self.dataOut.nChannels

2593

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

2612

self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()

2594

2613

2595

self.setBasicHeader()

2614

self.setBasicHeader()

2596

2615

2597

processingHeaderSize = 40 # bytes

2616

processingHeaderSize = 40 # bytes

2598

self.processingHeaderObj.dtype = 1 # Spectra

2617

self.processingHeaderObj.dtype = 1 # Spectra

2599

self.processingHeaderObj.blockSize = self.__getBlockSize()

2618

self.processingHeaderObj.blockSize = self.__getBlockSize()

2600

self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints

2619

self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints

2601

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

2620

self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile

2602

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

2621

self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows

2603

self.processingHeaderObj.processFlags = self.__getProcessFlags()

2622

self.processingHeaderObj.processFlags = self.__getProcessFlags()

2604

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval

2623

self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval

2605

self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt

2624

self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt

2606

self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels

2625

self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels

2607

self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT

2626

self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT

2608

2627

2609

if self.processingHeaderObj.totalSpectra > 0:

2628

if self.processingHeaderObj.totalSpectra > 0:

2610

channelList = []

2629

channelList = []

2611

for channel in range(self.dataOut.nChannels):

2630

for channel in range(self.dataOut.nChannels):

2612

channelList.append(channel)

2631

channelList.append(channel)

2613

channelList.append(channel)

2632

channelList.append(channel)

2614

2633

2615

pairsList = []

2634

pairsList = []

2616

if self.dataOut.nPairs > 0:

2635

if self.dataOut.nPairs > 0:

2617

for pair in self.dataOut.pairsList:

2636

for pair in self.dataOut.pairsList:

2618

pairsList.append(pair[0])

2637

pairsList.append(pair[0])

2619

pairsList.append(pair[1])

2638

pairsList.append(pair[1])

2620

2639

2621

spectraComb = channelList + pairsList

2640

spectraComb = channelList + pairsList

2622

spectraComb = numpy.array(spectraComb,dtype="u1")

2641

spectraComb = numpy.array(spectraComb,dtype="u1")

2623

self.processingHeaderObj.spectraComb = spectraComb

2642

self.processingHeaderObj.spectraComb = spectraComb

2624

sizeOfSpcComb = len(spectraComb)

2643

sizeOfSpcComb = len(spectraComb)

2625

processingHeaderSize += sizeOfSpcComb

2644

processingHeaderSize += sizeOfSpcComb

2626

2645

2627

# The processing header should not have information about code

2646

# The processing header should not have information about code

2628

# if self.dataOut.code != None:

2647

# if self.dataOut.code != None:

2629

# self.processingHeaderObj.code = self.dataOut.code

2648

# self.processingHeaderObj.code = self.dataOut.code

2630

# self.processingHeaderObj.nCode = self.dataOut.nCode

2649

# self.processingHeaderObj.nCode = self.dataOut.nCode

2631

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

2650

# self.processingHeaderObj.nBaud = self.dataOut.nBaud

2632

# nCodeSize = 4 # bytes

2651

# nCodeSize = 4 # bytes

2633

# nBaudSize = 4 # bytes

2652

# nBaudSize = 4 # bytes

2634

# codeSize = 4 # bytes

2653

# codeSize = 4 # bytes

2635

# sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)

2654

# sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)

2636

# processingHeaderSize += sizeOfCode

2655

# processingHeaderSize += sizeOfCode

2637

2656

2638

if self.processingHeaderObj.nWindows != 0:

2657

if self.processingHeaderObj.nWindows != 0:

2639

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

2658

self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]

2640

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2659

self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2641

self.processingHeaderObj.nHeights = self.dataOut.nHeights

2660

self.processingHeaderObj.nHeights = self.dataOut.nHeights

2642

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

2661

self.processingHeaderObj.samplesWin = self.dataOut.nHeights

2643

sizeOfFirstHeight = 4

2662

sizeOfFirstHeight = 4

2644

sizeOfdeltaHeight = 4

2663

sizeOfdeltaHeight = 4

2645

sizeOfnHeights = 4

2664

sizeOfnHeights = 4

2646

sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows

2665

sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows

2647

processingHeaderSize += sizeOfWindows

2666

processingHeaderSize += sizeOfWindows

2648

2667

2649

self.processingHeaderObj.size = processingHeaderSize

2668

self.processingHeaderObj.size = processingHeaderSize

2650

2669

2651

class SpectraHeisWriter(Operation):

2670

class SpectraHeisWriter(Operation):

2652

# set = None

2671

# set = None

2653

setFile = None

2672

setFile = None

2654

idblock = None

2673

idblock = None

2655

doypath = None

2674

doypath = None

2656

subfolder = None

2675

subfolder = None

2657

2676

2658

def __init__(self):

2677

def __init__(self):

2659

self.wrObj = FITS()

2678

self.wrObj = FITS()

2660

# self.dataOut = dataOut

2679

# self.dataOut = dataOut

2661

self.nTotalBlocks=0

2680

self.nTotalBlocks=0

2662

# self.set = None

2681

# self.set = None

2663

self.setFile = None

2682

self.setFile = None

2664

self.idblock = 0

2683

self.idblock = 0

2665

self.wrpath = None

2684

self.wrpath = None

2666

self.doypath = None

2685

self.doypath = None

2667

self.subfolder = None

2686

self.subfolder = None

2668

self.isConfig = False

2687

self.isConfig = False

2669

2688

2670

def isNumber(str):

2689

def isNumber(str):

2671

"""

2690

"""

2672

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

2691

Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

2673

2692

2674

Excepciones:

2693

Excepciones:

2675

Si un determinado string no puede ser convertido a numero

2694

Si un determinado string no puede ser convertido a numero

2676

Input:

2695

Input:

2677

str, string al cual se le analiza para determinar si convertible a un numero o no

2696

str, string al cual se le analiza para determinar si convertible a un numero o no

2678

2697

2679

Return:

2698

Return:

2680

True : si el string es uno numerico

2699

True : si el string es uno numerico

2681

False : no es un string numerico

2700

False : no es un string numerico

2682

"""

2701

"""

2683

try:

2702

try:

2684

float( str )

2703

float( str )

2685

return True

2704

return True

2686

except:

2705

except:

2687

return False

2706

return False

2688

2707

2689

def setup(self, dataOut, wrpath):

2708

def setup(self, dataOut, wrpath):

2690

2709

2691

if not(os.path.exists(wrpath)):

2710

if not(os.path.exists(wrpath)):

2692

os.mkdir(wrpath)

2711

os.mkdir(wrpath)

2693

2712

2694

self.wrpath = wrpath

2713

self.wrpath = wrpath

2695

# self.setFile = 0

2714

# self.setFile = 0

2696

self.dataOut = dataOut

2715

self.dataOut = dataOut

2697

2716

2698

def putData(self):

2717

def putData(self):

2699

name= time.localtime( self.dataOut.utctime)

2718

name= time.localtime( self.dataOut.utctime)

2700

ext=".fits"

2719

ext=".fits"

2701

2720

2702

if self.doypath == None:

2721

if self.doypath == None:

2703

self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))

2722

self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))

2704

self.doypath = os.path.join( self.wrpath, self.subfolder )

2723

self.doypath = os.path.join( self.wrpath, self.subfolder )

2705

os.mkdir(self.doypath)

2724

os.mkdir(self.doypath)

2706

2725

2707

if self.setFile == None:

2726

if self.setFile == None:

2708

# self.set = self.dataOut.set

2727

# self.set = self.dataOut.set

2709

self.setFile = 0

2728

self.setFile = 0

2710

# if self.set != self.dataOut.set:

2729

# if self.set != self.dataOut.set:

2711

## self.set = self.dataOut.set

2730

## self.set = self.dataOut.set

2712

# self.setFile = 0

2731

# self.setFile = 0

2713

2732

2714

#make the filename

2733

#make the filename

2715

file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)

2734

file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)

2716

2735

2717

filename = os.path.join(self.wrpath,self.subfolder, file)

2736

filename = os.path.join(self.wrpath,self.subfolder, file)

2718

2737

2719

idblock = numpy.array([self.idblock],dtype="int64")

2738

idblock = numpy.array([self.idblock],dtype="int64")

2720

header=self.wrObj.cFImage(idblock=idblock,

2739

header=self.wrObj.cFImage(idblock=idblock,

2721

year=time.gmtime(self.dataOut.utctime).tm_year,

2740

year=time.gmtime(self.dataOut.utctime).tm_year,

2722

month=time.gmtime(self.dataOut.utctime).tm_mon,

2741

month=time.gmtime(self.dataOut.utctime).tm_mon,

2723

day=time.gmtime(self.dataOut.utctime).tm_mday,

2742

day=time.gmtime(self.dataOut.utctime).tm_mday,

2724

hour=time.gmtime(self.dataOut.utctime).tm_hour,

2743

hour=time.gmtime(self.dataOut.utctime).tm_hour,

2725

minute=time.gmtime(self.dataOut.utctime).tm_min,

2744

minute=time.gmtime(self.dataOut.utctime).tm_min,

2726

second=time.gmtime(self.dataOut.utctime).tm_sec)

2745

second=time.gmtime(self.dataOut.utctime).tm_sec)

2727

2746

2728

c=3E8

2747

c=3E8

2729

deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2748

deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]

2730

freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))

2749

freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))

2731

2750

2732

colList = []

2751

colList = []

2733

2752

2734

colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)

2753

colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)

2735

2754

2736

colList.append(colFreq)

2755

colList.append(colFreq)

2737

2756

2738

nchannel=self.dataOut.nChannels

2757

nchannel=self.dataOut.nChannels

2739

2758

2740

for i in range(nchannel):

2759

for i in range(nchannel):

2741

col = self.wrObj.writeData(name="PCh"+str(i+1),

2760

col = self.wrObj.writeData(name="PCh"+str(i+1),

2742

format=str(self.dataOut.nFFTPoints)+'E',

2761

format=str(self.dataOut.nFFTPoints)+'E',

2743

data=10*numpy.log10(self.dataOut.data_spc[i,:]))

2762

data=10*numpy.log10(self.dataOut.data_spc[i,:]))

2744

2763

2745

colList.append(col)

2764

colList.append(col)

2746

2765

2747

data=self.wrObj.Ctable(colList=colList)

2766

data=self.wrObj.Ctable(colList=colList)

2748

2767

2749

self.wrObj.CFile(header,data)

2768

self.wrObj.CFile(header,data)

2750

2769

2751

self.wrObj.wFile(filename)

2770

self.wrObj.wFile(filename)

2752

2771

2753

#update the setFile

2772

#update the setFile

2754

self.setFile += 1

2773

self.setFile += 1

2755

self.idblock += 1

2774

self.idblock += 1

2756

2775

2757

return 1

2776

return 1

2758

2777

2759

def run(self, dataOut, **kwargs):

2778

def run(self, dataOut, **kwargs):

2760

2779

2761

if not(self.isConfig):

2780

if not(self.isConfig):

2762

2781

2763

self.setup(dataOut, **kwargs)

2782

self.setup(dataOut, **kwargs)

2764

self.isConfig = True

2783

self.isConfig = True

2765

2784

2766

self.putData()

2785

self.putData()

2767

2786

2768

2787

2769

2788

2770

class ParameterConf:

2789

class ParameterConf:

2771

ELEMENTNAME = 'Parameter'

2790

ELEMENTNAME = 'Parameter'

2772

def __init__(self):

2791

def __init__(self):

2773

self.name = ''

2792

self.name = ''

2774

self.value = ''

2793

self.value = ''

2775

2794

2776

def readXml(self, parmElement):

2795

def readXml(self, parmElement):

2777

self.name = parmElement.get('name')

2796

self.name = parmElement.get('name')

2778

self.value = parmElement.get('value')

2797

self.value = parmElement.get('value')

2779

2798

2780

def getElementName(self):

2799

def getElementName(self):

2781

return self.ELEMENTNAME

2800

return self.ELEMENTNAME

2782

2801

2783

class Metadata:

2802

class Metadata:

2784

2803

2785

def __init__(self, filename):

2804

def __init__(self, filename):

2786

self.parmConfObjList = []

2805

self.parmConfObjList = []

2787

self.readXml(filename)

2806

self.readXml(filename)

2788

2807

2789

def readXml(self, filename):

2808

def readXml(self, filename):

2790

self.projectElement = None

2809

self.projectElement = None

2791

self.procUnitConfObjDict = {}

2810

self.procUnitConfObjDict = {}

2792

self.projectElement = ElementTree().parse(filename)

2811

self.projectElement = ElementTree().parse(filename)

2793

self.project = self.projectElement.tag

2812

self.project = self.projectElement.tag

2794

2813

2795

parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())

2814

parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())

2796

2815

2797

for parmElement in parmElementList:

2816

for parmElement in parmElementList:

2798

parmConfObj = ParameterConf()

2817

parmConfObj = ParameterConf()

2799

parmConfObj.readXml(parmElement)

2818

parmConfObj.readXml(parmElement)

2800

self.parmConfObjList.append(parmConfObj)

2819

self.parmConfObjList.append(parmConfObj)

2801

2820

2802

class FitsWriter(Operation):

2821

class FitsWriter(Operation):

2803

2822

2804

def __init__(self):

2823

def __init__(self):

2805

self.isConfig = False

2824

self.isConfig = False

2806

self.dataBlocksPerFile = None

2825

self.dataBlocksPerFile = None

2807

self.blockIndex = 0

2826

self.blockIndex = 0

2808

self.flagIsNewFile = 1

2827

self.flagIsNewFile = 1

2809

self.fitsObj = None

2828

self.fitsObj = None

2810

self.optchar = 'P'

2829

self.optchar = 'P'

2811

self.ext = '.fits'

2830

self.ext = '.fits'

2812

self.setFile = 0

2831

self.setFile = 0

2813

2832

2814

def setFitsHeader(self, dataOut, metadatafile):

2833

def setFitsHeader(self, dataOut, metadatafile):

2815

2834

2816

header_data = pyfits.PrimaryHDU()

2835

header_data = pyfits.PrimaryHDU()

2817

2836

2818

metadata4fits = Metadata(metadatafile)

2837

metadata4fits = Metadata(metadatafile)

2819

for parameter in metadata4fits.parmConfObjList:

2838

for parameter in metadata4fits.parmConfObjList:

2820

parm_name = parameter.name

2839

parm_name = parameter.name

2821

parm_value = parameter.value

2840

parm_value = parameter.value

2822

2841

2823

# if parm_value == 'fromdatadatetime':

2842

# if parm_value == 'fromdatadatetime':

2824

# value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())

2843

# value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())

2825

# elif parm_value == 'fromdataheights':

2844

# elif parm_value == 'fromdataheights':

2826

# value = dataOut.nHeights

2845

# value = dataOut.nHeights

2827

# elif parm_value == 'fromdatachannel':

2846

# elif parm_value == 'fromdatachannel':

2828

# value = dataOut.nChannels

2847

# value = dataOut.nChannels

2829

# elif parm_value == 'fromdatasamples':

2848

# elif parm_value == 'fromdatasamples':

2830

# value = dataOut.nFFTPoints

2849

# value = dataOut.nFFTPoints

2831

# else:

2850

# else:

2832

# value = parm_value

2851

# value = parm_value

2833

2852

2834

header_data.header[parm_name] = parm_value

2853

header_data.header[parm_name] = parm_value

2835

2854

2836

2855

2837

header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())

2856

header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())

2838

header_data.header['CHANNELLIST'] = str(dataOut.channelList)

2857

header_data.header['CHANNELLIST'] = str(dataOut.channelList)

2839

header_data.header['NCHANNELS'] = dataOut.nChannels

2858

header_data.header['NCHANNELS'] = dataOut.nChannels

2840

#header_data.header['HEIGHTS'] = dataOut.heightList

2859

#header_data.header['HEIGHTS'] = dataOut.heightList

2841

header_data.header['NHEIGHTS'] = dataOut.nHeights

2860

header_data.header['NHEIGHTS'] = dataOut.nHeights

2842

2861

2843

header_data.header['IPPSECONDS'] = dataOut.ippSeconds

2862

header_data.header['IPPSECONDS'] = dataOut.ippSeconds

2844

header_data.header['NCOHINT'] = dataOut.nCohInt

2863

header_data.header['NCOHINT'] = dataOut.nCohInt

2845

header_data.header['NINCOHINT'] = dataOut.nIncohInt

2864

header_data.header['NINCOHINT'] = dataOut.nIncohInt

2846

header_data.header['TIMEZONE'] = dataOut.timeZone

2865

header_data.header['TIMEZONE'] = dataOut.timeZone

2847

header_data.header['NBLOCK'] = self.blockIndex

2866

header_data.header['NBLOCK'] = self.blockIndex

2848

2867

2849

header_data.writeto(self.filename)

2868

header_data.writeto(self.filename)

2850

2869

2851

self.addExtension(dataOut.heightList,'HEIGHTLIST')

2870

self.addExtension(dataOut.heightList,'HEIGHTLIST')

2852

2871

2853

2872

2854

def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):

2873

def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):

2855

2874

2856

self.path = path

2875

self.path = path

2857

self.dataOut = dataOut

2876

self.dataOut = dataOut

2858

self.metadatafile = metadatafile

2877

self.metadatafile = metadatafile

2859

self.dataBlocksPerFile = dataBlocksPerFile

2878

self.dataBlocksPerFile = dataBlocksPerFile

2860

2879

2861

def open(self):

2880

def open(self):

2862

self.fitsObj = pyfits.open(self.filename, mode='update')

2881

self.fitsObj = pyfits.open(self.filename, mode='update')

2863

2882

2864

2883

2865

def addExtension(self, data, tagname):

2884

def addExtension(self, data, tagname):

2866

self.open()

2885

self.open()

2867

extension = pyfits.ImageHDU(data=data, name=tagname)

2886

extension = pyfits.ImageHDU(data=data, name=tagname)

2868

#extension.header['TAG'] = tagname

2887

#extension.header['TAG'] = tagname

2869

self.fitsObj.append(extension)

2888

self.fitsObj.append(extension)

2870

self.write()

2889

self.write()

2871

2890

2872

def addData(self, data):

2891

def addData(self, data):

2873

self.open()

2892

self.open()

2874

extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])

2893

extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])

2875

extension.header['UTCTIME'] = self.dataOut.utctime

2894

extension.header['UTCTIME'] = self.dataOut.utctime

2876

self.fitsObj.append(extension)

2895

self.fitsObj.append(extension)

2877

self.blockIndex += 1

2896

self.blockIndex += 1

2878

self.fitsObj[0].header['NBLOCK'] = self.blockIndex

2897

self.fitsObj[0].header['NBLOCK'] = self.blockIndex

2879

2898

2880

self.write()

2899

self.write()

2881

2900

2882

def write(self):

2901

def write(self):

2883

2902

2884

self.fitsObj.flush(verbose=True)

2903

self.fitsObj.flush(verbose=True)

2885

self.fitsObj.close()

2904

self.fitsObj.close()

2886

2905

2887

2906

2888

def setNextFile(self):

2907

def setNextFile(self):

2889

2908

2890

ext = self.ext

2909

ext = self.ext

2891

path = self.path

2910

path = self.path

2892

2911

2893

timeTuple = time.localtime( self.dataOut.utctime)

2912

timeTuple = time.localtime( self.dataOut.utctime)

2894

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

2913

subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

2895

2914

2896

fullpath = os.path.join( path, subfolder )

2915

fullpath = os.path.join( path, subfolder )

2897

if not( os.path.exists(fullpath) ):

2916

if not( os.path.exists(fullpath) ):

2898

os.mkdir(fullpath)

2917

os.mkdir(fullpath)

2899

self.setFile = -1 #inicializo mi contador de seteo

2918

self.setFile = -1 #inicializo mi contador de seteo

2900

else:

2919

else:

2901

filesList = os.listdir( fullpath )

2920

filesList = os.listdir( fullpath )

2902

if len( filesList ) > 0:

2921

if len( filesList ) > 0:

2903

filesList = sorted( filesList, key=str.lower )

2922

filesList = sorted( filesList, key=str.lower )

2904

filen = filesList[-1]

2923

filen = filesList[-1]

2905

2924

2906

if isNumber( filen[8:11] ):

2925

if isNumber( filen[8:11] ):

2907

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

2926

self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file

2908

else:

2927

else:

2909

self.setFile = -1

2928

self.setFile = -1

2910

else:

2929

else:

2911

self.setFile = -1 #inicializo mi contador de seteo

2930

self.setFile = -1 #inicializo mi contador de seteo

2912

2931

2913

setFile = self.setFile

2932

setFile = self.setFile

2914

setFile += 1

2933

setFile += 1

2915

2934

2916

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

2935

file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,

2917

timeTuple.tm_year,

2936

timeTuple.tm_year,

2918

timeTuple.tm_yday,

2937

timeTuple.tm_yday,

2919

setFile,

2938

setFile,

2920

ext )

2939

ext )

2921

2940

2922

filename = os.path.join( path, subfolder, file )

2941

filename = os.path.join( path, subfolder, file )

2923

2942

2924

self.blockIndex = 0

2943

self.blockIndex = 0

2925

self.filename = filename

2944

self.filename = filename

2926

self.setFile = setFile

2945

self.setFile = setFile

2927

self.flagIsNewFile = 1

2946

self.flagIsNewFile = 1

2928

2947

2929

print 'Writing the file: %s'%self.filename

2948

print 'Writing the file: %s'%self.filename

2930

2949

2931

self.setFitsHeader(self.dataOut, self.metadatafile)

2950

self.setFitsHeader(self.dataOut, self.metadatafile)

2932

2951

2933

return 1

2952

return 1

2934

2953

2935

def writeBlock(self):

2954

def writeBlock(self):

2936

self.addData(self.dataOut.data_spc)

2955

self.addData(self.dataOut.data_spc)

2937

self.flagIsNewFile = 0

2956

self.flagIsNewFile = 0

2938

2957

2939

2958

2940

def __setNewBlock(self):

2959

def __setNewBlock(self):

2941

2960

2942

if self.flagIsNewFile:

2961

if self.flagIsNewFile:

2943

return 1

2962

return 1

2944

2963

2945

if self.blockIndex < self.dataBlocksPerFile:

2964

if self.blockIndex < self.dataBlocksPerFile:

2946

return 1

2965

return 1

2947

2966

2948

if not( self.setNextFile() ):

2967

if not( self.setNextFile() ):

2949

return 0

2968

return 0

2950

2969

2951

return 1

2970

return 1

2952

2971

2953

def writeNextBlock(self):

2972

def writeNextBlock(self):

2954

if not( self.__setNewBlock() ):

2973

if not( self.__setNewBlock() ):

2955

return 0

2974

return 0

2956

self.writeBlock()

2975

self.writeBlock()

2957

return 1

2976

return 1

2958

2977

2959

def putData(self):

2978

def putData(self):

2960

if self.flagIsNewFile:

2979

if self.flagIsNewFile:

2961

self.setNextFile()

2980

self.setNextFile()

2962

self.writeNextBlock()

2981

self.writeNextBlock()

2963

2982

2964

def run(self, dataOut, **kwargs):

2983

def run(self, dataOut, **kwargs):

2965

if not(self.isConfig):

2984

if not(self.isConfig):

2966

self.setup(dataOut, **kwargs)

2985

self.setup(dataOut, **kwargs)

2967

self.isConfig = True

2986

self.isConfig = True

2968

self.putData()

2987

self.putData()

2969

2988

2970

2989

2971

class FitsReader(ProcessingUnit):

2990

class FitsReader(ProcessingUnit):

2972

2991

2973

# __TIMEZONE = time.timezone

2992

# __TIMEZONE = time.timezone

2974

2993

2975

expName = None

2994

expName = None

2976

datetimestr = None

2995

datetimestr = None

2977

utc = None

2996

utc = None

2978

nChannels = None

2997

nChannels = None

2979

nSamples = None

2998

nSamples = None

2980

dataBlocksPerFile = None

2999

dataBlocksPerFile = None

2981

comments = None

3000

comments = None

2982

lastUTTime = None

3001

lastUTTime = None

2983

header_dict = None

3002

header_dict = None

2984

data = None

3003

data = None

2985

data_header_dict = None

3004

data_header_dict = None

2986

3005

2987

def __init__(self):

3006

def __init__(self):

2988

self.isConfig = False

3007

self.isConfig = False

2989

self.ext = '.fits'

3008

self.ext = '.fits'

2990

self.setFile = 0

3009

self.setFile = 0

2991

self.flagNoMoreFiles = 0

3010

self.flagNoMoreFiles = 0

2992

self.flagIsNewFile = 1

3011

self.flagIsNewFile = 1

2993

self.flagTimeBlock = None

3012

self.flagTimeBlock = None

2994

self.fileIndex = None

3013

self.fileIndex = None

2995

self.filename = None

3014

self.filename = None

2996

self.fileSize = None

3015

self.fileSize = None

2997

self.fitsObj = None

3016

self.fitsObj = None

2998

self.timeZone = None

3017

self.timeZone = None

2999

self.nReadBlocks = 0

3018

self.nReadBlocks = 0

3000

self.nTotalBlocks = 0

3019

self.nTotalBlocks = 0

3001

self.dataOut = self.createObjByDefault()

3020

self.dataOut = self.createObjByDefault()

3002

self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()

3021

self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()

3003

self.blockIndex = 1

3022

self.blockIndex = 1

3004

3023

3005

def createObjByDefault(self):

3024

def createObjByDefault(self):

3006

3025

3007

dataObj = Fits()

3026

dataObj = Fits()

3008

3027

3009

return dataObj

3028

return dataObj

3010

3029

3011

def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):

3030

def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):

3012

try:

3031

try:

3013

fitsObj = pyfits.open(filename,'readonly')

3032

fitsObj = pyfits.open(filename,'readonly')

3014

except:

3033

except:

3015

raise IOError, "The file %s can't be opened" %(filename)

3034

raise IOError, "The file %s can't be opened" %(filename)

3016

3035

3017

header = fitsObj[0].header

3036

header = fitsObj[0].header

3018

struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")

3037

struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")

3019

utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS

3038

utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS

3020

3039

3021

ltc = utc

3040

ltc = utc

3022

if useLocalTime:

3041

if useLocalTime:

3023

ltc -= time.timezone

3042

ltc -= time.timezone

3024

thisDatetime = datetime.datetime.utcfromtimestamp(ltc)

3043

thisDatetime = datetime.datetime.utcfromtimestamp(ltc)

3025

thisTime = thisDatetime.time()

3044

thisTime = thisDatetime.time()

3026

3045

3027

if not ((startTime <= thisTime) and (endTime > thisTime)):

3046

if not ((startTime <= thisTime) and (endTime > thisTime)):

3028

return None

3047

return None

3029

3048

3030

return thisDatetime

3049

return thisDatetime

3031

3050

3032

def __setNextFileOnline(self):

3051

def __setNextFileOnline(self):

3033

raise ValueError, "No implemented"

3052

raise ValueError, "No implemented"

3034

3053

3035

def __setNextFileOffline(self):

3054

def __setNextFileOffline(self):

3036

idFile = self.fileIndex

3055

idFile = self.fileIndex

3037

3056

3038

while (True):

3057

while (True):

3039

idFile += 1

3058

idFile += 1

3040

if not(idFile < len(self.filenameList)):

3059

if not(idFile < len(self.filenameList)):

3041

self.flagNoMoreFiles = 1

3060

self.flagNoMoreFiles = 1

3042

print "No more Files"

3061

print "No more Files"

3043

return 0

3062

return 0

3044

3063

3045

filename = self.filenameList[idFile]

3064

filename = self.filenameList[idFile]

3046

3065

3047

# if not(self.__verifyFile(filename)):

3066

# if not(self.__verifyFile(filename)):

3048

# continue

3067

# continue

3049

3068

3050

fileSize = os.path.getsize(filename)

3069

fileSize = os.path.getsize(filename)

3051

fitsObj = pyfits.open(filename,'readonly')

3070

fitsObj = pyfits.open(filename,'readonly')

3052

break

3071

break

3053

3072

3054

self.flagIsNewFile = 1

3073

self.flagIsNewFile = 1

3055

self.fileIndex = idFile

3074

self.fileIndex = idFile

3056

self.filename = filename

3075

self.filename = filename

3057

self.fileSize = fileSize

3076

self.fileSize = fileSize

3058

self.fitsObj = fitsObj

3077

self.fitsObj = fitsObj

3059

self.blockIndex = 0

3078

self.blockIndex = 0

3060

print "Setting the file: %s"%self.filename

3079

print "Setting the file: %s"%self.filename

3061

3080

3062

return 1

3081

return 1

3063

3082

3064

def readHeader(self):

3083

def readHeader(self):

3065

headerObj = self.fitsObj[0]

3084

headerObj = self.fitsObj[0]

3066

3085

3067

self.header_dict = headerObj.header

3086

self.header_dict = headerObj.header

3068

if 'EXPNAME' in headerObj.header.keys():

3087

if 'EXPNAME' in headerObj.header.keys():

3069

self.expName = headerObj.header['EXPNAME']

3088

self.expName = headerObj.header['EXPNAME']

3070

3089

3071

if 'DATATYPE' in headerObj.header.keys():

3090

if 'DATATYPE' in headerObj.header.keys():

3072

self.dataType = headerObj.header['DATATYPE']

3091

self.dataType = headerObj.header['DATATYPE']

3073

3092

3074

self.datetimestr = headerObj.header['DATETIME']

3093

self.datetimestr = headerObj.header['DATETIME']

3075

channelList = headerObj.header['CHANNELLIST']

3094

channelList = headerObj.header['CHANNELLIST']

3076

channelList = channelList.split('[')

3095

channelList = channelList.split('[')

3077

channelList = channelList[1].split(']')

3096

channelList = channelList[1].split(']')

3078

channelList = channelList[0].split(',')

3097

channelList = channelList[0].split(',')

3079

channelList = [int(ch) for ch in channelList]

3098

channelList = [int(ch) for ch in channelList]

3080

self.channelList = channelList

3099

self.channelList = channelList

3081

self.nChannels = headerObj.header['NCHANNELS']

3100

self.nChannels = headerObj.header['NCHANNELS']

3082

self.nHeights = headerObj.header['NHEIGHTS']

3101

self.nHeights = headerObj.header['NHEIGHTS']

3083

self.ippSeconds = headerObj.header['IPPSECONDS']

3102

self.ippSeconds = headerObj.header['IPPSECONDS']

3084

self.nCohInt = headerObj.header['NCOHINT']

3103

self.nCohInt = headerObj.header['NCOHINT']

3085

self.nIncohInt = headerObj.header['NINCOHINT']

3104

self.nIncohInt = headerObj.header['NINCOHINT']

3086

self.dataBlocksPerFile = headerObj.header['NBLOCK']

3105

self.dataBlocksPerFile = headerObj.header['NBLOCK']

3087

self.timeZone = headerObj.header['TIMEZONE']

3106

self.timeZone = headerObj.header['TIMEZONE']

3088

3107

3089

self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt

3108

self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt

3090

3109

3091

if 'COMMENT' in headerObj.header.keys():

3110

if 'COMMENT' in headerObj.header.keys():

3092

self.comments = headerObj.header['COMMENT']

3111

self.comments = headerObj.header['COMMENT']

3093

3112

3094

self.readHeightList()

3113

self.readHeightList()

3095

3114

3096

def readHeightList(self):

3115

def readHeightList(self):

3097

self.blockIndex = self.blockIndex + 1

3116

self.blockIndex = self.blockIndex + 1

3098

obj = self.fitsObj[self.blockIndex]

3117

obj = self.fitsObj[self.blockIndex]

3099

self.heightList = obj.data

3118

self.heightList = obj.data

3100

self.blockIndex = self.blockIndex + 1

3119

self.blockIndex = self.blockIndex + 1

3101

3120

3102

def readExtension(self):

3121

def readExtension(self):

3103

obj = self.fitsObj[self.blockIndex]

3122

obj = self.fitsObj[self.blockIndex]

3104

self.heightList = obj.data

3123

self.heightList = obj.data

3105

self.blockIndex = self.blockIndex + 1

3124

self.blockIndex = self.blockIndex + 1

3106

3125

3107

def setNextFile(self):

3126

def setNextFile(self):

3108

3127

3109

if self.online:

3128

if self.online:

3110

newFile = self.__setNextFileOnline()

3129

newFile = self.__setNextFileOnline()

3111

else:

3130

else:

3112

newFile = self.__setNextFileOffline()

3131

newFile = self.__setNextFileOffline()

3113

3132

3114

if not(newFile):

3133

if not(newFile):

3115

return 0

3134

return 0

3116

3135

3117

self.readHeader()

3136

self.readHeader()

3118

3137

3119

self.nReadBlocks = 0

3138

self.nReadBlocks = 0

3120

# self.blockIndex = 1

3139

# self.blockIndex = 1

3121

return 1

3140

return 1

3122

3141

3123

def __searchFilesOffLine(self,

3142

def __searchFilesOffLine(self,

3124

path,

3143

path,

3125

startDate,

3144

startDate,

3126

endDate,

3145

endDate,

3127

startTime=datetime.time(0,0,0),

3146

startTime=datetime.time(0,0,0),

3128

endTime=datetime.time(23,59,59),

3147

endTime=datetime.time(23,59,59),

3129

set=None,

3148

set=None,

3130

expLabel='',

3149

expLabel='',

3131

ext='.fits',

3150

ext='.fits',

3132

walk=True):

3151

walk=True):

3133

3152

3134

pathList = []

3153

pathList = []

3135

3154

3136

if not walk:

3155

if not walk:

3137

pathList.append(path)

3156

pathList.append(path)

3138

3157

3139

else:

3158

else:

3140

dirList = []

3159

dirList = []

3141

for thisPath in os.listdir(path):

3160

for thisPath in os.listdir(path):

3142

if not os.path.isdir(os.path.join(path,thisPath)):

3161

if not os.path.isdir(os.path.join(path,thisPath)):

3143

continue

3162

continue

3144

if not isDoyFolder(thisPath):

3163

if not isDoyFolder(thisPath):

3145

continue

3164

continue

3146

3165

3147

dirList.append(thisPath)

3166

dirList.append(thisPath)

3148

3167

3149

if not(dirList):

3168

if not(dirList):

3150

return None, None

3169

return None, None

3151

3170

3152

thisDate = startDate

3171

thisDate = startDate

3153

3172

3154

while(thisDate <= endDate):

3173

while(thisDate <= endDate):

3155

year = thisDate.timetuple().tm_year

3174

year = thisDate.timetuple().tm_year

3156

doy = thisDate.timetuple().tm_yday

3175

doy = thisDate.timetuple().tm_yday

3157

3176

3158

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

3177

matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')

3159

if len(matchlist) == 0:

3178

if len(matchlist) == 0:

3160

thisDate += datetime.timedelta(1)

3179

thisDate += datetime.timedelta(1)

3161

continue

3180

continue

3162

for match in matchlist:

3181

for match in matchlist:

3163

pathList.append(os.path.join(path,match,expLabel))

3182

pathList.append(os.path.join(path,match,expLabel))

3164

3183

3165

thisDate += datetime.timedelta(1)

3184

thisDate += datetime.timedelta(1)

3166

3185

3167

if pathList == []:

3186

if pathList == []:

3168

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

3187

print "Any folder was found for the date range: %s-%s" %(startDate, endDate)

3169

return None, None

3188

return None, None

3170

3189

3171

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

3190

print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)

3172

3191

3173

filenameList = []

3192

filenameList = []

3174

datetimeList = []

3193

datetimeList = []

3175

3194

3176

for i in range(len(pathList)):

3195

for i in range(len(pathList)):

3177

3196

3178

thisPath = pathList[i]

3197

thisPath = pathList[i]

3179

3198

3180

fileList = glob.glob1(thisPath, "*%s" %ext)

3199

fileList = glob.glob1(thisPath, "*%s" %ext)

3181

fileList.sort()

3200

fileList.sort()

3182

3201

3183

for file in fileList:

3202

for file in fileList:

3184

3203

3185

filename = os.path.join(thisPath,file)

3204

filename = os.path.join(thisPath,file)

3186

thisDatetime = self.isFileinThisTime(filename, startTime, endTime)

3205

thisDatetime = self.isFileinThisTime(filename, startTime, endTime)

3187

3206

3188

if not(thisDatetime):

3207

if not(thisDatetime):

3189

continue

3208

continue

3190

3209

3191

filenameList.append(filename)

3210

filenameList.append(filename)

3192

datetimeList.append(thisDatetime)

3211

datetimeList.append(thisDatetime)

3193

3212

3194

if not(filenameList):

3213

if not(filenameList):

3195

print "Any file was found for the time range %s - %s" %(startTime, endTime)

3214

print "Any file was found for the time range %s - %s" %(startTime, endTime)

3196

return None, None

3215

return None, None

3197

3216

3198

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

3217

print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)

3199

print

3218

print

3200

3219

3201

for i in range(len(filenameList)):

3220

for i in range(len(filenameList)):

3202

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

3221

print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

3203

3222

3204

self.filenameList = filenameList

3223

self.filenameList = filenameList

3205

self.datetimeList = datetimeList

3224

self.datetimeList = datetimeList

3206

3225

3207

return pathList, filenameList

3226

return pathList, filenameList

3208

3227

3209

def setup(self, path=None,

3228

def setup(self, path=None,

3210

startDate=None,

3229

startDate=None,

3211

endDate=None,

3230

endDate=None,

3212

startTime=datetime.time(0,0,0),

3231

startTime=datetime.time(0,0,0),

3213

endTime=datetime.time(23,59,59),

3232

endTime=datetime.time(23,59,59),

3214

set=0,

3233

set=0,

3215

expLabel = "",

3234

expLabel = "",

3216

ext = None,

3235

ext = None,

3217

online = False,

3236

online = False,

3218

delay = 60,

3237

delay = 60,

3219

walk = True):

3238

walk = True):

3220

3239

3221

if path == None:

3240

if path == None:

3222

raise ValueError, "The path is not valid"

3241

raise ValueError, "The path is not valid"

3223

3242

3224

if ext == None:

3243

if ext == None:

3225

ext = self.ext

3244

ext = self.ext

3226

3245

3227

if not(online):

3246

if not(online):

3228

print "Searching files in offline mode ..."

3247

print "Searching files in offline mode ..."

3229

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

3248

pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,

3230

startTime=startTime, endTime=endTime,

3249

startTime=startTime, endTime=endTime,

3231

set=set, expLabel=expLabel, ext=ext,

3250

set=set, expLabel=expLabel, ext=ext,

3232

walk=walk)

3251

walk=walk)

3233

3252

3234

if not(pathList):

3253

if not(pathList):

3235

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

3254

print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,

3236

datetime.datetime.combine(startDate,startTime).ctime(),

3255

datetime.datetime.combine(startDate,startTime).ctime(),

3237

datetime.datetime.combine(endDate,endTime).ctime())

3256

datetime.datetime.combine(endDate,endTime).ctime())

3238

3257

3239

sys.exit(-1)

3258

sys.exit(-1)

3240

3259

3241

self.fileIndex = -1

3260

self.fileIndex = -1

3242

self.pathList = pathList

3261

self.pathList = pathList

3243

self.filenameList = filenameList

3262

self.filenameList = filenameList

3244

3263

3245

self.online = online

3264

self.online = online

3246

self.delay = delay

3265

self.delay = delay

3247

ext = ext.lower()

3266

ext = ext.lower()

3248

self.ext = ext

3267

self.ext = ext

3249

3268

3250

if not(self.setNextFile()):

3269

if not(self.setNextFile()):

3251

if (startDate!=None) and (endDate!=None):

3270

if (startDate!=None) and (endDate!=None):

3252

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

3271

print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())

3253

elif startDate != None:

3272

elif startDate != None:

3254

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

3273

print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())

3255

else:

3274

else:

3256

print "No files"

3275

print "No files"

3257

3276

3258

sys.exit(-1)

3277

sys.exit(-1)

3259

3278

3260

3279

3261

3280

3262

def readBlock(self):

3281

def readBlock(self):

3263

dataObj = self.fitsObj[self.blockIndex]

3282

dataObj = self.fitsObj[self.blockIndex]

3264

3283

3265

self.data = dataObj.data

3284

self.data = dataObj.data

3266

self.data_header_dict = dataObj.header

3285

self.data_header_dict = dataObj.header

3267

self.utc = self.data_header_dict['UTCTIME']

3286

self.utc = self.data_header_dict['UTCTIME']

3268

3287

3269

self.flagIsNewFile = 0

3288

self.flagIsNewFile = 0

3270

self.blockIndex += 1

3289

self.blockIndex += 1

3271

self.nTotalBlocks += 1

3290

self.nTotalBlocks += 1

3272

self.nReadBlocks += 1

3291

self.nReadBlocks += 1

3273

3292

3274

return 1

3293

return 1

3275

3294

3276

def __jumpToLastBlock(self):

3295

def __jumpToLastBlock(self):

3277

raise ValueError, "No implemented"

3296

raise ValueError, "No implemented"

3278

3297

3279

def __waitNewBlock(self):

3298

def __waitNewBlock(self):

3280

"""

3299

"""

3281

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

3300

Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.

3282

3301

3283

Si el modo de lectura es OffLine siempre retorn 0

3302

Si el modo de lectura es OffLine siempre retorn 0

3284

"""

3303

"""

3285

if not self.online:

3304

if not self.online:

3286

return 0

3305

return 0

3287

3306

3288

if (self.nReadBlocks >= self.dataBlocksPerFile):

3307

if (self.nReadBlocks >= self.dataBlocksPerFile):

3289

return 0

3308

return 0

3290

3309

3291

currentPointer = self.fp.tell()

3310

currentPointer = self.fp.tell()

3292

3311

3293

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

3312

neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize

3294

3313

3295

for nTries in range( self.nTries ):

3314

for nTries in range( self.nTries ):

3296

3315

3297

self.fp.close()

3316

self.fp.close()

3298

self.fp = open( self.filename, 'rb' )

3317

self.fp = open( self.filename, 'rb' )

3299

self.fp.seek( currentPointer )

3318

self.fp.seek( currentPointer )

3300

3319

3301

self.fileSize = os.path.getsize( self.filename )

3320

self.fileSize = os.path.getsize( self.filename )

3302

currentSize = self.fileSize - currentPointer

3321

currentSize = self.fileSize - currentPointer

3303

3322

3304

if ( currentSize >= neededSize ):

3323

if ( currentSize >= neededSize ):

3305

self.__rdBasicHeader()

3324

self.__rdBasicHeader()

3306

return 1

3325

return 1

3307

3326

3308

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

3327

print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)

3309

time.sleep( self.delay )

3328

time.sleep( self.delay )

3310

3329

3311

3330

3312

return 0

3331

return 0

3313

3332

3314

def __setNewBlock(self):

3333

def __setNewBlock(self):

3315

3334

3316

if self.online:

3335

if self.online:

3317

self.__jumpToLastBlock()

3336

self.__jumpToLastBlock()

3318

3337

3319

if self.flagIsNewFile:

3338

if self.flagIsNewFile:

3320

return 1

3339

return 1

3321

3340

3322

self.lastUTTime = self.utc

3341

self.lastUTTime = self.utc

3323

3342

3324

if self.online:

3343

if self.online:

3325

if self.__waitNewBlock():

3344

if self.__waitNewBlock():

3326

return 1

3345

return 1

3327

3346

3328

if self.nReadBlocks < self.dataBlocksPerFile:

3347

if self.nReadBlocks < self.dataBlocksPerFile:

3329

return 1

3348

return 1

3330

3349

3331

if not(self.setNextFile()):

3350

if not(self.setNextFile()):

3332

return 0

3351

return 0

3333

3352

3334

deltaTime = self.utc - self.lastUTTime

3353

deltaTime = self.utc - self.lastUTTime

3335

3354

3336

self.flagTimeBlock = 0

3355

self.flagTimeBlock = 0

3337

3356

3338

if deltaTime > self.maxTimeStep:

3357

if deltaTime > self.maxTimeStep:

3339

self.flagTimeBlock = 1

3358

self.flagTimeBlock = 1

3340

3359

3341

return 1

3360

return 1

3342

3361

3343

3362

3344

def readNextBlock(self):

3363

def readNextBlock(self):

3345

if not(self.__setNewBlock()):

3364

if not(self.__setNewBlock()):

3346

return 0

3365

return 0

3347

3366

3348

if not(self.readBlock()):

3367

if not(self.readBlock()):

3349

return 0

3368

return 0

3350

3369

3351

return 1

3370

return 1

3352

3371

3353

3372

3354

def getData(self):

3373

def getData(self):

3355

3374

3356

if self.flagNoMoreFiles:

3375

if self.flagNoMoreFiles:

3357

self.dataOut.flagNoData = True

3376

self.dataOut.flagNoData = True

3358

print 'Process finished'

3377

print 'Process finished'

3359

return 0

3378

return 0

3360

3379

3361

self.flagTimeBlock = 0

3380

self.flagTimeBlock = 0

3362

self.flagIsNewBlock = 0

3381

self.flagIsNewBlock = 0

3363

3382

3364

if not(self.readNextBlock()):

3383

if not(self.readNextBlock()):

3365

return 0

3384

return 0

3366

3385

3367

if self.data == None:

3386

if self.data == None:

3368

self.dataOut.flagNoData = True

3387

self.dataOut.flagNoData = True

3369

return 0

3388

return 0

3370

3389

3371

self.dataOut.data = self.data

3390

self.dataOut.data = self.data

3372

self.dataOut.data_header = self.data_header_dict

3391

self.dataOut.data_header = self.data_header_dict

3373

self.dataOut.utctime = self.utc

3392

self.dataOut.utctime = self.utc

3374

3393

3375

self.dataOut.header = self.header_dict

3394

self.dataOut.header = self.header_dict

3376

self.dataOut.expName = self.expName

3395

self.dataOut.expName = self.expName

3377

self.dataOut.nChannels = self.nChannels

3396

self.dataOut.nChannels = self.nChannels

3378

self.dataOut.timeZone = self.timeZone

3397

self.dataOut.timeZone = self.timeZone

3379

self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile

3398

self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile

3380

self.dataOut.comments = self.comments

3399

self.dataOut.comments = self.comments

3381

self.dataOut.timeInterval = self.timeInterval

3400

self.dataOut.timeInterval = self.timeInterval

3382

self.dataOut.channelList = self.channelList

3401

self.dataOut.channelList = self.channelList

3383

self.dataOut.heightList = self.heightList

3402

self.dataOut.heightList = self.heightList

3384

self.dataOut.flagNoData = False

3403

self.dataOut.flagNoData = False

3385

3404

3386

return self.dataOut.data

3405

return self.dataOut.data

3387

3406

3388

def run(self, **kwargs):

3407

def run(self, **kwargs):

3389

3408

3390

if not(self.isConfig):

3409

if not(self.isConfig):

3391

self.setup(**kwargs)

3410

self.setup(**kwargs)

3392

self.isConfig = True

3411

self.isConfig = True

3393

3412

3394

self.getData() No newline at end of file

3413

self.getData()

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages