From def509924d6b2e4542f43f4e8639c1d7a842c385 2012-04-04 16:17:46 From: Victor Sarmiento Date: 2012-04-04 16:17:46 Subject: [PATCH] DataIO.py Clase para la lectura y escritura de datos valentin Sarmiento / miercoles 4 de abril de 2012 --- diff --git a/schainpy/IO/DataIO.py b/schainpy/IO/DataIO.py index 7ed5767..60a1571 100644 --- a/schainpy/IO/DataIO.py +++ b/schainpy/IO/DataIO.py @@ -8,26 +8,63 @@ import os, sys import glob import time import numpy +import fnmatch +import time, datetime path = os.path.split(os.getcwd())[0] sys.path.append(path) from Model.JROHeader import * +from Model.JROData import JROData -def checkForRealPath( path, year, doy, set, ext ): + +def isFileOK(filename): + """ + Determina si la cabecera de un archivo es valido o no, si lo es entonces seria un archivo que podria contener data, + si no seria un archivo invalido + + Return: + True : si es un archivo valido + False : si no es un archivo valido + + Exceptions: + Si al leer la cabecera esta no coincide con el tipo de las variables que la contienen entonces se dispara + una exception + """ + m_BasicHeader = BasicHeader() + m_ProcessingHeader = ProcessingHeader() + m_RadarControllerHeader = RadarControllerHeader() + m_SystemHeader = SystemHeader() + fp = None + + try: + fp = open( filename,'rb' ) + m_BasicHeader.read(fp) + m_SystemHeader.read(fp) + m_RadarControllerHeader.read(fp) + m_ProcessingHeader.read(fp) + fp.close() + except: + if fp != None: fp.close() + return False + + return True + + +def checkForRealPath(path, year, doy, set, ext): """ Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path, Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar el path exacto de un determinado file. Example : - nombre correcto del file es ../RAWDATA/D2009307/P2009307367 + nombre correcto del file es .../.../D2009307/P2009307367.ext Entonces la funcion prueba con las siguientes combinaciones - ../RAWDATA/d2009307/p2009307367 - ../RAWDATA/d2009307/P2009307367 - ../RAWDATA/D2009307/p2009307367 - ../RAWDATA/D2009307/P2009307367 + .../.../x2009307/y2009307367.ext + .../.../x2009307/Y2009307367.ext + .../.../X2009307/y2009307367.ext + .../.../X2009307/Y2009307367.ext siendo para este caso, la ultima combinacion de letras, identica al file buscado Return: @@ -38,11 +75,20 @@ def checkForRealPath( path, year, doy, set, ext ): filepath = None find_flag = False filename = None - - for dir in "dD": #barrido por las dos combinaciones posibles de "D" - for fil in "dD": #barrido por las dos combinaciones posibles de "D" + + if ext.lower() == ".r": #voltage + str1 = "dD" + str2 = "dD" + elif ext.lower() == ".pdata": #spectra + str1 = "dD" + str2 = "pP" + else: + return None, filename + + for dir in str1: #barrido por las dos combinaciones posibles de "D" + for fil in str2: #barrido por las dos combinaciones posibles de "D" doypath = "%s%04d%03d" % ( dir, year, doy ) #formo el nombre del directorio xYYYYDDD (x=d o x=D) - filename = "%s%04d%03d%03d%s" % ( fil, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext (p=d o p=D) + filename = "%s%04d%03d%03d%s" % ( fil, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext filepath = os.path.join( path, doypath, filename ) #formo el path completo if os.path.exists( filepath ): #verifico que exista find_flag = True @@ -56,7 +102,7 @@ def checkForRealPath( path, year, doy, set, ext ): return filepath, filename -def isNumber( str ): +def isNumber(str): """ Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. @@ -78,8 +124,7 @@ def isNumber( str ): def isThisFileinRange(filename, startUTSeconds, endUTSeconds): """ - Esta funcion determina si un archivo de datos en formato Jicamarca(.r) se encuentra - o no dentro del rango de fecha especificado. + Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado. Inputs: filename : nombre completo del archivo de datos en formato Jicamarca (.r) @@ -116,7 +161,7 @@ def isThisFileinRange(filename, startUTSeconds, endUTSeconds): return 1 -def getlastFileFromPath( pathList, ext ): +def getlastFileFromPath(pathList, ext): """ Depura el pathList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext" al final de la depuracion devuelve el ultimo file de la lista que quedo. @@ -158,17 +203,19 @@ class DataReader(): def __init__(self): pass + class DataWriter(): def __init__(self): pass -class JRODataReader(): + +class JRODataReader(DataReader): """ - Esta clase es usada como la clase padre de las clases VoltageReader and SpectraReader, + Esta clase es usada como la clase padre de las clases DataReader, contiene todos lo metodos necesarios para leer datos desde archivos en formato - Jicamarca (.r o .pdata). La lectura de los datos siempre se realiza por bloques. Los datos + jicamarca o pdata (.r o .pdata). La lectura de los datos siempre se realiza por bloques. Los datos leidos son array de 3 dimensiones: perfiles*alturas*canales @@ -178,10 +225,9 @@ class JRODataReader(): RadarControllerHeader y DataObj. Los tres primeros se usan para almacenar informacion de la cabecera de datos (metadata), y el cuarto (DataObj) para obtener y almacenar los datos desde el "datablock" cada vez que se ejecute el metodo "getData". - - """ + """ m_BasicHeader = BasicHeader() m_SystemHeader = SystemHeader() @@ -243,45 +289,173 @@ class JRODataReader(): __nFiles = 3 #number of files for searching + __pts2read = 0 + __blocksize = 0 + __utc = 0 + nBlocks = 0 + """ #speed of light __c = 3E8 - def __init__(self): - + def __init__(self, m_DataObj=None): """ - Inicializador + Inicializador de la clase JRODataReader para la lectura de datos. + + Input: + m_DataObj : Objeto de la clase JROData. Este objeto sera utilizado para + almacenar un perfil de datos cada vez que se haga un requerimiento + (getData). El perfil sera obtenido a partir del buffer de datos, + si el buffer esta vacio se hara un nuevo proceso de lectura de un + bloque de datos. + Si este parametro no es pasado se creara uno internamente. + + Variables afectadas: + self.m_DataObj + self.m_BasicHeader + self.m_SystemHeader + self.m_RadarControllerHeader + self.m_ProcessingHeader + + Return: + None """ + if m_DataObj == None: + m_DataObj = JROData() + + if not(isinstance(m_DataObj, JROData)): + raise ValueError, "in JRODataReader, m_DataObj must be an class object" + + self.m_DataObj = m_DataObj + + self.m_BasicHeader = BasicHeader() - raise ValueError, "This class has not been implemented" + self.m_SystemHeader = SystemHeader() + + self.m_RadarControllerHeader = RadarControllerHeader() + + self.m_ProcessingHeader = ProcessingHeader() - def __rdSystemHeader(self,fp=None): + self.__fp = None + + self.__fileIndex = None + + self.__startDateTime = None + + self.__endDateTime = None + + self.__dataType = None + + self.__fileSizeByHeader = 0 + + self.__pathList = [] + + self.filenameList = [] + + self.__lastUTTime = 0 + + self.__maxTimeStep = 30 + + self.__flagIsNewFile = 0 + + self.__ippSeconds = 0 + + self.flagResetProcessing = 0 + + self.flagIsNewBlock = 0 + + self.noMoreFiles = 0 + + self.nReadBlocks = 0 + + self.online = 0 + + self.filename = None + + self.fileSize = None + + self.firstHeaderSize = 0 + + self.basicHeaderSize = 24 + + self.idProfile = 0 + + self.datablock = None + + self.__datablockIndex = 9999 + + self.__delay = 7 #seconds + self.__nTries = 3 #quantity tries + self.__nFiles = 3 #number of files for searching + self.__year = 0 + self.__doy = 0 + self.__set = 0 + self.__ext = None + self.__path = None + self.__blocksize = 0 + self.__utc = 0 + self.nBlocks = 0 + + self.__pts2read = 0 + self.__pts2read_SelfSpectra = 0 + self.__pts2read_CrossSpectra = 0 + self.__pts2read_DCchannels = 0 + self.__blocksize = 0 + + self.__format = None + + + def __rdSystemHeader(self, fp=None): if fp == None: fp = self.__fp self.m_SystemHeader.read(fp) + - def __rdRadarControllerHeader(self,fp=None): + def __rdRadarControllerHeader(self, fp=None): if fp == None: fp = self.__fp self.m_RadarControllerHeader.read(fp) + - def __rdProcessingHeader(self,fp=None): + def __rdProcessingHeader(self, fp=None): if fp == None: fp = self.__fp self.m_ProcessingHeader.read(fp) + def __rdBasicHeader(self, fp=None): if fp == None: fp = self.__fp self.m_BasicHeader.read(fp) + def __readFirstHeader(self): - + """ + Lectura del First Header, es decir el Basic Header y el Long Header + + Affected: + self.m_BasicHeader + self.m_SystemHeader + self.m_RadarControllerHeader + self.m_ProcessingHeader + self.firstHeaderSize + self.__heights + self.__dataType + self.__fileSizeByHeader + self.__ippSeconds + self.nChannels + self.nPairs + self.__pts2read_SelfSpectra + self.__pts2read_CrossSpectra + + Return: + None + """ self.__rdBasicHeader() self.__rdSystemHeader() self.__rdRadarControllerHeader() @@ -317,13 +491,35 @@ class JRODataReader(): self.__heights = numpy.arange(xi, xf, step) self.__dataType = tmp self.__fileSizeByHeader = self.m_ProcessingHeader.dataBlocksPerFile * self.m_ProcessingHeader.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.m_ProcessingHeader.dataBlocksPerFile - 1) - self.__ippSeconds = 2*1000*self.m_RadarControllerHeader.ipp/self.__c + self.__ippSeconds = 2 * 1000 * self.m_RadarControllerHeader.ipp / self.__c - self.__pts2read = self.m_ProcessingHeader.profilesPerBlock * self.m_ProcessingHeader.numHeights * self.m_SystemHeader.numChannels - self.__blocksize = self.__pts2read + if self.__format == "jicamarca": + self.__pts2read = self.m_ProcessingHeader.profilesPerBlock * self.m_ProcessingHeader.numHeights * self.m_SystemHeader.numChannels + self.__blocksize = self.__pts2read + elif self.__format == "pdata": + self.nChannels = 0 + self.nPairs = 0 + + for i in range( 0, self.m_ProcessingHeader.totalSpectra*2, 2 ): + if self.m_ProcessingHeader.spectraComb[i] == self.m_ProcessingHeader.spectraComb[i+1]: + self.nChannels = self.nChannels + 1 + else: + self.nPairs = self.nPairs + 1 + + pts2read = self.m_ProcessingHeader.profilesPerBlock * self.m_ProcessingHeader.numHeights + self.__pts2read_SelfSpectra = int( pts2read * self.nChannels ) + self.__pts2read_CrossSpectra = int( pts2read * self.nPairs ) + self.__pts2read_DCchannels = int( self.m_ProcessingHeader.numHeights * self.m_SystemHeader.numChannels ) + + self.__blocksize = self.__pts2read_SelfSpectra + self.__pts2read_CrossSpectra + self.__pts2read_DCchannels + + self.m_DataObj.nChannels = self.nChannels + self.m_DataObj.nPairs = self.nPairs + - def __setNextFileOnline( self ): + + def __setNextFileOnline(self): """ Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files @@ -347,17 +543,13 @@ class JRODataReader(): countFiles = 0 countTries = 0 - fileStatus = 0 notFirstTime_flag = False - bChangeDir = False - - fileSize = 0 - fp = None + fileOk_flag = False + changeDir_flag = False + self.__flagIsNewFile = 0 - self.__flagNewFile = 0 - - #este loop permite llevar la cuenta de intentos, de files y carpetas, si no encuentra alguno sale del bucle - while( True ): + while( True ): #este loop permite llevar la cuenta de intentos, de files y carpetas, + #si no encuentra alguno sale del bucle countFiles += 1 if countFiles > (self.__nFiles + 1): @@ -368,94 +560,103 @@ class JRODataReader(): if countFiles > self.__nFiles: #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta self.__set = 0 self.__doy += 1 - bChangeDir = True + changeDir_flag = True file = None filename = None + fileOk_flag = False - countTries = 0 - - #espero hasta encontrar el 1er file disponible - while( True ): - - countTries += 1 - if( countTries >= self.__nTries ): #checkeo que no haya ido mas alla de la cantidad de intentos - break - - file, filename = checkForRealPath( self.__path, self.__year, self.__doy, self.__set, self.__ext ) - if file != None: - break - - if notFirstTime_flag: #este flag me sirve solo para esperar por el 1er file, en lo siguientes no espera solo checkea si existe o no - countTries = self.__nTries - print "\tsearching next \"%s\" file ..." % filename - break - - print "\twaiting new \"%s\" file ..." % filename - time.sleep( self.__delay ) - - if countTries >= self.__nTries: #se realizaron n intentos y no hubo un file nuevo - notFirstTime_flag = True - continue #vuelvo al inico del while principal + #busca el 1er file disponible + file, filename = checkForRealPath( self.__path, self.__year, self.__doy, self.__set, self.__ext ) + + if file == None: + if notFirstTime_flag: #si no es la primera vez que busca el file entonces no espera y busca for el siguiente file + print "\tsearching next \"%s\" file ..." % ( filename ) + continue + else: #si es la primera vez que busca el file entonces espera self.__nTries veces hasta encontrarlo o no + for nTries in range( self.__nTries ): + print "\twaiting new \"%s\" file, try %03d ..." % ( filename, nTries+1 ) + time.sleep( self.__delay ) + + file, filename = checkForRealPath( self.__path, self.__year, self.__doy, self.__set, self.__ext ) + if file != None: + fileOk_flag = True + break + + if not( fileOk_flag ): #no encontro ningun file valido a leer despues de haber esperado alguno + notFirstTime_flag = True + continue + + #una vez que se obtuvo el 1er file valido se procede a checkear si si tamanho es suficiente para empezar a leerlo + currentSize = os.path.getsize( file ) + neededSize = self.m_ProcessingHeader.blockSize + self.firstHeaderSize - countTries = 0 + #si el tamanho es suficiente entonces deja de buscar + if currentSize > neededSize: + fileOk_flag = True + break - #una vez que se obtuvo el 1er file valido se procede a checkear su contenido, y se espera una cierta cantidad - #de tiempo por una cierta cantidad de veces hasta que el contenido del file sea un contenido valido - while( True ): - countTries += 1 - if countTries > self.__nTries: - break - - try: - fp = open(file) - except: - print "The file \"%s\" can't be opened" % file - break - - fileSize = os.path.getsize( file ) - currentSize = fileSize - fp.tell() + fileOk_flag = False + #si el file no tiene el tamanho necesario se espera una cierta cantidad de tiempo + #por una cierta cantidad de veces hasta que el contenido del file sea valido + if changeDir_flag: #si al buscar un file cambie de directorio ya no espero y sigo con el siguiente file + print "\tsearching next \"%s\" file ..." % filename + changeDir_flag = False + continue + + for nTries in range( self.__nTries ): + print "\twaiting for the First Header block of \"%s\" file, try %03d ..." % ( filename, nTries+1 ) + time.sleep( self.__delay ) + + currentSize = os.path.getsize( file ) neededSize = self.m_ProcessingHeader.blockSize + self.firstHeaderSize if currentSize > neededSize: - fileStatus = 1 + fileOk_flag = True break - fp.close() - - if bChangeDir: #si al buscar un file cambie de directorio ya no espero y salgo del bucle while - print "\tsearching next \"%s\" file ..." % filename - break - - print "\twaiting for block of \"%s\" file ..." % filename - time.sleep( self.__delay ) - - if fileStatus == 1: + if fileOk_flag: #si encontro un file valido sale del bucle y deja de buscar break print "Skipping the file \"%s\" due to this files is empty" % filename countFiles = 0 - - if fileStatus == 1: - self.fileSize = fileSize + if fileOk_flag: + self.fileSize = os.path.getsize( file ) self.filename = file - self.__flagNewFile = 1 - self.__fp = fp - self.flagNoMoreFiles = 0 + self.__flagIsNewFile = 1 + if self.__fp != None: self.__fp.close() + self.__fp = open(file) + self.noMoreFiles = 0 print 'Setting the file: %s' % file else: self.fileSize = 0 self.filename = None self.__fp = None - self.flagNoMoreFiles = 1 + self.noMoreFiles = 1 print 'No more Files' - return fileStatus + return fileOk_flag def __setNextFileOffline(self): + """ + Busca el siguiente file dentro de un folder que tenga suficiente data para ser leida + + Affected: + self.__flagIsNewFile + self.__fileIndex + self.filename + self.fileSize + self.__fp + + Return: + 0 : si un determinado file no puede ser abierto + 1 : si el file fue abierto con exito + Excepciones: + Si un determinado file no puede ser abierto + """ idFile = self.__fileIndex while(True): @@ -463,6 +664,7 @@ class JRODataReader(): if not(idFile < len(self.__filenameList)): self.flagNoMoreFiles = 1 + print 'No more Files' return 0 filename = self.__filenameList[idFile] @@ -491,6 +693,7 @@ class JRODataReader(): print 'Setting the file: %s'%self.filename return 1 + def __setNextFile( self ): """ @@ -515,15 +718,18 @@ class JRODataReader(): else: newFile = self.__setNextFileOffline() + if self.noMoreFiles: + sys.exit(0) + if not(newFile): return 0 self.__readFirstHeader() - + self.nBlocks = 0 return 1 - def __setNewBlock( self ): + def __setNewBlock(self): """ Lee el Basic Header y posiciona le file pointer en la posicion inicial del bloque a leer @@ -539,7 +745,7 @@ class JRODataReader(): if self.__fp == None: return 0 - if self.__flagNewFile: + if self.__flagIsNewFile: return 1 currentSize = self.fileSize - self.__fp.tell() @@ -549,107 +755,199 @@ class JRODataReader(): if ( currentSize >= neededSize ): self.__rdBasicHeader() return 1 - elif self.online: - nTries = 0 - while( nTries < self.__nTries ): - nTries += 1 - print "Waiting for the next block, try %03d ..." % nTries + + #si es OnLine y ademas aun no se han leido un bloque completo entonces se espera por uno valido + elif (self.nBlocks != self.m_ProcessingHeader.dataBlocksPerFile) and self.online: + for nTries in range( self.__nTries ): + + fpointer = self.__fp.tell() + self.__fp.close() + + print "\tWaiting for the next block, try %03d ..." % (nTries+1) time.sleep( self.__delay ) - - fileSize = os.path.getsize(self.filename) - currentSize = fileSize - self.__fp.tell() + + self.__fp = open( self.filename, 'rb' ) + self.__fp.seek( fpointer ) + + self.fileSize = os.path.getsize( self.filename ) + currentSize = self.fileSize - self.__fp.tell() neededSize = self.m_ProcessingHeader.blockSize + self.basicHeaderSize if ( currentSize >= neededSize ): self.__rdBasicHeader() return 1 - + #Setting new file if not( self.__setNextFile() ): return 0 deltaTime = self.m_BasicHeader.utc - self.__lastUTTime # check this - self.flagNoContinuousBlock = 0 + self.flagResetProcessing = 0 if deltaTime > self.__maxTimeStep: - self.flagNoContinuousBlock = 1 - self.nReadBlocks = 0 + self.flagResetProcessing = 1 + #self.nReadBlocks = 0 return 1 + - def __readBlock(self): + def __readBlockVoltage(self): """ __readBlock lee el bloque de datos desde la posicion actual del puntero del archivo (self.__fp) y actualiza todos los parametros relacionados al bloque de datos (metadata + data). La data leida es almacenada en el buffer y el contador del buffer es seteado a 0 - Inputs: None Return: None - Variables afectadas: - + Affected: self.__datablockIndex - self.datablock - - self.__flagNewFile - - self.__flagNewBlock - + self.__flagIsNewFile + self.idProfile + self.flagIsNewBlock self.nReadBlocks + Exceptions: + Si un bloque leido no es un bloque valido """ - - #pts2read = self.m_ProcessingHeader.profilesPerBlock*self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels - + blockOk_flag = False fpointer = self.__fp.tell() junk = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read ) if self.online: if junk.size != self.__blocksize: - nTries = 0 - while( nTries < self.__nTries ): - nTries += 1 - print "Waiting for the next block, try %03d ..." % nTries + for nTries in range( self.__nTries ): + print "\tWaiting for the next block, try %03d ..." % (nTries+1) time.sleep( self.__delay ) self.__fp.seek( fpointer ) fpointer = self.__fp.tell() junk = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read ) if junk.size == self.__blocksize: - nTries = 0 + blockOk_flag = True break - if nTries > 0: - return + + if not( blockOk_flag ): + return 0 - junk = junk.reshape( (self.m_ProcessingHeader.profilesPerBlock, self.m_ProcessingHeader.numHeights, self.m_SystemHeader.numChannels) ) + try: + junk = junk.reshape( (self.m_ProcessingHeader.profilesPerBlock, self.m_ProcessingHeader.numHeights, self.m_SystemHeader.numChannels) ) + except: + print "Data file %s is invalid" % self.filename + return 0 - data = junk['real'] + junk['imag']*1j + self.datablock = junk['real'] + junk['imag']*1j self.__datablockIndex = 0 + self.__flagIsNewFile = 0 + self.idProfile = 0 + self.flagIsNewBlock = 1 + + self.nReadBlocks += 1 + self.nBlocks += 1 + + return 1 + + + def __readBlockSpectra(self): + """ + Lee el bloque de datos desde la posicion actual del puntero del archivo + (self.__fp) y actualiza todos los parametros relacionados al bloque de datos + (metadata + data). La data leida es almacenada en el buffer y el contador del buffer + es seteado a 0 - self.datablock = data + Return: None - self.__flagNewFile = 0 + Variables afectadas: + self.__datablockIndex + self.__flagIsNewFile + self.flagIsNewBlock + self.nReadBlocks + self.__data_spc + self.__data_cspc + self.__data_dc + + Exceptions: + Si un bloque leido no es un bloque valido + """ + blockOk_flag = False + fpointer = self.__fp.tell() + + spc = numpy.fromfile( self.__fp, self.__dataType[0], self.__pts2read_SelfSpectra ) + cspc = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read_CrossSpectra ) + dc = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) ) + + if self.online: + if (spc.size + cspc.size + dc.size) != self.__blocksize: + for nTries in range( self.__nTries ): + #nTries = 0 + #while( nTries < self.__nTries ): + #nTries += 1 + print "\tWaiting for the next block, try %03d ..." % (nTries+1) + time.sleep( self.__delay ) + self.__fp.seek( fpointer ) + fpointer = self.__fp.tell() + spc = numpy.fromfile( self.__fp, self.__dataType[0], self.__pts2read_SelfSpectra ) + cspc = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read_CrossSpectra ) + dc = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) ) + + if (spc.size + cspc.size + dc.size) == self.__blocksize: + blockOk_flag = True + break + #if (spc.size + cspc.size + dc.size) == self.__blocksize: + # nTries = 0 + # break + if not( blockOk_flag ): + return 0 + #if nTries > 0: + # return 0 + + try: + spc = spc.reshape( (self.nChannels, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D + cspc = cspc.reshape( (self.nPairs, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D + dc = dc.reshape( (self.m_SystemHeader.numChannels, self.m_ProcessingHeader.numHeights) ) #transforma a un arreglo 2D + except: + print "Data file %s is invalid" % self.filename + return 0 - self.__flagNewBlock = 1 + if not( self.m_ProcessingHeader.shif_fft ): + spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones + cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones + spc = numpy.transpose( spc, (0,2,1) ) + cspc = numpy.transpose( cspc, (0,2,1) ) + #dc = numpy.transpose(dc, (0,2,1)) + + self.__data_spc = spc + self.__data_cspc = cspc['real'] + cspc['imag']*1j + self.__data_dc = dc['real'] + dc['imag']*1j + + self.datablock_id = 0 + self.__flagIsNewFile = 0 + self.flagIsNewBlock = 1 + self.nReadBlocks += 1 + self.nBlocks += 1 + + return 1 + def __hasNotDataInBuffer(self): - if self.__datablockIndex >= self.m_ProcessingHeader.profilesPerBlock: - return 1 - - return 0 + if self.__format == "jicamarca": + if self.__datablockIndex >= self.m_ProcessingHeader.profilesPerBlock: + return 1 + else: return 0 + + return 1 - def __searchFilesOnLine( self, path, startDateTime=None, ext = ".r" ): + def __searchFilesOnLine( self, path, startDateTime=None, ext = ".pdata" ): """ Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y devuelve el archivo encontrado ademas de otros datos. @@ -679,7 +977,7 @@ class JRODataReader(): dirList = sorted( dirList, key=str.lower ) #para que quede ordenado al margen de si el nombre esta en mayusculas o minusculas, utilizo la funcion sorted if len(dirList) > 0 : - directory = dirList[-1] + directory = dirList[-1] #me quedo con el ultimo directorio de una carpeta else: year = startDateTime.timetuple().tm_year doy = startDateTime.timetuple().tm_yday @@ -814,17 +1112,11 @@ class JRODataReader(): file = os.path.join( path, dirfilename, filename ) - nTries = 0 - while(True): - - nTries += 1 - if nTries > self.__nTries: - break - + for nTries in range( self.__nTries+1 ): try: fp = open( file,'rb' ) #lectura binaria except: - raise IOError, "The file %s can't be opened" %(file) + raise IOError, "The file %s can't be opened" % (file) try: m_BasicHeader.read(fp) @@ -836,16 +1128,19 @@ class JRODataReader(): if m_BasicHeader.size > 24: break - print 'waiting for new block: try %02d' % ( nTries ) - time.sleep( self.__delay) - + if nTries >= self.__nTries: #si ya espero la cantidad de veces necesarias entonces ya no vuelve a esperar + break + + print '\twaiting for new block of file %s: try %02d' % ( file, nTries ) + time.sleep( self.__delay ) + if m_BasicHeader.size <= 24: return 0 return 1 - def setup(self, path, startDateTime, endDateTime=None, set=None, expLabel = "", ext = ".r", online = 0): + def setup(self, path, startDateTime, endDateTime=None, set=0, expLabel = "", ext = None, online = 0): """ setup configura los parametros de lectura de la clase VoltageReader. @@ -889,24 +1184,30 @@ class JRODataReader(): self.online """ if online: - nTries = 0 - while( nTries < self.__nTries ): - nTries += 1 - subfolder = "D%04d%03d" % ( startDateTime.timetuple().tm_year, startDateTime.timetuple().tm_yday ) - year, doy, set, filename, dirfilename = self.__searchFilesOnLine( path, startDateTime, ext ) - if filename == None: - file = os.path.join( path, subfolder ) - print "Searching first file in \"%s\", try %03d ..." % ( file, nTries ) - time.sleep( self.__delay ) - else: - break - - if filename == None: - print "No files On Line" + fileOK_flag = False + subfolder = "D%04d%03d" % ( startDateTime.timetuple().tm_year, startDateTime.timetuple().tm_yday ) + file = os.path.join( path, subfolder ) + + for nTries in range( self.__nTries+1 ): #espera por el 1er file + year, doy, set, filename, dirfilename = self.__searchFilesOnLine( path, startDateTime, ext ) + + if filename != None: + if isFileOK( os.path.join( path,dirfilename,filename ) ): + fileOK_flag = True + break + + if nTries >= self.__nTries: #si ya espero la cantidad de veces necesarias entonces ya no vuelve a esperar + break + + print "Searching first file in \"%s\", try %03d ..." % ( file, nTries+1 ) + time.sleep( self.__delay ) + + if not( fileOK_flag ): #filename == None: + print "No files on line or invalid first file" return 0 if self.__initFilesOnline( path, dirfilename, filename ) == 0: - print "The file %s hasn't enough data" + print "The file %s hasn't enough data" % filename return 0 self.__year = year @@ -916,13 +1217,29 @@ class JRODataReader(): else: pathList, filenameList = self.__searchFilesOffLine( path, startDateTime, endDateTime, set, expLabel, ext ) - self.__fileIndex = -1 + self.__fileIndex = -1 self.__pathList = pathList - self.__filenameList = filenameList + self.filenameList = filenameList self.online = online self.__ext = ext + ext = ext.lower() + + if ext == '.hdf5': + print 'call hdf5 library' + return 0 + + elif ext == '.r': + self.__format = 'jicamarca' + + elif ext == '.pdata': + self.__format = 'pdata' + + else: + print 'unknow format !!!' + return 0 + if not( self.__setNextFile() ): if (startDateTime != None) and (endDateTime != None): print "No files in range: %s - %s" %(startDateTime.ctime(), endDateTime.ctime()) @@ -943,11 +1260,11 @@ class JRODataReader(): self.endDoy = endDateTime.timetuple().tm_yday #call fillHeaderValues() - to Data Object - self.m_Voltage.m_BasicHeader = self.m_BasicHeader.copy() - self.m_Voltage.m_ProcessingHeader = self.m_ProcessingHeader.copy() - self.m_Voltage.m_RadarControllerHeader = self.m_RadarControllerHeader.copy() - self.m_Voltage.m_SystemHeader = self.m_SystemHeader.copy() - self.m_Voltage.dataType = self.__dataType + self.m_DataObj.m_BasicHeader = self.m_BasicHeader.copy() + self.m_DataObj.m_ProcessingHeader = self.m_ProcessingHeader.copy() + self.m_DataObj.m_RadarControllerHeader = self.m_RadarControllerHeader.copy() + self.m_DataObj.m_SystemHeader = self.m_SystemHeader.copy() + self.m_DataObj.dataType = self.__dataType return 1 @@ -965,7 +1282,11 @@ class JRODataReader(): if not(self.__setNewBlock()): return 0 - self.__readBlock() + if self.__format == "jicamarca": + self.__readBlockVoltage() + + elif self.__format == "pdata": + self.__readBlockSpectra() self.__lastUTTime = self.m_BasicHeader.utc @@ -994,521 +1315,540 @@ class JRODataReader(): self.flagNoContinuousBlock self.__flagNewBlock """ - self.flagNoContinuousBlock = 0 - self.__flagNewBlock = 0 + if self.noMoreFiles: return 0 + + self.flagResetProcessing = 0 + self.flagIsNewBlock = 0 - if self.__hasNotDataInBuffer(): + if self.__hasNotDataInBuffer(): - self.readNextBlock() - - self.m_Voltage.m_BasicHeader = self.m_BasicHeader.copy() - self.m_Voltage.m_ProcessingHeader = self.m_ProcessingHeader.copy() - self.m_Voltage.m_RadarControllerHeader = self.m_RadarControllerHeader.copy() - self.m_Voltage.m_SystemHeader = self.m_SystemHeader.copy() - self.m_Voltage.heights = self.__heights - self.m_Voltage.dataType = self.__dataType - - if self.flagNoMoreFiles == 1: + if not( self.readNextBlock() ): + self.__setNextFile() + return 0 + + self.m_DataObj.m_BasicHeader = self.m_BasicHeader.copy() + self.m_DataObj.m_ProcessingHeader = self.m_ProcessingHeader.copy() + self.m_DataObj.m_RadarControllerHeader = self.m_RadarControllerHeader.copy() + self.m_DataObj.m_SystemHeader = self.m_SystemHeader.copy() + self.m_DataObj.heights = self.__heights + self.m_DataObj.dataType = self.__dataType + + if self.noMoreFiles == 1: print 'Process finished' - return None + return 0 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales) - time = self.m_BasicHeader.utc + self.__datablockIndex * self.__ippSeconds - self.m_Voltage.m_BasicHeader.utc = time + if self.__format == "jicamarca": + if self.datablock == None: + self.m_Voltage.flagNoData = True + return 0 + + time = self.m_BasicHeader.utc + self.__datablockIndex * self.__ippSeconds + self.__utc = time + #self.m_DataObj.m_BasicHeader.utc = time + + self.m_DataObj.flagNoData = False + self.m_DataObj.flagResetProcessing = self.flagResetProcessing + + self.m_DataObj.data = self.datablock[self.__datablockIndex,:,:] + self.m_DataObj.idProfile = self.idProfile + + self.__datablockIndex += 1 + self.idProfile += 1 + + elif self.__format == "pdata": + if self.__data_dc == None: + self.m_Voltage.flagNoData = True + return 0 + + self.m_DataObj.flagNoData = False + self.m_DataObj.flagResetProcessing = self.flagResetProcessing + + self.m_DataObj.data_spc = self.__data_spc + self.m_DataObj.data_cspc = self.__data_cspc + self.m_DataObj.data_dc = self.__data_dc + + else: + return 0 + + return 1 #self.m_Voltage.data + + +class JRODataWriter(DataWriter): + """ + Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura + de los datos siempre se realiza por bloques. + """ + + def __init__(self): + """ + Inicializador de la clase VoltageWriter para la escritura de datos de espectros. + + Affected: + self.m_Voltage + self.m_BasicHeader + self.m_SystemHeader + self.m_RadarControllerHeader + self.m_ProcessingHeader + + Return: None + """ + if m_Voltage == None: + m_Voltage = Voltage() + + self.m_Voltage = m_Voltage + + self.__path = None + + self.__fp = None + + self.__format = None + + self.__blocksCounter = 0 + + self.__setFile = None + + self.__flagIsNewFile = 1 + + self.__dataType = None + + self.datablock = None + + self.__datablockIndex = 0 + + self.__ext = None + + self.__shapeBuffer = None + + self.__shape_spc_Buffer = None + self.__shape_cspc_Buffer = None + self.__shape_dc_Buffer = None + + self.nWriteBlocks = 0 + + self.flagIsNewBlock = 0 + + self.noMoreFiles = 0 + + self.filename = None + + self.m_BasicHeader= BasicHeader() + + self.m_SystemHeader = SystemHeader() + + self.m_RadarControllerHeader = RadarControllerHeader() + + self.m_ProcessingHeader = ProcessingHeader() + + self.__data_spc = None + self.__data_cspc = None + self.__data_dc = None + + + def __writeFirstHeader(self): + """ + Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader) + + Affected: + __dataType + + Return: + None + """ + self.__writeBasicHeader() + self.__wrSystemHeader() + self.__wrRadarControllerHeader() + self.__wrProcessingHeader() + self.__dataType = self.m_DataObj.dataType + + + def __writeBasicHeader(self, fp=None): + """ + Escribe solo el Basic header en el file creado + + Return: + None + """ + if fp == None: + fp = self.__fp + + self.m_BasicHeader.write(fp) + + + def __wrSystemHeader(self, fp=None): + """ + Escribe solo el System header en el file creado + + Return: + None + """ + if fp == None: + fp = self.__fp + + self.m_SystemHeader.write(fp) + + + def __wrRadarControllerHeader(self, fp=None): + """ + Escribe solo el RadarController header en el file creado + + Return: + None + """ + if fp == None: + fp = self.__fp + + self.m_RadarControllerHeader.write(fp) + + + def __wrProcessingHeader(self, fp=None): + """ + Escribe solo el Processing header en el file creado + + Return: + None + """ + if fp == None: + fp = self.__fp + + self.m_ProcessingHeader.write(fp) + + + def __setNextFile(self): + """ + Determina el siguiente file que sera escrito + + Affected: + self.filename + self.__subfolder + self.__fp + self.__setFile + self.__flagIsNewFile + + Return: + 0 : Si el archivo no puede ser escrito + 1 : Si el archivo esta listo para ser escrito + """ + ext = self.__ext + path = self.__path + + if self.__fp != None: + self.__fp.close() + + timeTuple = time.localtime( self.m_DataObj.m_BasicHeader.utc ) # utc from m_Voltage + subfolder = 'D%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) + + tmp = os.path.join( path, subfolder ) + if not( os.path.exists(tmp) ): + os.mkdir(tmp) + self.__setFile = -1 #inicializo mi contador de seteo + else: + filesList = os.listdir( tmp ) + if len( filesList ) > 0: + filesList = sorted( filesList, key=str.lower ) + filen = filesList[-1] + # el filename debera tener el siguiente formato + # 0 1234 567 89A BCDE (hex) + # D YYYY DDD SSS .ext + if isNumber( filen[8:11] ): + self.__setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file + else: + self.__setFile = -1 + else: + self.__setFile = -1 #inicializo mi contador de seteo + + setFile = self.__setFile + setFile += 1 + + if self.__format == "jicamarca": + shead = "D" + elif self.__format == "pdata": + shead = "P" + + file = '%s%4.4d%3.3d%3.3d%s' % ( shead, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext ) + + filename = os.path.join( path, subfolder, file ) + + fp = open( filename,'wb' ) + + self.__blocksCounter = 0 - self.m_Voltage.flagNoData = False - self.m_Voltage.flagNoContinuousBlock = self.flagNoContinuousBlock + #guardando atributos + self.filename = filename + self.__subfolder = subfolder + self.__fp = fp + self.__setFile = setFile + self.__flagIsNewFile = 1 + + print 'Writing the file: %s'%self.filename + + self.__writeFirstHeader() + + return 1 + + + def __setNewBlock(self): + """ + Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header + + Return: + 0 : si no pudo escribir nada + 1 : Si escribio el Basic el First Header + """ + if self.__fp == None: + self.__setNextFile() + + if self.__flagIsNewFile: + return 1 + + if self.__blocksCounter < self.m_ProcessingHeader.dataBlocksPerFile: + self.__writeBasicHeader() + return 1 + + if not( self.__setNextFile() ): + return 0 + + return 1 + + + def __writeVoltageBlock(self): + """ + Escribe el buffer en el file designado + + Affected: + self.__datablockIndex + self.__flagIsNewFile + self.flagIsNewBlock + self.nWriteBlocks + self.__blocksCounter + + Return: None + """ + data = numpy.zeros( self.__shapeBuffer, self.__dataType ) + + data['real'] = self.datablock.real + data['imag'] = self.datablock.imag + + data = data.reshape( (-1) ) + + data.tofile( self.__fp ) + + self.datablock.fill(0) + self.__datablockIndex = 0 + self.__flagIsNewFile = 0 + self.flagIsNewBlock = 1 + self.nWriteBlocks += 1 + self.__blocksCounter += 1 + + + def __writeSpectraBlock(self): + """ + Escribe el buffer en el file designado + + Affected: + self.__data_spc + self.__data_cspc + self.__data_dc + self.__flagIsNewFile + self.flagIsNewBlock + self.nWriteBlocks + self.__blocksCounter + + Return: None + """ + spc = numpy.transpose( self.__data_spc, (0,2,1) ) + if not( self.m_ProcessingHeader.shif_fft ): + spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones + data = spc.reshape((-1)) + data.tofile(self.__fp) + + data = numpy.zeros( self.__shape_cspc_Buffer, self.__dataType ) + cspc = numpy.transpose( self.__data_cspc, (0,2,1) ) + if not( self.m_ProcessingHeader.shif_fft ): + cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones + data['real'] = cspc.real + data['imag'] = cspc.imag + data = data.reshape((-1)) + data.tofile(self.__fp) + + data = numpy.zeros( self.__shape_dc_Buffer, self.__dataType ) + dc = self.__data_dc + data['real'] = dc.real + data['imag'] = dc.imag + data = data.reshape((-1)) + data.tofile(self.__fp) + + self.__data_spc.fill(0) + self.__data_cspc.fill(0) + self.__data_dc.fill(0) + + self.__flagIsNewFile = 0 + self.flagIsNewBlock = 1 + self.nWriteBlocks += 1 + self.__blocksCounter += 1 + + + def writeNextBlock(self): + """ + Selecciona el bloque siguiente de datos y los escribe en un file + + Return: + 0 : Si no hizo pudo escribir el bloque de datos + 1 : Si no pudo escribir el bloque de datos + """ + if not( self.__setNewBlock() ): + return 0 + + if self.__format == "jicamarca": + self.__writeVoltageBlock() + + if self.__format == "pdata": + self.__writeSpectraBlock() + + return 1 + + + def __hasAllDataInBuffer(self): + if self.__format == "jicamarca": + if self.__datablockIndex >= self.m_ProcessingHeader.profilesPerBlock: + return 1 + else: return 0 + + return 1 + + + def putData(self): + """ + Setea un bloque de datos y luego los escribe en un file + + Affected: + self.flagIsNewBlock + self.__datablockIndex + + Return: + 0 : Si no hay data o no hay mas files que puedan escribirse + 1 : Si se escribio la data de un bloque en un file + """ + self.flagIsNewBlock = 0 - self.m_Voltage.data = self.datablock[self.__datablockIndex,:,:] - self.m_Voltage.profileIndex = self.__datablockIndex + if self.m_DataObj.flagNoData: + return 0 + if self.m_DataObj.flagResetProcessing: + if self.__format == "jicamarca": + self.datablock.fill(0) + self.__datablockIndex = 0 + + elif self.__format == "pdata": + self.__data_spc.fill(0) + self.__data_cspc.fill(0) + self.__data_dc.fill(0) + + self.__setNextFile() + + if self.__format == "jicamarca": + self.datablock[self.__datablockIndex,:,:] = self.m_Voltage.data + + elif self.__format == "pdata": + self.__data_spc = self.m_Spectra.data_spc + self.__data_cspc = self.m_Spectra.data_cspc + self.__data_dc = self.m_Spectra.data_dc + self.__datablockIndex += 1 - #call setData - to Data Object - - return self.m_Voltage.data - - -#class VoltageWriter(DataWriter): -# """ -# Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura -# de los datos siempre se realiza por bloques. -# """ -# __configHeaderFile = 'wrSetHeadet.txt' -# -# def __init__( self, m_Voltage = None ): -# """ -# Inicializador de la clase VoltageWriter para la escritura de datos de espectros. -# -# Affected: -# self.m_Voltage -# self.m_BasicHeader -# self.m_SystemHeader -# self.m_RadarControllerHeader -# self.m_ProcessingHeader -# -# Return: None -# """ -# if m_Voltage == None: -# m_Voltage = Voltage() -# -# self.m_Voltage = m_Voltage -# -# self.__path = None -# -# self.__fp = None -# -# self.__format = None -# -# self.__blocksCounter = 0 -# -# self.__setFile = None -# -# self.__flagNewFile = 1 -# -# self.datablock = None -# -# self.__datablockIndex = 0 -# -# self.__dataType = None -# -# self.__ext = None -# -# self.__shapeBuffer = None -# -# self.nWriteBlocks = 0 -# -# self.__flagNewBlock = 0 -# -# self.flagNoMoreFiles = 0 -# -# self.filename = None -# -# self.m_BasicHeader= BasicHeader() -# -# self.m_SystemHeader = SystemHeader() -# -# self.m_RadarControllerHeader = RadarControllerHeader() -# -# self.m_ProcessingHeader = ProcessingHeader() -# -# -# def __writeFirstHeader( self ): -# """ -# Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader) -# -# Affected: -# __dataType -# -# Return: -# None -# """ -# self.__writeBasicHeader() -# self.__wrSystemHeader() -# self.__wrRadarControllerHeader() -# self.__wrProcessingHeader() -# self.__dataType = self.m_Voltage.dataType -# -# -# def __writeBasicHeader( self, fp=None ): -# """ -# Escribe solo el Basic header en el file creado -# -# Return: -# None -# """ -# if fp == None: -# fp = self.__fp -# -# self.m_BasicHeader.write(fp) -# -# -# def __wrSystemHeader( self, fp=None ): -# """ -# Escribe solo el System header en el file creado -# -# Return: -# None -# """ -# if fp == None: -# fp = self.__fp -# -# self.m_SystemHeader.write(fp) -# -# -# def __wrRadarControllerHeader( self, fp=None ): -# """ -# Escribe solo el RadarController header en el file creado -# -# Return: -# None -# """ -# if fp == None: -# fp = self.__fp -# -# self.m_RadarControllerHeader.write(fp) -# -# -# def __wrProcessingHeader( self, fp=None ): -# """ -# Escribe solo el Processing header en el file creado -# -# Return: -# None -# """ -# if fp == None: -# fp = self.__fp -# -# self.m_ProcessingHeader.write(fp) -# -# def __setNextFile( self ): -# """ -# Determina el siguiente file que sera escrito -# -# Affected: -# self.filename -# self.__subfolder -# self.__fp -# self.__setFile -# self.__flagNewFile -# -# Return: -# 0 : Si el archivo no puede ser escrito -# 1 : Si el archivo esta listo para ser escrito -# """ -# #setFile = self.__setFile -# ext = self.__ext -# path = self.__path -# -# #setFile += 1 -# -# if self.__fp != None: -# self.__fp.close() -# -# """ -# timeTuple = time.localtime(self.m_Voltage.m_BasicHeader.utc) # utc from m_Voltage -# file = 'D%4.4d%3.3d%3.3d%s' % (timeTuple.tm_year,timeTuple.tm_yday,setFile,ext) -# subfolder = 'D%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) -# tmp = os.path.join(path,subfolder) -# if not(os.path.exists(tmp)): -# os.mkdir(tmp) -# """ -# ################################## -# if self.m_BasicHeader.size <= 24: return 0 #no existe la suficiente data para ser escrita -# -# timeTuple = time.localtime( self.m_Voltage.m_BasicHeader.utc ) # utc from m_Voltage -# subfolder = 'D%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) -# -# tmp = os.path.join( path, subfolder ) -# if not( os.path.exists(tmp) ): -# os.mkdir(tmp) -# self.__setFile = -1 #inicializo mi contador de seteo -# else: -# filesList = os.listdir( tmp ) -# if len( filesList ) > 0: -# filesList = sorted( filesList, key=str.lower ) -# filen = filesList[-1] -# # el filename debera tener el siguiente formato -# # 0 1234 567 89A BCDE (hex) -# # D YYYY DDD SSS .ext -# if isNumber( filen[8:11] ): -# self.__setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file -# else: -# self.__setFile = -1 -# else: -# self.__setFile = -1 #inicializo mi contador de seteo -# -# setFile = self.__setFile -# setFile += 1 -# file = 'D%4.4d%3.3d%3.3d%s' % ( timeTuple.tm_year, timeTuple.tm_yday, setFile, ext ) -# ################################## -# -# filename = os.path.join( path, subfolder, file ) -# -# fp = open( filename,'wb' ) -# -# self.__blocksCounter = 0 -# -# #guardando atributos -# self.filename = filename -# self.__subfolder = subfolder -# self.__fp = fp -# self.__setFile = setFile -# self.__flagNewFile = 1 -# -# print 'Writing the file: %s'%self.filename -# -# self.__writeFirstHeader() -# -# return 1 -# -# -# def __setNewBlock( self ): -# """ -# Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header -# -# Return: -# 0 : si no pudo escribir nada -# 1 : Si escribio el Basic el First Header -# """ -# if self.__fp == None: -# self.__setNextFile() -# -# if self.__flagNewFile: -# return 1 -# -# if self.__blocksCounter < self.m_ProcessingHeader.dataBlocksPerFile: -# self.__writeBasicHeader() -# return 1 -# -# if not( self.__setNextFile() ): -# return 0 -# -# return 1 -# -# def __writeBlock( self ): -# """ -# Escribe el buffer en el file designado -# -# Affected: -# self.__datablockIndex -# self.__flagNewFile -# self.__flagNewBlock -# self.nWriteBlocks -# self.__blocksCounter -# -# Return: None -# """ -# data = numpy.zeros( self.__shapeBuffer, self.__dataType ) -# -# data['real'] = self.datablock.real -# data['imag'] = self.datablock.imag -# -# data = data.reshape( (-1) ) -# -# data.tofile( self.__fp ) -# -# self.datablock.fill(0) -# -# self.__datablockIndex = 0 -# -# self.__flagNewFile = 0 -# -# self.__flagNewBlock = 1 -# -# self.nWriteBlocks += 1 -# -# self.__blocksCounter += 1 -# -# -# def writeNextBlock( self ): -# """ -# Selecciona el bloque siguiente de datos y los escribe en un file -# -# Return: -# 0 : Si no hizo pudo escribir el bloque de datos -# 1 : Si no pudo escribir el bloque de datos -# """ -# if not(self.__setNewBlock()): -# return 0 -# -# self.__writeBlock() -# -# return 1 -# -# -# def __hasAllDataInBuffer( self ): -# if self.__datablockIndex >= self.m_ProcessingHeader.profilesPerBlock: -# return 1 -# -# return 0 -# -# -# def putData( self ): -# """ -# Setea un bloque de datos y luego los escribe en un file -# -# Affected: -# self.__flagNewBlock -# self.__datablockIndex -# -# Return: -# 0 : Si no hay data o no hay mas files que puedan escribirse -# 1 : Si se escribio la data de un bloque en un file -# """ -# self.__flagNewBlock = 0 -# -# if self.m_Voltage.flagNoData: -# return 0 -# -# if self.m_Voltage.flagNoContinuousBlock: -# -# self.datablock.fill(0) -# -# self.__datablockIndex = 0 -# self.__setNextFile() -# -# self.datablock[self.__datablockIndex,:,:] = self.m_Voltage.data -# -# self.__datablockIndex += 1 -# -# if self.__hasAllDataInBuffer(): -# -# self.__getHeader() -# self.writeNextBlock() -# -# if self.flagNoMoreFiles: -# #print 'Process finished' -# return 0 -# -# return 1 -# -# -# def __getHeader( self ): -# """ -# Obtiene una copia del First Header -# -# Affected: -# self.m_BasicHeader -# self.m_SystemHeader -# self.m_RadarControllerHeader -# self.m_ProcessingHeader -# self.__dataType -# -# Return: -# None -# """ -# self.m_BasicHeader = self.m_Voltage.m_BasicHeader.copy() -# self.m_SystemHeader = self.m_Voltage.m_SystemHeader.copy() -# self.m_RadarControllerHeader = self.m_Voltage.m_RadarControllerHeader.copy() -# self.m_ProcessingHeader = self.m_Voltage.m_ProcessingHeader.copy() -# self.__dataType = self.m_Voltage.dataType -# -# -# def __setHeaderByFile( self ): -# -# format = self.__format -# header = ['Basic','System','RadarController','Processing'] -# -# fmtFromFile = None -# headerFromFile = None -# -# -# fileTable = self.__configHeaderFile -# -# if os.access(fileTable, os.R_OK): -# import re, string -# -# f = open(fileTable,'r') -# lines = f.read() -# f.close() -# -# #Delete comments into expConfig -# while 1: -# -# startComment = string.find(lines.lower(),'#') -# if startComment == -1: -# break -# endComment = string.find(lines.lower(),'\n',startComment) -# lines = string.replace(lines,lines[startComment:endComment+1],'', 1) -# -# while expFromFile == None: -# -# currFmt = string.find(lines.lower(),'format="%s"' %(expName)) -# nextFmt = string.find(lines.lower(),'format',currFmt+10) -# -# if currFmt == -1: -# break -# if nextFmt == -1: -# nextFmt = len(lines)-1 -# -# fmtTable = lines[currFmt:nextFmt] -# lines = lines[nextFmt:] -# -# fmtRead = self.__getValueFromArg(fmtTable,'format') -# if fmtRead != format: -# continue -# fmtFromFile = fmtRead -# -# lines2 = fmtTable -# -# while headerFromFile == None: -# -# currHeader = string.find(lines2.lower(),'header="%s"' %(header)) -# nextHeader = string.find(lines2.lower(),'header',currHeader+10) -# -# if currHeader == -1: -# break -# if nextHeader == -1: -# nextHeader = len(lines2)-1 -# -# headerTable = lines2[currHeader:nextHeader] -# lines2 = lines2[nextHeader:] -# -# headerRead = self.__getValueFromArg(headerTable,'site') -# if not(headerRead in header): -# continue -# headerFromFile = headerRead -# -# if headerRead == 'Basic': -# self.m_BasicHeader.size = self.__getValueFromArg(headerTable,'size',lower=False) -# self.m_BasicHeader.version = self.__getValueFromArg(headerTable,'version',lower=False) -# self.m_BasicHeader.dataBlock = self.__getValueFromArg(headerTable,'dataBlock',lower=False) -# self.m_BasicHeader.utc = self.__getValueFromArg(headerTable,'utc',lower=False) -# self.m_BasicHeader.miliSecond = self.__getValueFromArg(headerTable,'miliSecond',lower=False) -# self.m_BasicHeader.timeZone = self.__getValueFromArg(headerTable,'timeZone',lower=False) -# self.m_BasicHeader.dstFlag = self.__getValueFromArg(headerTable,'dstFlag',lower=False) -# self.m_BasicHeader.errorCount = self.__getValueFromArg(headerTable,'errorCount',lower=False) -# -# else: -# print "file access denied:%s"%fileTable -# sys.exit(0) -# -# -# def setup( self, path, set=0, format='rawdata' ): -# """ -# Setea el tipo de formato en la cual sera guardada la data y escribe el First Header -# -# Inputs: -# path : el path destino en el cual se escribiran los files a crear -# format : formato en el cual sera salvado un file -# set : el setebo del file -# -# Return: -# 0 : Si no realizo un buen seteo -# 1 : Si realizo un buen seteo -# """ -# if format == 'hdf5': -# ext = '.hdf5' -# format = 'hdf5' -# print 'call hdf5 library' -# return 0 -# -# if format == 'rawdata': -# ext = '.r' -# format = 'Jicamarca' -# -# #call to config_headers -# #self.__setHeaderByFile() -# -# self.__path = path -# self.__setFile = set - 1 -# self.__ext = ext -# self.__format = format -# -# self.__getHeader() -# self.__shapeBuffer = (self.m_ProcessingHeader.profilesPerBlock, -# self.m_ProcessingHeader.numHeights, -# self.m_SystemHeader.numChannels ) -# -# self.datablock = numpy.zeros(self.__shapeBuffer, numpy.dtype('complex')) -# -## if not(self.__setNextFile()): -## return 0 -# return 1 - -class JRODataWriter(): + if self.__hasAllDataInBuffer(): + self.__getHeader() + self.writeNextBlock() + + if self.noMoreFiles: + #print 'Process finished' + return 0 + + return 1 + + + def __getHeader(self): + """ + Obtiene una copia del First Header + + Affected: + self.m_BasicHeader + self.m_SystemHeader + self.m_RadarControllerHeader + self.m_ProcessingHeader + self.__dataType + + Return: + None + """ + self.m_BasicHeader = self.m_DataObj.m_BasicHeader.copy() + self.m_SystemHeader = self.m_DataObj.m_SystemHeader.copy() + self.m_RadarControllerHeader = self.m_DataObj.m_RadarControllerHeader.copy() + self.m_ProcessingHeader = self.m_DataObj.m_ProcessingHeader.copy() + self.__dataType = self.m_DataObj.dataType + + + def setup(self, path, set=0, ext='.pdata'): + """ + Setea el tipo de formato en la cual sera guardada la data y escribe el First Header + + Inputs: + path : el path destino en el cual se escribiran los files a crear + format : formato en el cual sera salvado un file + set : el setebo del file + + Return: + 0 : Si no realizo un buen seteo + 1 : Si realizo un buen seteo + """ + + ext = ext.lower() + + if ext == '.hdf5': + print 'call hdf5 library' + return 0 + + elif ext == '.r': + format = 'jicamarca' + + elif ext == '.pdata': + format = 'pdata' + + else: + print 'unknow format !!!' + return 0 + + self.__path = path + self.__setFile = set - 1 + self.__ext = ext + self.__format = format + self.__getHeader() + + if self.__format == "jicamarca": + self.__shapeBuffer = (self.m_ProcessingHeader.profilesPerBlock, + self.m_ProcessingHeader.numHeights, + self.m_SystemHeader.numChannels ) + + self.datablock = numpy.zeros(self.__shapeBuffer, numpy.dtype('complex')) + + elif self.__format == "pdata": + self.__shape_spc_Buffer = ( self.m_Spectra.nChannels, + self.m_ProcessingHeader.numHeights, + self.m_ProcessingHeader.profilesPerBlock + ) - def __init__(self): - pass \ No newline at end of file + self.__shape_cspc_Buffer = ( self.m_Spectra.nPairs, + self.m_ProcessingHeader.numHeights, + self.m_ProcessingHeader.profilesPerBlock + ) + + self.__shape_dc_Buffer = ( self.m_SystemHeader.numChannels, + self.m_ProcessingHeader.numHeights + ) + + if not( self.__setNextFile() ): + print "There isn't a next file" + return 0 + + return 1