SpectraIO.py
433 lines
| 14.6 KiB
| text/x-python
|
PythonLexer
|
r24 | ''' | |
File: SpectraIO.py | |||
Created on 20/02/2012 | |||
@author $Author$ | |||
@version $Id$ | |||
''' | |||
import os, sys | |||
import numpy | |||
import glob | |||
import fnmatch | |||
import time, datetime | |||
path = os.path.split(os.getcwd())[0] | |||
sys.path.append(path) | |||
|
r49 | from Model.JROHeader import * | |
|
r24 | from Model.Spectra import Spectra | |
|
r49 | from DataIO import JRODataReader | |
from DataIO import JRODataWriter | |||
|
r53 | from DataIO import isNumber | |
|
r24 | ||
|
r49 | class SpectraReader( JRODataReader ): | |
|
r30 | """ | |
|
r29 | Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura | |
|
r30 | de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones) | |
son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel. | |||
|
r59 | pares * alturas * perfiles (Self Spectra) | |
canales * alturas * perfiles (Cross Spectra) | |||
canales * alturas (DC Channels) | |||
|
r29 | Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader, | |
RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la | |||
cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de | |||
datos desde el "buffer" cada vez que se ejecute el metodo "getData". | |||
|
r30 | Example: | |
|
r24 | dpath = "/home/myuser/data" | |
startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0) | |||
endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0) | |||
readerObj = SpectraReader() | |||
readerObj.setup(dpath, startTime, endTime) | |||
while(True): | |||
readerObj.getData() | |||
print readerObj.m_Spectra.data | |||
|
r59 | if readerObj.flagNoMoreFiles: | |
|
r24 | break | |
""" | |||
|
r59 | m_DataObj = None | |
data_spc = None | |||
data_cspc = None | |||
data_dc = None | |||
|
r62 | nPairsEqualChannels = 0 | |
|
r59 | ||
|
r62 | nPairsUnequalChannels = 0 | |
|
r24 | ||
|
r59 | pts2read_SelfSpectra = 0 | |
pts2read_CrossSpectra = 0 | |||
pts2read_DCchannels = 0 | |||
|
r30 | ||
|
r59 | ext = ".pdata" | |
def __init__(self, m_Spectra=None): | |||
|
r30 | """ | |
|
r29 | Inicializador de la clase SpectraReader para la lectura de datos de espectros. | |
|
r30 | ||
Inputs: | |||
|
r24 | m_Spectra : Objeto de la clase Spectra. Este objeto sera utilizado para | |
almacenar un perfil de datos cada vez que se haga un requerimiento | |||
(getData). El perfil sera obtenido a partir del buffer de datos, | |||
si el buffer esta vacio se hara un nuevo proceso de lectura de un | |||
bloque de datos. | |||
Si este parametro no es pasado se creara uno internamente. | |||
|
r30 | ||
Affected: | |||
|
r53 | self.m_DataObj | |
|
r30 | ||
Return : None | |||
|
r24 | """ | |
if m_Spectra == None: | |||
m_Spectra = Spectra() | |||
if not( isinstance(m_Spectra, Spectra) ): | |||
raise ValueError, "in SpectraReader, m_Spectra must be an Spectra class object" | |||
|
r30 | ||
|
r59 | self.m_DataObj = m_Spectra | |
|
r36 | ||
|
r30 | ||
|
r59 | def __hasNotDataInBuffer(self): | |
|
r53 | return 1 | |
|
r30 | ||
|
r59 | ||
|
r53 | def getBlockDimension(self): | |
|
r59 | """ | |
Obtiene la cantidad de puntos a leer por cada bloque de datos | |||
|
r24 | ||
|
r59 | Affected: | |
|
r62 | self.nPairsEqualChannels | |
self.nPairsUnequalChannels | |||
|
r59 | self.pts2read_SelfSpectra | |
self.pts2read_CrossSpectra | |||
self.pts2read_DCchannels | |||
self.blocksize | |||
|
r62 | self.m_DataObj.nPairsEqualChannels | |
self.m_DataObj.nPairsUnequalChannels | |||
|
r59 | ||
Return: | |||
None | |||
""" | |||
|
r62 | self.nPairsEqualChannels = 0 | |
self.nPairsUnequalChannels = 0 | |||
|
r28 | ||
|
r30 | for i in range( 0, self.m_ProcessingHeader.totalSpectra*2, 2 ): | |
if self.m_ProcessingHeader.spectraComb[i] == self.m_ProcessingHeader.spectraComb[i+1]: | |||
|
r62 | self.nPairsEqualChannels = self.nPairsEqualChannels + 1 #par de canales iguales | |
|
r30 | else: | |
|
r62 | self.nPairsUnequalChannels = self.nPairsUnequalChannels + 1 #par de canales diferentes | |
|
r30 | ||
|
r62 | pts2read = self.m_ProcessingHeader.numHeights * self.m_ProcessingHeader.profilesPerBlock | |
self.pts2read_SelfSpectra = int( self.nPairsEqualChannels * pts2read ) | |||
self.pts2read_CrossSpectra = int( self.nPairsUnequalChannels * pts2read ) | |||
self.pts2read_DCchannels = int( self.m_SystemHeader.numChannels * self.m_ProcessingHeader.numHeights ) | |||
|
r24 | ||
|
r53 | self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels | |
|
r24 | ||
|
r62 | self.m_DataObj.nPairsEqualChannels = self.nPairsEqualChannels | |
self.m_DataObj.nPairsUnequalChannels = self.nPairsUnequalChannels | |||
|
r59 | ||
|
r24 | ||
|
r53 | def readBlock(self): | |
|
r24 | """ | |
|
r30 | Lee el bloque de datos desde la posicion actual del puntero del archivo | |
|
r53 | (self.fp) y actualiza todos los parametros relacionados al bloque de datos | |
|
r29 | (metadata + data). La data leida es almacenada en el buffer y el contador del buffer | |
es seteado a 0 | |||
|
r24 | ||
|
r30 | Return: None | |
|
r24 | ||
Variables afectadas: | |||
|
r53 | self.datablockIndex | |
self.flagIsNewFile | |||
|
r24 | self.flagIsNewBlock | |
self.nReadBlocks | |||
|
r53 | self.data_spc | |
self.data_cspc | |||
self.data_dc | |||
|
r47 | ||
Exceptions: | |||
Si un bloque leido no es un bloque valido | |||
|
r24 | """ | |
|
r47 | blockOk_flag = False | |
|
r53 | fpointer = self.fp.tell() | |
|
r33 | ||
|
r53 | spc = numpy.fromfile( self.fp, self.dataType[0], self.pts2read_SelfSpectra ) | |
cspc = numpy.fromfile( self.fp, self.dataType, self.pts2read_CrossSpectra ) | |||
dc = numpy.fromfile( self.fp, self.dataType, self.pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) ) | |||
|
r33 | ||
|
r35 | if self.online: | |
|
r53 | if (spc.size + cspc.size + dc.size) != self.blocksize: | |
for nTries in range( self.nTries ): | |||
|
r59 | print "\tWaiting %0.2f sec for the next block, try %03d ..." % (self.delay, nTries+1) | |
|
r53 | time.sleep( self.delay ) | |
self.fp.seek( fpointer ) | |||
fpointer = self.fp.tell() | |||
spc = numpy.fromfile( self.fp, self.dataType[0], self.pts2read_SelfSpectra ) | |||
cspc = numpy.fromfile( self.fp, self.dataType, self.pts2read_CrossSpectra ) | |||
dc = numpy.fromfile( self.fp, self.dataType, self.pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) ) | |||
|
r47 | ||
|
r53 | if (spc.size + cspc.size + dc.size) == self.blocksize: | |
|
r47 | blockOk_flag = True | |
|
r35 | break | |
|
r47 | if not( blockOk_flag ): | |
return 0 | |||
try: | |||
|
r62 | spc = spc.reshape( (self.nPairsEqualChannels, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D | |
cspc = cspc.reshape( (self.nPairsUnequalChannels, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D | |||
|
r47 | dc = dc.reshape( (self.m_SystemHeader.numChannels, self.m_ProcessingHeader.numHeights) ) #transforma a un arreglo 2D | |
except: | |||
print "Data file %s is invalid" % self.filename | |||
return 0 | |||
|
r26 | ||
|
r30 | if not( self.m_ProcessingHeader.shif_fft ): | |
spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones | |||
cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones | |||
|
r26 | ||
|
r60 | # spc = numpy.transpose( spc, (0,2,1) ) | |
# cspc = numpy.transpose( cspc, (0,2,1) ) | |||
|
r26 | #dc = numpy.transpose(dc, (0,2,1)) | |
|
r30 | ||
|
r53 | self.data_spc = spc | |
self.data_cspc = cspc['real'] + cspc['imag']*1j | |||
self.data_dc = dc['real'] + dc['imag']*1j | |||
|
r30 | ||
|
r59 | self.datablockIndex = 0 | |
|
r53 | self.flagIsNewFile = 0 | |
|
r24 | self.flagIsNewBlock = 1 | |
|
r47 | ||
|
r24 | self.nReadBlocks += 1 | |
|
r47 | self.nBlocks += 1 | |
|
r30 | ||
|
r47 | return 1 | |
|
r30 | ||
|
r59 | ||
|
r47 | def getData(self): | |
|
r24 | """ | |
|
r30 | Copia el buffer de lectura a la clase "Spectra", | |
|
r29 | con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de | |
lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock" | |||
|
r24 | ||
Return: | |||
|
r30 | 0 : Si no hay mas archivos disponibles | |
1 : Si hizo una buena copia del buffer | |||
|
r24 | ||
|
r36 | Affected: | |
|
r53 | self.m_DataObj | |
self.datablockIndex | |||
|
r30 | self.flagResetProcessing | |
self.flagIsNewBlock | |||
|
r24 | """ | |
|
r59 | if self.flagNoMoreFiles: return 0 | |
|
r24 | self.flagResetProcessing = 0 | |
self.flagIsNewBlock = 0 | |||
|
r59 | if self.__hasNotDataInBuffer(): | |
|
r33 | ||
|
r47 | if not( self.readNextBlock() ): | |
|
r53 | self.setNextFile() | |
|
r47 | return 0 | |
|
r24 | ||
|
r53 | self.m_DataObj.m_BasicHeader = self.m_BasicHeader.copy() | |
self.m_DataObj.m_ProcessingHeader = self.m_ProcessingHeader.copy() | |||
self.m_DataObj.m_RadarControllerHeader = self.m_RadarControllerHeader.copy() | |||
self.m_DataObj.m_SystemHeader = self.m_SystemHeader.copy() | |||
self.m_DataObj.heights = self.heights | |||
self.m_DataObj.dataType = self.dataType | |||
|
r24 | ||
|
r59 | if self.flagNoMoreFiles == 1: | |
|
r24 | print 'Process finished' | |
return 0 | |||
#data es un numpy array de 3 dmensiones (perfiles, alturas y canales) | |||
|
r30 | ||
|
r53 | if self.data_dc == None: | |
|
r59 | self.m_DataObj.flagNoData = True | |
|
r47 | return 0 | |
|
r53 | self.m_DataObj.flagNoData = False | |
self.m_DataObj.flagResetProcessing = self.flagResetProcessing | |||
|
r24 | ||
|
r53 | self.m_DataObj.data_spc = self.data_spc | |
self.m_DataObj.data_cspc = self.data_cspc | |||
self.m_DataObj.data_dc = self.data_dc | |||
|
r24 | ||
#call setData - to Data Object | |||
|
r59 | #self.datablockIndex += 1 | |
|
r30 | #self.idProfile += 1 | |
|
r24 | return 1 | |
|
r59 | ||
class SpectraWriter(JRODataWriter): | |||
|
r30 | """ | |
Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura | |||
de los datos siempre se realiza por bloques. | |||
""" | |||
|
r24 | ||
|
r59 | m_DataObj = None | |
ext = ".pdata" | |||
optchar = "P" | |||
shape_spc_Buffer = None | |||
shape_cspc_Buffer = None | |||
shape_dc_Buffer = None | |||
def __init__(self, m_Spectra=None): | |||
|
r30 | """ | |
Inicializador de la clase SpectraWriter para la escritura de datos de espectros. | |||
Affected: | |||
|
r53 | self.m_DataObj | |
|
r30 | self.m_BasicHeader | |
self.m_SystemHeader | |||
self.m_RadarControllerHeader | |||
self.m_ProcessingHeader | |||
Return: None | |||
""" | |||
|
r24 | if m_Spectra == None: | |
m_Spectra = Spectra() | |||
|
r30 | ||
|
r59 | if not( isinstance(m_Spectra, Spectra) ): | |
raise ValueError, "in SpectraReader, m_Spectra must be an Spectra class object" | |||
|
r53 | self.m_DataObj = m_Spectra | |
|
r47 | ||
|
r59 | ||
|
r53 | def hasAllDataInBuffer(self): | |
return 1 | |||
|
r59 | ||
|
r30 | ||
|
r53 | def setBlockDimension(self): | |
|
r59 | """ | |
Obtiene las formas dimensionales del los subbloques de datos que componen un bloque | |||
Affected: | |||
self.shape_spc_Buffer | |||
self.shape_cspc_Buffer | |||
self.shape_dc_Buffer | |||
|
r30 | ||
|
r59 | Return: None | |
""" | |||
|
r62 | self.shape_spc_Buffer = (self.m_DataObj.nPairsEqualChannels, | |
|
r53 | self.m_ProcessingHeader.numHeights, | |
self.m_ProcessingHeader.profilesPerBlock) | |||
|
r30 | ||
|
r62 | self.shape_cspc_Buffer = (self.m_DataObj.nPairsUnequalChannels, | |
|
r53 | self.m_ProcessingHeader.numHeights, | |
self.m_ProcessingHeader.profilesPerBlock) | |||
|
r24 | ||
|
r53 | self.shape_dc_Buffer = (self.m_SystemHeader.numChannels, | |
self.m_ProcessingHeader.numHeights) | |||
|
r30 | ||
|
r47 | ||
|
r53 | def writeBlock(self): | |
|
r30 | """ | |
Escribe el buffer en el file designado | |||
Affected: | |||
|
r53 | self.data_spc | |
self.data_cspc | |||
self.data_dc | |||
self.flagIsNewFile | |||
|
r30 | self.flagIsNewBlock | |
self.nWriteBlocks | |||
|
r53 | self.blocksCounter | |
|
r30 | ||
Return: None | |||
""" | |||
|
r60 | spc = self.data_spc | |
# spc = numpy.transpose( self.data_spc, (0,2,1) ) | |||
|
r30 | if not( self.m_ProcessingHeader.shif_fft ): | |
spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones | |||
data = spc.reshape((-1)) | |||
|
r53 | data.tofile(self.fp) | |
|
r30 | ||
|
r53 | data = numpy.zeros( self.shape_cspc_Buffer, self.dataType ) | |
|
r60 | cspc = self.data_cspc | |
# cspc = numpy.transpose( self.data_cspc, (0,2,1) ) | |||
|
r30 | if not( self.m_ProcessingHeader.shif_fft ): | |
cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones | |||
data['real'] = cspc.real | |||
data['imag'] = cspc.imag | |||
data = data.reshape((-1)) | |||
|
r53 | data.tofile(self.fp) | |
|
r30 | ||
|
r53 | data = numpy.zeros( self.shape_dc_Buffer, self.dataType ) | |
dc = self.data_dc | |||
|
r30 | data['real'] = dc.real | |
data['imag'] = dc.imag | |||
data = data.reshape((-1)) | |||
|
r53 | data.tofile(self.fp) | |
|
r30 | ||
|
r53 | self.data_spc.fill(0) | |
self.data_cspc.fill(0) | |||
self.data_dc.fill(0) | |||
|
r24 | ||
|
r53 | self.flagIsNewFile = 0 | |
|
r24 | self.flagIsNewBlock = 1 | |
self.nWriteBlocks += 1 | |||
|
r53 | self.blocksCounter += 1 | |
|
r24 | ||
|
r59 | ||
|
r47 | def putData(self): | |
|
r30 | """ | |
Setea un bloque de datos y luego los escribe en un file | |||
Affected: | |||
|
r53 | self.data_spc | |
self.data_cspc | |||
self.data_dc | |||
|
r30 | ||
Return: | |||
0 : Si no hay data o no hay mas files que puedan escribirse | |||
1 : Si se escribio la data de un bloque en un file | |||
""" | |||
|
r24 | self.flagIsNewBlock = 0 | |
|
r53 | if self.m_DataObj.flagNoData: | |
|
r30 | return 0 | |
|
r24 | ||
|
r53 | if self.m_DataObj.flagResetProcessing: | |
self.data_spc.fill(0) | |||
self.data_cspc.fill(0) | |||
self.data_dc.fill(0) | |||
self.setNextFile() | |||
|
r24 | ||
|
r53 | self.data_spc = self.m_DataObj.data_spc | |
self.data_cspc = self.m_DataObj.data_cspc | |||
self.data_dc = self.m_DataObj.data_dc | |||
|
r24 | ||
|
r59 | # #self.m_ProcessingHeader.dataBlocksPerFile) | |
|
r30 | if True: | |
|
r53 | self.getHeader() | |
|
r29 | self.writeNextBlock() | |
|
r59 | if self.flagNoMoreFiles: | |
|
r30 | #print 'Process finished' | |
return 0 | |||
|
r24 | ||
|
r30 | return 1 |