SpectraIO.py
535 lines
| 15.8 KiB
| text/x-python
|
PythonLexer
|
r24 | ''' | |
File: SpectraIO.py | |||
Created on 20/02/2012 | |||
@author $Author$ | |||
@version $Id$ | |||
''' | |||
import os, sys | |||
import numpy | |||
import glob | |||
import fnmatch | |||
import time, datetime | |||
path = os.path.split(os.getcwd())[0] | |||
sys.path.append(path) | |||
|
r49 | from Model.JROHeader import * | |
|
r24 | from Model.Spectra import Spectra | |
|
r89 | from JRODataIO import JRODataReader | |
from JRODataIO import JRODataWriter | |||
from JRODataIO import isNumber | |||
|
r24 | ||
|
r49 | class SpectraReader( JRODataReader ): | |
|
r30 | """ | |
|
r29 | Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura | |
|
r30 | de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones) | |
son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel. | |||
|
r65 | paresCanalesIguales * alturas * perfiles (Self Spectra) | |
paresCanalesDiferentes * alturas * perfiles (Cross Spectra) | |||
canales * alturas (DC Channels) | |||
|
r59 | ||
|
r29 | Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader, | |
RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la | |||
cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de | |||
datos desde el "buffer" cada vez que se ejecute el metodo "getData". | |||
|
r30 | Example: | |
|
r24 | dpath = "/home/myuser/data" | |
startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0) | |||
endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0) | |||
readerObj = SpectraReader() | |||
readerObj.setup(dpath, startTime, endTime) | |||
while(True): | |||
readerObj.getData() | |||
print readerObj.m_Spectra.data | |||
|
r59 | if readerObj.flagNoMoreFiles: | |
|
r24 | break | |
""" | |||
|
r59 | m_DataObj = None | |
data_spc = None | |||
data_cspc = None | |||
data_dc = None | |||
pts2read_SelfSpectra = 0 | |||
pts2read_CrossSpectra = 0 | |||
pts2read_DCchannels = 0 | |||
|
r30 | ||
|
r79 | nChannels = 0 | |
|
r65 | ||
|
r79 | nPairs = 0 | |
|
r65 | ||
|
r86 | #pairList = None | |
|
r59 | ||
|
r86 | channelList = None | |
|
r59 | ||
def __init__(self, m_Spectra=None): | |||
|
r30 | """ | |
|
r29 | Inicializador de la clase SpectraReader para la lectura de datos de espectros. | |
|
r30 | ||
Inputs: | |||
|
r24 | m_Spectra : Objeto de la clase Spectra. Este objeto sera utilizado para | |
almacenar un perfil de datos cada vez que se haga un requerimiento | |||
(getData). El perfil sera obtenido a partir del buffer de datos, | |||
si el buffer esta vacio se hara un nuevo proceso de lectura de un | |||
bloque de datos. | |||
Si este parametro no es pasado se creara uno internamente. | |||
|
r30 | ||
Affected: | |||
|
r53 | self.m_DataObj | |
|
r30 | ||
Return : None | |||
|
r24 | """ | |
if m_Spectra == None: | |||
m_Spectra = Spectra() | |||
if not( isinstance(m_Spectra, Spectra) ): | |||
raise ValueError, "in SpectraReader, m_Spectra must be an Spectra class object" | |||
|
r30 | ||
|
r59 | self.m_DataObj = m_Spectra | |
|
r36 | ||
|
r79 | self.data_spc = None | |
self.data_cspc = None | |||
self.data_dc = None | |||
self.pts2read_SelfSpectra = 0 | |||
self.pts2read_CrossSpectra = 0 | |||
|
r86 | self.pts2read_DCs = 0 | |
|
r79 | ||
self.nChannels = 0 | |||
self.nPairs = 0 | |||
self.ext = ".pdata" | |||
self.optchar = "P" | |||
###################### | |||
self.m_BasicHeader = BasicHeader() | |||
self.m_SystemHeader = SystemHeader() | |||
self.m_RadarControllerHeader = RadarControllerHeader() | |||
self.m_ProcessingHeader = ProcessingHeader() | |||
self.online = 0 | |||
self.fp = None | |||
self.fileSizeByHeader = None | |||
self.filenameList = [] | |||
self.filename = None | |||
self.fileSize = None | |||
self.firstHeaderSize = 0 | |||
self.basicHeaderSize = 24 | |||
self.dataType = None | |||
self.maxTimeStep = 30 | |||
self.flagNoMoreFiles = 0 | |||
self.set = 0 | |||
self.path = None | |||
self.delay = 3 #seconds | |||
self.nTries = 3 #quantity tries | |||
self.nFiles = 3 #number of files for searching | |||
|
r89 | self.nReadBlocks = 0 | |
|
r79 | ||
self.flagIsNewFile = 1 | |||
self.ippSeconds = 0 | |||
self.flagResetProcessing = 0 | |||
self.flagIsNewBlock = 0 | |||
|
r89 | self.nTotalBlocks = 0 | |
|
r79 | ||
self.blocksize = 0 | |||
|
r86 | #pairList = None | |
channelList = None | |||
|
r79 | ||
|
r30 | ||
|
r59 | def __hasNotDataInBuffer(self): | |
|
r53 | return 1 | |
|
r30 | ||
|
r59 | ||
|
r53 | def getBlockDimension(self): | |
|
r59 | """ | |
Obtiene la cantidad de puntos a leer por cada bloque de datos | |||
|
r24 | ||
|
r59 | Affected: | |
|
r79 | self.nChannels | |
self.nPairs | |||
|
r59 | self.pts2read_SelfSpectra | |
self.pts2read_CrossSpectra | |||
self.pts2read_DCchannels | |||
self.blocksize | |||
|
r79 | self.m_DataObj.nChannels | |
self.m_DataObj.nPairs | |||
|
r59 | ||
Return: | |||
None | |||
""" | |||
|
r79 | self.nChannels = 0 | |
self.nPairs = 0 | |||
|
r89 | self.pairList = [] | |
|
r28 | ||
|
r30 | for i in range( 0, self.m_ProcessingHeader.totalSpectra*2, 2 ): | |
if self.m_ProcessingHeader.spectraComb[i] == self.m_ProcessingHeader.spectraComb[i+1]: | |||
|
r79 | self.nChannels = self.nChannels + 1 #par de canales iguales | |
|
r30 | else: | |
|
r79 | self.nPairs = self.nPairs + 1 #par de canales diferentes | |
|
r89 | self.pairList.append( (self.m_ProcessingHeader.spectraComb[i], self.m_ProcessingHeader.spectraComb[i+1]) ) | |
|
r86 | ||
|
r62 | pts2read = self.m_ProcessingHeader.numHeights * self.m_ProcessingHeader.profilesPerBlock | |
|
r79 | self.pts2read_SelfSpectra = int( self.nChannels * pts2read ) | |
self.pts2read_CrossSpectra = int( self.nPairs * pts2read ) | |||
|
r62 | self.pts2read_DCchannels = int( self.m_SystemHeader.numChannels * self.m_ProcessingHeader.numHeights ) | |
|
r24 | ||
|
r89 | self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels | |
|
r24 | ||
|
r86 | self.channelList = numpy.arange( self.nChannels ) | |
|
r59 | ||
|
r24 | ||
|
r53 | def readBlock(self): | |
|
r24 | """ | |
|
r30 | Lee el bloque de datos desde la posicion actual del puntero del archivo | |
|
r53 | (self.fp) y actualiza todos los parametros relacionados al bloque de datos | |
|
r29 | (metadata + data). La data leida es almacenada en el buffer y el contador del buffer | |
es seteado a 0 | |||
|
r24 | ||
|
r30 | Return: None | |
|
r24 | ||
Variables afectadas: | |||
|
r53 | self.datablockIndex | |
self.flagIsNewFile | |||
|
r24 | self.flagIsNewBlock | |
|
r89 | self.nTotalBlocks | |
|
r53 | self.data_spc | |
self.data_cspc | |||
self.data_dc | |||
|
r47 | ||
Exceptions: | |||
Si un bloque leido no es un bloque valido | |||
|
r24 | """ | |
|
r47 | blockOk_flag = False | |
|
r53 | fpointer = self.fp.tell() | |
|
r33 | ||
|
r53 | spc = numpy.fromfile( self.fp, self.dataType[0], self.pts2read_SelfSpectra ) | |
cspc = numpy.fromfile( self.fp, self.dataType, self.pts2read_CrossSpectra ) | |||
dc = numpy.fromfile( self.fp, self.dataType, self.pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) ) | |||
|
r47 | ||
try: | |||
|
r79 | spc = spc.reshape( (self.nChannels, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D | |
if self.nPairs != 0: | |||
cspc = cspc.reshape( (self.nPairs, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D | |||
else: | |||
cspc = None | |||
|
r47 | dc = dc.reshape( (self.m_SystemHeader.numChannels, self.m_ProcessingHeader.numHeights) ) #transforma a un arreglo 2D | |
except: | |||
print "Data file %s is invalid" % self.filename | |||
return 0 | |||
|
r26 | ||
|
r30 | if not( self.m_ProcessingHeader.shif_fft ): | |
spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones | |||
|
r79 | if cspc != None: | |
cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones | |||
|
r26 | ||
|
r79 | spc = numpy.transpose( spc, (0,2,1) ) | |
if cspc != None: cspc = numpy.transpose( cspc, (0,2,1) ) | |||
|
r30 | ||
|
r53 | self.data_spc = spc | |
|
r79 | if cspc != None: | |
self.data_cspc = cspc['real'] + cspc['imag']*1j | |||
else: | |||
self.data_cspc = None | |||
|
r53 | self.data_dc = dc['real'] + dc['imag']*1j | |
|
r30 | ||
|
r59 | self.datablockIndex = 0 | |
|
r53 | self.flagIsNewFile = 0 | |
|
r24 | self.flagIsNewBlock = 1 | |
|
r47 | ||
|
r89 | self.nTotalBlocks += 1 | |
|
r24 | self.nReadBlocks += 1 | |
|
r30 | ||
|
r47 | return 1 | |
|
r30 | ||
|
r59 | ||
|
r47 | def getData(self): | |
|
r24 | """ | |
|
r30 | Copia el buffer de lectura a la clase "Spectra", | |
|
r29 | con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de | |
lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock" | |||
|
r24 | ||
Return: | |||
|
r30 | 0 : Si no hay mas archivos disponibles | |
1 : Si hizo una buena copia del buffer | |||
|
r24 | ||
|
r36 | Affected: | |
|
r53 | self.m_DataObj | |
self.datablockIndex | |||
|
r30 | self.flagResetProcessing | |
self.flagIsNewBlock | |||
|
r24 | """ | |
|
r59 | if self.flagNoMoreFiles: return 0 | |
|
r24 | self.flagResetProcessing = 0 | |
self.flagIsNewBlock = 0 | |||
|
r59 | if self.__hasNotDataInBuffer(): | |
|
r33 | ||
|
r47 | if not( self.readNextBlock() ): | |
return 0 | |||
|
r24 | ||
|
r89 | self.updateDataHeader() | |
|
r24 | ||
|
r59 | if self.flagNoMoreFiles == 1: | |
|
r24 | print 'Process finished' | |
return 0 | |||
#data es un numpy array de 3 dmensiones (perfiles, alturas y canales) | |||
|
r30 | ||
|
r53 | if self.data_dc == None: | |
|
r59 | self.m_DataObj.flagNoData = True | |
|
r47 | return 0 | |
|
r53 | self.m_DataObj.flagNoData = False | |
self.m_DataObj.flagResetProcessing = self.flagResetProcessing | |||
|
r24 | ||
|
r53 | self.m_DataObj.data_spc = self.data_spc | |
self.m_DataObj.data_cspc = self.data_cspc | |||
self.m_DataObj.data_dc = self.data_dc | |||
|
r30 | ||
|
r24 | return 1 | |
|
r59 | ||
class SpectraWriter(JRODataWriter): | |||
|
r30 | """ | |
Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura | |||
de los datos siempre se realiza por bloques. | |||
""" | |||
|
r24 | ||
|
r59 | m_DataObj = None | |
shape_spc_Buffer = None | |||
shape_cspc_Buffer = None | |||
shape_dc_Buffer = None | |||
|
r79 | data_spc = None | |
data_cspc = None | |||
data_dc = None | |||
|
r59 | ||
def __init__(self, m_Spectra=None): | |||
|
r30 | """ | |
Inicializador de la clase SpectraWriter para la escritura de datos de espectros. | |||
Affected: | |||
|
r53 | self.m_DataObj | |
|
r30 | self.m_BasicHeader | |
self.m_SystemHeader | |||
self.m_RadarControllerHeader | |||
self.m_ProcessingHeader | |||
Return: None | |||
""" | |||
|
r24 | if m_Spectra == None: | |
m_Spectra = Spectra() | |||
|
r30 | ||
|
r59 | if not( isinstance(m_Spectra, Spectra) ): | |
raise ValueError, "in SpectraReader, m_Spectra must be an Spectra class object" | |||
|
r53 | self.m_DataObj = m_Spectra | |
|
r47 | ||
|
r79 | self.ext = ".pdata" | |
self.optchar = "P" | |||
self.shape_spc_Buffer = None | |||
self.shape_cspc_Buffer = None | |||
self.shape_dc_Buffer = None | |||
self.data_spc = None | |||
self.data_cspc = None | |||
self.data_dc = None | |||
#################################### | |||
self.fp = None | |||
|
r89 | self.nWriteBlocks = 0 | |
|
r79 | ||
self.flagIsNewFile = 1 | |||
|
r89 | self.nTotalBlocks = 0 | |
|
r79 | ||
self.flagIsNewBlock = 0 | |||
self.flagNoMoreFiles = 0 | |||
self.setFile = None | |||
self.dataType = None | |||
self.path = None | |||
self.noMoreFiles = 0 | |||
self.filename = None | |||
self.m_BasicHeader= BasicHeader() | |||
self.m_SystemHeader = SystemHeader() | |||
self.m_RadarControllerHeader = RadarControllerHeader() | |||
self.m_ProcessingHeader = ProcessingHeader() | |||
|
r59 | ||
|
r53 | def hasAllDataInBuffer(self): | |
return 1 | |||
|
r59 | ||
|
r30 | ||
|
r53 | def setBlockDimension(self): | |
|
r59 | """ | |
Obtiene las formas dimensionales del los subbloques de datos que componen un bloque | |||
Affected: | |||
self.shape_spc_Buffer | |||
self.shape_cspc_Buffer | |||
self.shape_dc_Buffer | |||
|
r30 | ||
|
r59 | Return: None | |
""" | |||
|
r79 | self.shape_spc_Buffer = (self.m_DataObj.nChannels, | |
|
r53 | self.m_ProcessingHeader.numHeights, | |
self.m_ProcessingHeader.profilesPerBlock) | |||
|
r30 | ||
|
r79 | self.shape_cspc_Buffer = (self.m_DataObj.nPairs, | |
|
r53 | self.m_ProcessingHeader.numHeights, | |
self.m_ProcessingHeader.profilesPerBlock) | |||
|
r24 | ||
|
r53 | self.shape_dc_Buffer = (self.m_SystemHeader.numChannels, | |
self.m_ProcessingHeader.numHeights) | |||
|
r30 | ||
|
r47 | ||
|
r53 | def writeBlock(self): | |
|
r30 | """ | |
Escribe el buffer en el file designado | |||
Affected: | |||
|
r53 | self.data_spc | |
self.data_cspc | |||
self.data_dc | |||
self.flagIsNewFile | |||
|
r30 | self.flagIsNewBlock | |
|
r89 | self.nTotalBlocks | |
self.nWriteBlocks | |||
|
r30 | ||
Return: None | |||
""" | |||
|
r79 | ||
spc = numpy.transpose( self.data_spc, (0,2,1) ) | |||
|
r30 | if not( self.m_ProcessingHeader.shif_fft ): | |
spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones | |||
data = spc.reshape((-1)) | |||
|
r53 | data.tofile(self.fp) | |
|
r30 | ||
|
r79 | if self.data_cspc != None: | |
data = numpy.zeros( self.shape_cspc_Buffer, self.dataType ) | |||
cspc = numpy.transpose( self.data_cspc, (0,2,1) ) | |||
if not( self.m_ProcessingHeader.shif_fft ): | |||
cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones | |||
data['real'] = cspc.real | |||
data['imag'] = cspc.imag | |||
data = data.reshape((-1)) | |||
data.tofile(self.fp) | |||
|
r30 | ||
|
r53 | data = numpy.zeros( self.shape_dc_Buffer, self.dataType ) | |
dc = self.data_dc | |||
|
r30 | data['real'] = dc.real | |
data['imag'] = dc.imag | |||
data = data.reshape((-1)) | |||
|
r53 | data.tofile(self.fp) | |
|
r30 | ||
|
r53 | self.data_spc.fill(0) | |
self.data_dc.fill(0) | |||
|
r86 | if self.data_cspc != None: | |
self.data_cspc.fill(0) | |||
|
r24 | ||
|
r53 | self.flagIsNewFile = 0 | |
|
r24 | self.flagIsNewBlock = 1 | |
|
r89 | self.nTotalBlocks += 1 | |
|
r24 | self.nWriteBlocks += 1 | |
|
r59 | ||
|
r47 | def putData(self): | |
|
r30 | """ | |
Setea un bloque de datos y luego los escribe en un file | |||
Affected: | |||
|
r53 | self.data_spc | |
self.data_cspc | |||
self.data_dc | |||
|
r30 | ||
Return: | |||
0 : Si no hay data o no hay mas files que puedan escribirse | |||
1 : Si se escribio la data de un bloque en un file | |||
""" | |||
|
r24 | self.flagIsNewBlock = 0 | |
|
r53 | if self.m_DataObj.flagNoData: | |
|
r30 | return 0 | |
|
r24 | ||
|
r53 | if self.m_DataObj.flagResetProcessing: | |
self.data_spc.fill(0) | |||
self.data_cspc.fill(0) | |||
self.data_dc.fill(0) | |||
self.setNextFile() | |||
|
r24 | ||
|
r53 | self.data_spc = self.m_DataObj.data_spc | |
self.data_cspc = self.m_DataObj.data_cspc | |||
self.data_dc = self.m_DataObj.data_dc | |||
|
r24 | ||
|
r59 | # #self.m_ProcessingHeader.dataBlocksPerFile) | |
|
r79 | if self.hasAllDataInBuffer(): | |
|
r89 | self.getDataHeader() | |
|
r29 | self.writeNextBlock() | |
|
r59 | if self.flagNoMoreFiles: | |
|
r30 | #print 'Process finished' | |
return 0 | |||
|
r24 | ||
|
r30 | return 1 |