SpectraIO.py
501 lines
| 15.9 KiB
| text/x-python
|
PythonLexer
|
r24 | ''' | |
File: SpectraIO.py | |||
Created on 20/02/2012 | |||
@author $Author$ | |||
@version $Id$ | |||
''' | |||
import os, sys | |||
import numpy | |||
import glob | |||
import fnmatch | |||
import time, datetime | |||
path = os.path.split(os.getcwd())[0] | |||
sys.path.append(path) | |||
|
r49 | from Model.JROHeader import * | |
|
r24 | from Model.Spectra import Spectra | |
|
r49 | from DataIO import JRODataReader | |
from DataIO import JRODataWriter | |||
|
r53 | from DataIO import isNumber | |
|
r24 | ||
|
r49 | class SpectraReader( JRODataReader ): | |
|
r30 | """ | |
|
r29 | Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura | |
|
r30 | de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones) | |
son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel. | |||
|
r29 | Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader, | |
RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la | |||
cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de | |||
datos desde el "buffer" cada vez que se ejecute el metodo "getData". | |||
|
r30 | Example: | |
|
r24 | dpath = "/home/myuser/data" | |
startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0) | |||
endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0) | |||
readerObj = SpectraReader() | |||
readerObj.setup(dpath, startTime, endTime) | |||
while(True): | |||
readerObj.getData() | |||
print readerObj.m_Spectra.data | |||
if readerObj.noMoreFiles: | |||
break | |||
""" | |||
|
r30 | ||
|
r47 | def __init__(self,m_Spectra=None): | |
|
r30 | """ | |
|
r29 | Inicializador de la clase SpectraReader para la lectura de datos de espectros. | |
|
r30 | ||
Inputs: | |||
|
r24 | m_Spectra : Objeto de la clase Spectra. Este objeto sera utilizado para | |
almacenar un perfil de datos cada vez que se haga un requerimiento | |||
(getData). El perfil sera obtenido a partir del buffer de datos, | |||
si el buffer esta vacio se hara un nuevo proceso de lectura de un | |||
bloque de datos. | |||
Si este parametro no es pasado se creara uno internamente. | |||
|
r30 | ||
Affected: | |||
|
r53 | self.m_DataObj | |
|
r24 | self.m_BasicHeader | |
self.m_SystemHeader | |||
self.m_RadarControllerHeader | |||
self.m_ProcessingHeader | |||
|
r30 | ||
Return : None | |||
|
r24 | """ | |
if m_Spectra == None: | |||
m_Spectra = Spectra() | |||
if not( isinstance(m_Spectra, Spectra) ): | |||
raise ValueError, "in SpectraReader, m_Spectra must be an Spectra class object" | |||
|
r53 | self.m_DataObj = m_Spectra | |
|
r24 | ||
self.m_BasicHeader = BasicHeader() | |||
self.m_SystemHeader = SystemHeader() | |||
self.m_RadarControllerHeader = RadarControllerHeader() | |||
self.m_ProcessingHeader = ProcessingHeader() | |||
|
r53 | self.fp = None | |
|
r24 | ||
|
r53 | self.idFile = None | |
|
r24 | ||
|
r53 | self.startDateTime = None | |
|
r24 | ||
|
r53 | self.endDateTime = None | |
|
r24 | ||
|
r53 | self.dataType = None | |
|
r24 | ||
|
r53 | self.fileSizeByHeader = 0 | |
|
r24 | ||
|
r53 | self.pathList = [] | |
|
r24 | ||
self.filenameList = [] | |||
|
r53 | self.lastUTTime = 0 | |
|
r24 | ||
|
r53 | self.maxTimeStep = 30 | |
|
r24 | ||
|
r53 | self.flagIsNewFile = 0 | |
|
r24 | ||
self.flagResetProcessing = 0 | |||
self.flagIsNewBlock = 0 | |||
self.noMoreFiles = 0 | |||
self.nReadBlocks = 0 | |||
self.online = 0 | |||
self.firstHeaderSize = 0 | |||
self.basicHeaderSize = 24 | |||
self.filename = None | |||
self.fileSize = None | |||
|
r53 | self.data_spc = None | |
self.data_cspc = None | |||
self.data_dc = None | |||
|
r30 | ||
self.nChannels = 0 | |||
self.nPairs = 0 | |||
|
r53 | self.pts2read_SelfSpectra = 0 | |
self.pts2read_CrossSpectra = 0 | |||
self.pts2read_DCchannels = 0 | |||
self.blocksize = 0 | |||
|
r24 | ||
|
r53 | self.datablockIndex = 0 | |
|
r24 | ||
|
r53 | self.ippSeconds = 0 | |
|
r24 | ||
self.nSelfChannels = 0 | |||
self.nCrossPairs = 0 | |||
|
r36 | self.datablock_id = 9999 | |
|
r53 | self.delay = 2 #seconds | |
self.nTries = 3 #quantity tries | |||
self.nFiles = 3 #number of files for searching | |||
self.year = 0 | |||
self.doy = 0 | |||
self.set = 0 | |||
self.ext = ".pdata" | |||
self.path = None | |||
self.optchar = "P" | |||
|
r47 | self.nBlocks = 0 | |
|
r30 | ||
|
r53 | def hasNotDataInBuffer(self): | |
return 1 | |||
|
r30 | ||
|
r53 | def getBlockDimension(self): | |
|
r24 | ||
|
r30 | self.nChannels = 0 | |
self.nPairs = 0 | |||
|
r28 | ||
|
r30 | for i in range( 0, self.m_ProcessingHeader.totalSpectra*2, 2 ): | |
if self.m_ProcessingHeader.spectraComb[i] == self.m_ProcessingHeader.spectraComb[i+1]: | |||
self.nChannels = self.nChannels + 1 | |||
else: | |||
self.nPairs = self.nPairs + 1 | |||
pts2read = self.m_ProcessingHeader.profilesPerBlock * self.m_ProcessingHeader.numHeights | |||
|
r53 | self.pts2read_SelfSpectra = int( pts2read * self.nChannels ) | |
self.pts2read_CrossSpectra = int( pts2read * self.nPairs ) | |||
self.pts2read_DCchannels = int( self.m_ProcessingHeader.numHeights * self.m_SystemHeader.numChannels ) | |||
|
r24 | ||
|
r53 | self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels | |
|
r24 | ||
|
r53 | self.m_DataObj.nChannels = self.nChannels | |
self.m_DataObj.nPairs = self.nPairs | |||
|
r24 | ||
|
r53 | def readBlock(self): | |
|
r24 | """ | |
|
r30 | Lee el bloque de datos desde la posicion actual del puntero del archivo | |
|
r53 | (self.fp) y actualiza todos los parametros relacionados al bloque de datos | |
|
r29 | (metadata + data). La data leida es almacenada en el buffer y el contador del buffer | |
es seteado a 0 | |||
|
r24 | ||
|
r30 | Return: None | |
|
r24 | ||
Variables afectadas: | |||
|
r53 | self.datablockIndex | |
self.flagIsNewFile | |||
|
r24 | self.flagIsNewBlock | |
self.nReadBlocks | |||
|
r53 | self.data_spc | |
self.data_cspc | |||
self.data_dc | |||
|
r47 | ||
Exceptions: | |||
Si un bloque leido no es un bloque valido | |||
|
r24 | """ | |
|
r47 | #self.datablock_id = 0 | |
|
r53 | #self.flagIsNewFile = 0 | |
|
r47 | #self.flagIsNewBlock = 1 | |
|
r24 | ||
|
r47 | blockOk_flag = False | |
|
r53 | fpointer = self.fp.tell() | |
|
r33 | ||
|
r53 | spc = numpy.fromfile( self.fp, self.dataType[0], self.pts2read_SelfSpectra ) | |
cspc = numpy.fromfile( self.fp, self.dataType, self.pts2read_CrossSpectra ) | |||
dc = numpy.fromfile( self.fp, self.dataType, self.pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) ) | |||
|
r33 | ||
|
r35 | if self.online: | |
|
r53 | if (spc.size + cspc.size + dc.size) != self.blocksize: | |
for nTries in range( self.nTries ): | |||
|
r47 | #nTries = 0 | |
|
r53 | #while( nTries < self.nTries ): | |
|
r47 | #nTries += 1 | |
print "\tWaiting for the next block, try %03d ..." % (nTries+1) | |||
|
r53 | time.sleep( self.delay ) | |
self.fp.seek( fpointer ) | |||
fpointer = self.fp.tell() | |||
spc = numpy.fromfile( self.fp, self.dataType[0], self.pts2read_SelfSpectra ) | |||
cspc = numpy.fromfile( self.fp, self.dataType, self.pts2read_CrossSpectra ) | |||
dc = numpy.fromfile( self.fp, self.dataType, self.pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) ) | |||
|
r47 | ||
|
r53 | if (spc.size + cspc.size + dc.size) == self.blocksize: | |
|
r47 | blockOk_flag = True | |
|
r35 | break | |
|
r53 | #if (spc.size + cspc.size + dc.size) == self.blocksize: | |
|
r47 | # nTries = 0 | |
# break | |||
if not( blockOk_flag ): | |||
return 0 | |||
#if nTries > 0: | |||
# return 0 | |||
try: | |||
spc = spc.reshape( (self.nChannels, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D | |||
cspc = cspc.reshape( (self.nPairs, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D | |||
dc = dc.reshape( (self.m_SystemHeader.numChannels, self.m_ProcessingHeader.numHeights) ) #transforma a un arreglo 2D | |||
except: | |||
print "Data file %s is invalid" % self.filename | |||
return 0 | |||
|
r26 | ||
|
r30 | if not( self.m_ProcessingHeader.shif_fft ): | |
spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones | |||
cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones | |||
|
r26 | ||
|
r30 | spc = numpy.transpose( spc, (0,2,1) ) | |
cspc = numpy.transpose( cspc, (0,2,1) ) | |||
|
r26 | #dc = numpy.transpose(dc, (0,2,1)) | |
|
r30 | ||
|
r53 | self.data_spc = spc | |
self.data_cspc = cspc['real'] + cspc['imag']*1j | |||
self.data_dc = dc['real'] + dc['imag']*1j | |||
|
r30 | ||
|
r47 | self.datablock_id = 0 | |
|
r53 | self.flagIsNewFile = 0 | |
|
r24 | self.flagIsNewBlock = 1 | |
|
r47 | ||
|
r24 | self.nReadBlocks += 1 | |
|
r47 | self.nBlocks += 1 | |
|
r30 | ||
|
r47 | return 1 | |
|
r30 | ||
|
r47 | def getData(self): | |
|
r24 | """ | |
|
r30 | Copia el buffer de lectura a la clase "Spectra", | |
|
r29 | con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de | |
lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock" | |||
|
r24 | ||
Return: | |||
|
r30 | 0 : Si no hay mas archivos disponibles | |
1 : Si hizo una buena copia del buffer | |||
|
r24 | ||
|
r36 | Affected: | |
|
r53 | self.m_DataObj | |
self.datablockIndex | |||
|
r30 | self.flagResetProcessing | |
self.flagIsNewBlock | |||
|
r24 | """ | |
self.flagResetProcessing = 0 | |||
self.flagIsNewBlock = 0 | |||
|
r53 | if self.hasNotDataInBuffer(): | |
|
r33 | ||
|
r47 | if not( self.readNextBlock() ): | |
|
r53 | self.setNextFile() | |
|
r47 | return 0 | |
|
r24 | ||
|
r53 | self.m_DataObj.m_BasicHeader = self.m_BasicHeader.copy() | |
self.m_DataObj.m_ProcessingHeader = self.m_ProcessingHeader.copy() | |||
self.m_DataObj.m_RadarControllerHeader = self.m_RadarControllerHeader.copy() | |||
self.m_DataObj.m_SystemHeader = self.m_SystemHeader.copy() | |||
self.m_DataObj.heights = self.heights | |||
self.m_DataObj.dataType = self.dataType | |||
|
r24 | ||
if self.noMoreFiles == 1: | |||
print 'Process finished' | |||
return 0 | |||
#data es un numpy array de 3 dmensiones (perfiles, alturas y canales) | |||
|
r30 | ||
|
r53 | if self.data_dc == None: | |
|
r47 | self.m_Voltage.flagNoData = True | |
return 0 | |||
|
r53 | self.m_DataObj.flagNoData = False | |
self.m_DataObj.flagResetProcessing = self.flagResetProcessing | |||
|
r24 | ||
|
r53 | self.m_DataObj.data_spc = self.data_spc | |
self.m_DataObj.data_cspc = self.data_cspc | |||
self.m_DataObj.data_dc = self.data_dc | |||
|
r24 | ||
#call setData - to Data Object | |||
|
r30 | #self.datablock_id += 1 | |
#self.idProfile += 1 | |||
|
r24 | return 1 | |
|
r49 | class SpectraWriter( JRODataWriter ): | |
|
r30 | """ | |
Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura | |||
de los datos siempre se realiza por bloques. | |||
""" | |||
|
r24 | ||
|
r47 | def __init__(self,m_Spectra=None): | |
|
r30 | """ | |
Inicializador de la clase SpectraWriter para la escritura de datos de espectros. | |||
Affected: | |||
|
r53 | self.m_DataObj | |
|
r30 | self.m_BasicHeader | |
self.m_SystemHeader | |||
self.m_RadarControllerHeader | |||
self.m_ProcessingHeader | |||
Return: None | |||
""" | |||
|
r24 | if m_Spectra == None: | |
m_Spectra = Spectra() | |||
|
r30 | ||
|
r53 | self.m_DataObj = m_Spectra | |
|
r30 | ||
|
r53 | self.fp = None | |
|
r30 | ||
|
r53 | self.format = None | |
|
r24 | ||
|
r53 | self.blocksCounter = 0 | |
|
r24 | ||
|
r53 | self.setFile = None | |
|
r24 | ||
|
r53 | self.flagIsNewFile = 1 | |
|
r24 | ||
|
r53 | self.dataType = None | |
|
r24 | ||
|
r53 | self.ext = ".pdata" | |
|
r24 | ||
|
r53 | self.path = None | |
self.optchar = "P" | |||
self.shape_spc_Buffer = None | |||
self.shape_cspc_Buffer = None | |||
self.shape_dc_Buffer = None | |||
|
r30 | ||
|
r24 | self.nWriteBlocks = 0 | |
self.flagIsNewBlock = 0 | |||
self.noMoreFiles = 0 | |||
self.filename = None | |||
self.m_BasicHeader= BasicHeader() | |||
self.m_SystemHeader = SystemHeader() | |||
self.m_RadarControllerHeader = RadarControllerHeader() | |||
self.m_ProcessingHeader = ProcessingHeader() | |||
|
r30 | ||
|
r53 | self.data_spc = None | |
self.data_cspc = None | |||
self.data_dc = None | |||
|
r47 | ||
|
r53 | def hasAllDataInBuffer(self): | |
return 1 | |||
|
r30 | ||
|
r53 | def setBlockDimension(self): | |
|
r30 | ||
|
r53 | self.shape_spc_Buffer = (self.m_DataObj.nChannels, | |
self.m_ProcessingHeader.numHeights, | |||
self.m_ProcessingHeader.profilesPerBlock) | |||
|
r30 | ||
|
r53 | self.shape_cspc_Buffer = (self.m_DataObj.nPairs, | |
self.m_ProcessingHeader.numHeights, | |||
self.m_ProcessingHeader.profilesPerBlock) | |||
|
r24 | ||
|
r53 | self.shape_dc_Buffer = (self.m_SystemHeader.numChannels, | |
self.m_ProcessingHeader.numHeights) | |||
|
r30 | ||
|
r47 | ||
|
r53 | def writeBlock(self): | |
|
r30 | """ | |
Escribe el buffer en el file designado | |||
Affected: | |||
|
r53 | self.data_spc | |
self.data_cspc | |||
self.data_dc | |||
self.flagIsNewFile | |||
|
r30 | self.flagIsNewBlock | |
self.nWriteBlocks | |||
|
r53 | self.blocksCounter | |
|
r30 | ||
Return: None | |||
""" | |||
|
r53 | spc = numpy.transpose( self.data_spc, (0,2,1) ) | |
|
r30 | if not( self.m_ProcessingHeader.shif_fft ): | |
spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones | |||
data = spc.reshape((-1)) | |||
|
r53 | data.tofile(self.fp) | |
|
r30 | ||
|
r53 | data = numpy.zeros( self.shape_cspc_Buffer, self.dataType ) | |
cspc = numpy.transpose( self.data_cspc, (0,2,1) ) | |||
|
r30 | if not( self.m_ProcessingHeader.shif_fft ): | |
cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones | |||
data['real'] = cspc.real | |||
data['imag'] = cspc.imag | |||
data = data.reshape((-1)) | |||
|
r53 | data.tofile(self.fp) | |
|
r30 | ||
|
r53 | data = numpy.zeros( self.shape_dc_Buffer, self.dataType ) | |
dc = self.data_dc | |||
|
r30 | data['real'] = dc.real | |
data['imag'] = dc.imag | |||
data = data.reshape((-1)) | |||
|
r53 | data.tofile(self.fp) | |
|
r30 | ||
|
r53 | self.data_spc.fill(0) | |
self.data_cspc.fill(0) | |||
self.data_dc.fill(0) | |||
|
r24 | ||
|
r53 | self.flagIsNewFile = 0 | |
|
r24 | self.flagIsNewBlock = 1 | |
self.nWriteBlocks += 1 | |||
|
r53 | self.blocksCounter += 1 | |
|
r24 | ||
|
r47 | def putData(self): | |
|
r30 | """ | |
Setea un bloque de datos y luego los escribe en un file | |||
Affected: | |||
|
r53 | self.data_spc | |
self.data_cspc | |||
self.data_dc | |||
|
r30 | ||
Return: | |||
0 : Si no hay data o no hay mas files que puedan escribirse | |||
1 : Si se escribio la data de un bloque en un file | |||
""" | |||
|
r24 | self.flagIsNewBlock = 0 | |
|
r53 | if self.m_DataObj.flagNoData: | |
|
r30 | return 0 | |
|
r24 | ||
|
r53 | if self.m_DataObj.flagResetProcessing: | |
self.data_spc.fill(0) | |||
self.data_cspc.fill(0) | |||
self.data_dc.fill(0) | |||
self.setNextFile() | |||
|
r24 | ||
|
r53 | self.data_spc = self.m_DataObj.data_spc | |
self.data_cspc = self.m_DataObj.data_cspc | |||
self.data_dc = self.m_DataObj.data_dc | |||
|
r24 | ||
|
r30 | if True: | |
|
r53 | self.getHeader() | |
|
r29 | self.writeNextBlock() | |
|
r24 | if self.noMoreFiles: | |
|
r30 | #print 'Process finished' | |
return 0 | |||
|
r24 | ||
|
r30 | return 1 |