VoltageReader.py
601 lines
| 18.9 KiB
| text/x-python
|
PythonLexer
|
r2 | ''' | ||
Created on 23/01/2012 | ||||
@author: danielangelsuarezmunoz | ||||
''' | ||||
from DataReader import DataReader | ||||
import numpy | ||||
import os.path | ||||
import glob | ||||
import fnmatch | ||||
import time | ||||
import datetime | ||||
class PROCFLAG: | ||||
COHERENT_INTEGRATION = numpy.uint32(0x00000001) | ||||
DECODE_DATA = numpy.uint32(0x00000002) | ||||
SPECTRA_CALC = numpy.uint32(0x00000004) | ||||
INCOHERENT_INTEGRATION = numpy.uint32(0x00000008) | ||||
POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010) | ||||
SHIFT_FFT_DATA = numpy.uint32(0x00000020) | ||||
DATATYPE_CHAR = numpy.uint32(0x00000040) | ||||
DATATYPE_SHORT = numpy.uint32(0x00000080) | ||||
DATATYPE_LONG = numpy.uint32(0x00000100) | ||||
DATATYPE_INT64 = numpy.uint32(0x00000200) | ||||
DATATYPE_FLOAT = numpy.uint32(0x00000400) | ||||
DATATYPE_DOUBLE = numpy.uint32(0x00000800) | ||||
DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000) | ||||
DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000) | ||||
DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000) | ||||
SAVE_CHANNELS_DC = numpy.uint32(0x00008000) | ||||
DEFLIP_DATA = numpy.uint32(0x00010000) | ||||
DEFINE_PROCESS_CODE = numpy.uint32(0x00020000) | ||||
ACQ_SYS_NATALIA = numpy.uint32(0x00040000) | ||||
ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000) | ||||
ACQ_SYS_ADRXD = numpy.uint32(0x000C0000) | ||||
ACQ_SYS_JULIA = numpy.uint32(0x00100000) | ||||
ACQ_SYS_XXXXXX = numpy.uint32(0x00140000) | ||||
EXP_NAME_ESP = numpy.uint32(0x00200000) | ||||
CHANNEL_NAMES_ESP = numpy.uint32(0x00400000) | ||||
OPERATION_MASK = numpy.uint32(0x0000003F) | ||||
DATATYPE_MASK = numpy.uint32(0x00000FC0) | ||||
DATAARRANGE_MASK = numpy.uint32(0x00007000) | ||||
ACQ_SYS_MASK = numpy.uint32(0x001C0000) | ||||
|
r5 | class BasicHeader(): | ||
|
r2 | |||
size = 0 | ||||
version = 0 | ||||
dataBlock = 0 | ||||
utc = 0 | ||||
miliSecond = 0 | ||||
timeZone = 0 | ||||
dstFlag = 0 | ||||
errorCount = 0 | ||||
struct = numpy.dtype([ | ||||
('nSize','<u4'), | ||||
('nVersion','<u2'), | ||||
('nDataBlockId','<u4'), | ||||
('nUtime','<u4'), | ||||
('nMilsec','<u2'), | ||||
('nTimezone','<i2'), | ||||
('nDstflag','<i2'), | ||||
('nErrorCount','<u4') | ||||
]) | ||||
def __init__(self): | ||||
pass | ||||
def read(self, fp): | ||||
header = numpy.fromfile(fp, self.struct,1) | ||||
self.size = header['nSize'][0] | ||||
self.version = header['nVersion'][0] | ||||
self.dataBlock = header['nDataBlockId'][0] | ||||
self.utc = header['nUtime'][0] | ||||
self.miliSecond = header['nMilsec'][0] | ||||
self.timeZone = header['nTimezone'][0] | ||||
self.dstFlag = header['nDstflag'][0] | ||||
self.errorCount = header['nErrorCount'][0] | ||||
return 1 | ||||
|
r5 | class SystemHeader(): | ||
|
r2 | size = 0 | ||
numSamples = 0 | ||||
numProfiles = 0 | ||||
numChannels = 0 | ||||
adcResolution = 0 | ||||
pciDioBusWidth = 0 | ||||
struct = numpy.dtype([ | ||||
('nSize','<u4'), | ||||
('nNumSamples','<u4'), | ||||
('nNumProfiles','<u4'), | ||||
('nNumChannels','<u4'), | ||||
('nADCResolution','<u4'), | ||||
('nPCDIOBusWidth','<u4'), | ||||
]) | ||||
def __init__(self): | ||||
pass | ||||
def read(self, fp): | ||||
|
r5 | header = numpy.fromfile(fp,self.struct,1) | ||
self.size = header['nSize'][0] | ||||
self.numSamples = header['nNumSamples'][0] | ||||
self.numProfiles = header['nNumProfiles'][0] | ||||
self.numChannels = header['nNumChannels'][0] | ||||
self.adcResolution = header['nADCResolution'][0] | ||||
self.pciDioBusWidth = header['nPCDIOBusWidth'][0] | ||||
|
r2 | |||
return 1 | ||||
|
r5 | class RadarControllerHeader(): | ||
|
r2 | size = 0 | ||
expType = 0 | ||||
nTx = 0 | ||||
ipp = 0 | ||||
txA = 0 | ||||
txB = 0 | ||||
numWindows = 0 | ||||
numTaus = 0 | ||||
codeType = 0 | ||||
line6Function = 0 | ||||
line5Fuction = 0 | ||||
fClock = 0 | ||||
prePulseBefore = 0 | ||||
prePulserAfter = 0 | ||||
rangeIpp = 0 | ||||
rangeTxA = 0 | ||||
rangeTxB = 0 | ||||
struct = numpy.dtype([ | ||||
('nSize','<u4'), | ||||
('nExpType','<u4'), | ||||
('nNTx','<u4'), | ||||
('fIpp','<f4'), | ||||
('fTxA','<f4'), | ||||
('fTxB','<f4'), | ||||
('nNumWindows','<u4'), | ||||
('nNumTaus','<u4'), | ||||
('nCodeType','<u4'), | ||||
('nLine6Function','<u4'), | ||||
('nLine5Function','<u4'), | ||||
('fClock','<f4'), | ||||
('nPrePulseBefore','<u4'), | ||||
('nPrePulseAfter','<u4'), | ||||
('sRangeIPP','<a20'), | ||||
('sRangeTxA','<a20'), | ||||
('sRangeTxB','<a20'), | ||||
]) | ||||
def __init__(self): | ||||
pass | ||||
def read(self, fp): | ||||
|
r5 | header = numpy.fromfile(fp,self.struct,1) | ||
self.size = header['nSize'][0] | ||||
self.expType = header['nExpType'][0] | ||||
self.nTx = header['nNTx'][0] | ||||
self.ipp = header['fIpp'][0] | ||||
self.txA = header['fTxA'][0] | ||||
self.txB = header['fTxB'][0] | ||||
self.numWindows = header['nNumWindows'][0] | ||||
self.numTaus = header['nNumTaus'][0] | ||||
self.codeType = header['nCodeType'][0] | ||||
self.line6Function = header['nLine6Function'][0] | ||||
self.line5Fuction = header['nLine5Function'][0] | ||||
self.fClock = header['fClock'][0] | ||||
self.prePulseBefore = header['nPrePulseBefore'][0] | ||||
self.prePulserAfter = header['nPrePulseAfter'][0] | ||||
self.rangeIpp = header['sRangeIPP'][0] | ||||
self.rangeTxA = header['sRangeTxA'][0] | ||||
self.rangeTxB = header['sRangeTxB'][0] | ||||
# jump Dynamic Radar Controller Header | ||||
jumpHeader = self.size - 116 | ||||
fp.seek(fp.tell() + jumpHeader) | ||||
|
r2 | |||
return 1 | ||||
|
r5 | class ProcessingHeader(): | ||
|
r2 | size = 0 | ||
dataType = 0 | ||||
blockSize = 0 | ||||
profilesPerBlock = 0 | ||||
dataBlocksPerFile = 0 | ||||
numWindows = 0 | ||||
processFlags = 0 | ||||
coherentInt = 0 | ||||
incoherentInt = 0 | ||||
totalSpectra = 0 | ||||
struct = numpy.dtype([ | ||||
('nSize','<u4'), | ||||
('nDataType','<u4'), | ||||
('nSizeOfDataBlock','<u4'), | ||||
('nProfilesperBlock','<u4'), | ||||
('nDataBlocksperFile','<u4'), | ||||
('nNumWindows','<u4'), | ||||
('nProcessFlags','<u4'), | ||||
('nCoherentIntegrations','<u4'), | ||||
('nIncoherentIntegrations','<u4'), | ||||
('nTotalSpectra','<u4') | ||||
]) | ||||
samplingWindow = 0 | ||||
structSamplingWindow = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')]) | ||||
numHeights = 0 | ||||
firstHeight = 0 | ||||
deltaHeight = 0 | ||||
samplesWin = 0 | ||||
spectraComb = 0 | ||||
numCode = 0 | ||||
codes = 0 | ||||
numBaud = 0 | ||||
def __init__(self): | ||||
pass | ||||
def read(self, fp): | ||||
|
r5 | header = numpy.fromfile(fp,self.struct,1) | ||
self.size = header['nSize'][0] | ||||
self.dataType = header['nDataType'][0] | ||||
self.blockSize = header['nSizeOfDataBlock'][0] | ||||
self.profilesPerBlock = header['nProfilesperBlock'][0] | ||||
self.dataBlocksPerFile = header['nDataBlocksperFile'][0] | ||||
self.numWindows = header['nNumWindows'][0] | ||||
self.processFlags = header['nProcessFlags'] | ||||
self.coherentInt = header['nCoherentIntegrations'][0] | ||||
self.incoherentInt = header['nIncoherentIntegrations'][0] | ||||
self.totalSpectra = header['nTotalSpectra'][0] | ||||
self.samplingWindow = numpy.fromfile(fp,self.structSamplingWindow,self.numWindows) | ||||
self.numHeights = numpy.sum(self.samplingWindow['nsa']) | ||||
self.firstHeight = self.samplingWindow['h0'] | ||||
self.deltaHeight = self.samplingWindow['dh'] | ||||
self.samplesWin = self.samplingWindow['nsa'] | ||||
self.spectraComb = numpy.fromfile(fp,'u1',2*self.totalSpectra) | ||||
if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE: | ||||
self.numCode = numpy.fromfile(fp,'<u4',1) | ||||
self.numBaud = numpy.fromfile(fp,'<u4',1) | ||||
self.codes = numpy.fromfile(fp,'<f4',self.numCode*self.numBaud).reshape(self.numBaud,self.numCode) | ||||
|
r2 | |||
return 1 | ||||
class VoltageReader(DataReader): | ||||
# Este flag indica que la data leida no es continua | ||||
__jumpDataFlag = False | ||||
__idFile = 0 | ||||
__fp = 0 | ||||
__startDateTime = 0 | ||||
__endDateTime = 0 | ||||
__dataType = 0 | ||||
__sizeOfFileByHeader = 0 | ||||
|
r5 | __pathList = [] | ||
|
r2 | |||
filenameList = [] | ||||
__lastUTTime = 0 | ||||
__maxTimeStep = 5 | ||||
__flagResetProcessing = 0 | ||||
__flagIsNewFile = 0 | ||||
noMoreFiles = 0 | ||||
online = 0 | ||||
filename = '' | ||||
fileSize = 0 | ||||
firstHeaderSize = 0 | ||||
basicHeaderSize = 24 | ||||
|
r5 | objBasicHeader = BasicHeader() | ||
|
r2 | |||
|
r5 | objSystemHeader = SystemHeader() | ||
|
r2 | |||
|
r5 | objRadarControllerHeader = RadarControllerHeader() | ||
|
r2 | |||
|
r5 | objProcessingHeader = ProcessingHeader() | ||
|
r2 | |||
__buffer = 0 | ||||
__buffer_id = 9999 | ||||
def __init__(self): | ||||
pass | ||||
|
r5 | def __rdSystemHeader(self,fp=None): | ||
if fp == None: | ||||
fp = self.__fp | ||||
self.objSystemHeader.read(fp) | ||||
|
r2 | |||
|
r5 | def __rdRadarControllerHeader(self,fp=None): | ||
if fp == None: | ||||
fp = self.__fp | ||||
self.objRadarControllerHeader.read(fp) | ||||
def __rdProcessingHeader(self,fp=None): | ||||
if fp == None: | ||||
fp = self.__fp | ||||
self.objProcessingHeader.read(fp) | ||||
|
r2 | def __searchFiles(self,path, startDateTime, endDateTime, set=None, expLabel = "", ext = "*.r"): | ||
startUtSeconds = time.mktime(startDateTime.timetuple()) | ||||
endUtSeconds = time.mktime(endDateTime.timetuple()) | ||||
startYear = startDateTime.timetuple().tm_year | ||||
endYear = endDateTime.timetuple().tm_year | ||||
startDoy = startDateTime.timetuple().tm_yday | ||||
endDoy = endDateTime.timetuple().tm_yday | ||||
rangeOfYears = range(startYear,endYear+1) | ||||
listOfListDoys = [] | ||||
if startYear == endYear: | ||||
doyList = range(startDoy,endDoy+1) | ||||
else: | ||||
for year in rangeOfYears: | ||||
if (year == startYear): | ||||
listOfListDoys.append(range(startDoy,365+1)) | ||||
elif (year == endYear): | ||||
listOfListDoys.append(range(1,endDoy+1)) | ||||
else: | ||||
listOfListDoys.append(range(1,365+1)) | ||||
doyList = [] | ||||
for list in listOfListDoys: | ||||
doyList = doyList + list | ||||
folders = [] | ||||
for thisPath in os.listdir(path): | ||||
if os.path.isdir(os.path.join(path,thisPath)): | ||||
#folders.append(os.path.join(path,thisPath)) | ||||
folders.append(thisPath) | ||||
|
r5 | pathList = [] | ||
|
r2 | dicOfPath = {} | ||
for year in rangeOfYears: | ||||
for doy in doyList: | ||||
tmp = fnmatch.filter(folders, 'D' + '%4.4d%3.3d' % (year,doy)) | ||||
if len(tmp) == 0: | ||||
continue | ||||
if expLabel == '': | ||||
|
r5 | pathList.append(os.path.join(path,tmp[0])) | ||
|
r2 | dicOfPath.setdefault(os.path.join(path,tmp[0])) | ||
dicOfPath[os.path.join(path,tmp[0])] = [] | ||||
else: | ||||
|
r5 | pathList.append(os.path.join(path,os.path.join(tmp[0],expLabel))) | ||
|
r2 | dicOfPath.setdefault(os.path.join(path,os.path.join(tmp[0],expLabel))) | ||
dicOfPath[os.path.join(path,os.path.join(tmp[0],expLabel))] = [] | ||||
filenameList = [] | ||||
|
r5 | for thisPath in pathList: | ||
|
r2 | fileList = glob.glob1(thisPath, ext) | ||
#dicOfPath[thisPath].append(fileList) | ||||
fileList.sort() | ||||
for file in fileList: | ||||
filename = os.path.join(thisPath,file) | ||||
if self.isThisFileinRange(filename, startUtSeconds, endUtSeconds): | ||||
filenameList.append(filename) | ||||
self.filenameList = filenameList | ||||
|
r5 | return pathList, filenameList | ||
|
r2 | |||
def isThisFileinRange(self, filename, startUTSeconds=None, endUTSeconds=None): | ||||
try: | ||||
fp = open(filename,'rb') | ||||
except: | ||||
raise IOError, "The file %s can't be opened" %(filename) | ||||
if startUTSeconds==None: | ||||
startUTSeconds = self.startUTCSeconds | ||||
if endUTSeconds==None: | ||||
endUTSeconds = self.endUTCSeconds | ||||
|
r5 | objBasicHeader = BasicHeader() | ||
|
r2 | |||
|
r5 | if not(objBasicHeader.read(fp)): | ||
|
r2 | return 0 | ||
|
r5 | fp.close() | ||
if not ((startUTSeconds <= objBasicHeader.utc) and (endUTSeconds >= objBasicHeader.utc)): | ||||
|
r2 | return 0 | ||
return 1 | ||||
def __readBasicHeader(self, fp=None): | ||||
if fp == None: | ||||
fp = self.__fp | ||||
|
r5 | self.objBasicHeader.read(fp) | ||
|
r2 | |||
def __readFirstHeader(self): | ||||
self.__readBasicHeader() | ||||
self.__rdSystemHeader() | ||||
self.__rdRadarControllerHeader() | ||||
self.__rdProcessingHeader() | ||||
|
r5 | self.firstHeaderSize = self.objBasicHeader.size | ||
|
r2 | |||
|
r5 | data_type=int(numpy.log2((self.objProcessingHeader.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR)) | ||
|
r2 | if data_type == 0: | ||
tmp=numpy.dtype([('real','<i1'),('imag','<i1')]) | ||||
elif data_type == 1: | ||||
tmp=numpy.dtype([('real','<i2'),('imag','<i2')]) | ||||
elif data_type == 2: | ||||
tmp=numpy.dtype([('real','<i4'),('imag','<i4')]) | ||||
elif data_type == 3: | ||||
tmp=numpy.dtype([('real','<i8'),('imag','<i8')]) | ||||
elif data_type == 4: | ||||
tmp=numpy.dtype([('real','<f4'),('imag','<f4')]) | ||||
elif data_type == 5: | ||||
tmp=numpy.dtype([('real','<f8'),('imag','<f8')]) | ||||
else: | ||||
print 'no define data type' | ||||
tmp = 0 | ||||
self.__flagIsNewFile = 0 | ||||
self.__dataType = tmp | ||||
|
r5 | self.__sizeOfFileByHeader = self.objProcessingHeader.dataBlocksPerFile * self.objProcessingHeader.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.objProcessingHeader.dataBlocksPerFile - 1) | ||
|
r2 | |||
def __setNextFileOnline(self): | ||||
return 0 | ||||
def __setNextFileOffline(self): | ||||
idFile = self.__idFile | ||||
while(True): | ||||
idFile += 1 | ||||
if not(idFile < len(self.filenameList)): | ||||
self.noMoreFiles = 1 | ||||
return 0 | ||||
filename = self.filenameList[idFile] | ||||
fileSize = os.path.getsize(filename) | ||||
fp = open(filename,'rb') | ||||
currentSize = fileSize - fp.tell() | ||||
|
r5 | neededSize = self.objProcessingHeader.blockSize + self.firstHeaderSize | ||
|
r2 | |||
if (currentSize < neededSize): | ||||
continue | ||||
break | ||||
self.__flagIsNewFile = 1 | ||||
self.__idFile = idFile | ||||
self.filename = filename | ||||
self.fileSize = fileSize | ||||
self.__fp = fp | ||||
print 'Setting the file: %s'%self.filename | ||||
return 1 | ||||
def __setNextFile(self): | ||||
if self.online: | ||||
return self.__setNextFileOnline() | ||||
else: | ||||
return self.__setNextFileOffline() | ||||
def __setNewBlock(self): | ||||
currentSize = self.fileSize - self.__fp.tell() | ||||
|
r5 | neededSize = self.objProcessingHeader.blockSize + self.basicHeaderSize | ||
|
r2 | |||
# Bloque Completo | ||||
if (currentSize >= neededSize): | ||||
self.__readBasicHeader() | ||||
return 1 | ||||
self.__setNextFile() | ||||
self.__readFirstHeader() | ||||
|
r5 | deltaTime = self.objBasicHeader.utc - self.__lastUTTime # check this | ||
|
r2 | if deltaTime > self.__maxTimeStep: | ||
self.__flagResetProcessing = 1 | ||||
return 1 | ||||
def __readBlock(self): | ||||
"""Lee el bloque de datos desde la posicion actual del puntero del archivo y | ||||
actualiza todos los parametros relacionados al bloque de datos (data, time, | ||||
etc). La data leida es almacenada en el buffer y el contador de datos leidos es | ||||
seteado a 0 | ||||
""" | ||||
|
r5 | pts2read = self.objProcessingHeader.profilesPerBlock*self.objProcessingHeader.numHeights*self.objSystemHeader.numChannels | ||
|
r2 | |||
data = numpy.fromfile(self.__fp,self.__dataType,pts2read) | ||||
|
r5 | data = data.reshape((self.objProcessingHeader.profilesPerBlock, self.objProcessingHeader.numHeights, self.objSystemHeader.numChannels)) | ||
|
r2 | |||
self.__buffer = data | ||||
self.__buffer_id = 0 | ||||
def readNextBlock(self): | ||||
self.__setNewBlock() | ||||
self.__readBlock() | ||||
|
r5 | self.__lastUTTime = self.objBasicHeader.utc | ||
|
r2 | |||
def __hasNotDataInBuffer(self): | ||||
|
r5 | if self.__buffer_id >= self.objProcessingHeader.profilesPerBlock: | ||
|
r2 | return 1 | ||
return 0 | ||||
def getData(self): | ||||
"""Obtiene un unidad de datos del buffer de lectura y es copiada a la clase "Data" | ||||
con todos los parametros asociados a este. cuando no hay datos en el buffer de | ||||
lectura es necesario hacer una nueva lectura de los bloques de datos | ||||
"__readBlock" | ||||
""" | ||||
if self.__hasNotDataInBuffer(): | ||||
self.readNextBlock() | ||||
if self.noMoreFiles == 1: | ||||
print 'read finished' | ||||
return None | ||||
data = self.__buffer[self.__buffer_id,:,:] | ||||
#print self.__buffer_id | ||||
self.__buffer_id += 1 | ||||
#call setData - to Data Object | ||||
return data | ||||
def setup(self, path, startDateTime, endDateTime, set=None, expLabel = "", ext = ".r", online = 0): | ||||
if online == 0: | ||||
pathList, filenameList = self.__searchFiles(path, startDateTime, endDateTime, set, expLabel, ext) | ||||
if len(filenameList) == 0: | ||||
print 'Do not exist files in range: %s - %s'%(startDateTime.ctime(), endDateTime.ctime()) | ||||
return 0 | ||||
# for thisFile in filenameList: | ||||
# print thisFile | ||||
self.__idFile = -1 | ||||
if not(self.__setNextFile()): | ||||
print "No more files" | ||||
return 0 | ||||
self.__readFirstHeader() | ||||
self.startUTCSeconds = time.mktime(startDateTime.timetuple()) | ||||
self.endUTCSeconds = time.mktime(endDateTime.timetuple()) | ||||
self.startYear = startDateTime.timetuple().tm_year | ||||
self.endYear = endDateTime.timetuple().tm_year | ||||
self.startDoy = startDateTime.timetuple().tm_yday | ||||
self.endDoy = endDateTime.timetuple().tm_yday | ||||
#call fillHeaderValues() - to Data Object | ||||
|
r5 | self.__pathList = pathList | ||
|
r2 | self.filenameList = filenameList | ||
self.online = online | ||||