schain-jc Commit - r8:cd80c0608691 · Jicamarca Repository

En esta version se corrigieron los bugs de lectura, la lectura se ha implementado para modo offline, el script TestReader.py es un ejemplo que muestra como debe hacerse la lectura de rawdata usando estas librerias.

Daniel Valdez -

r8:cd80c0608691

parent child

Context file:

r8:cd80c0608691

Collapse all files

schainpy/IO/Header.py +2 0

+            import numpy
             class PROCFLAG:
                 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
                 DECODE_DATA = numpy.uint32(0x00000002)
                 SPECTRA_CALC = numpy.uint32(0x00000004)
                 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
                 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
                 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
                 DATATYPE_CHAR = numpy.uint32(0x00000040)
                 DATATYPE_SHORT = numpy.uint32(0x00000080)
                 DATATYPE_LONG = numpy.uint32(0x00000100)
                 DATATYPE_INT64 = numpy.uint32(0x00000200)
                 DATATYPE_FLOAT = numpy.uint32(0x00000400)
                 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
                 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
                 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
                 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
                 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
                 DEFLIP_DATA = numpy.uint32(0x00010000)
                 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
                 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
                 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
                 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
                 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
                 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
                 EXP_NAME_ESP = numpy.uint32(0x00200000)
                 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
                 OPERATION_MASK = numpy.uint32(0x0000003F)
                 DATATYPE_MASK = numpy.uint32(0x00000FC0)
                 DATAARRANGE_MASK = numpy.uint32(0x00007000)
                 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
             class BasicHeader:
                 size = 0
                 version = 0
                 dataBlock = 0
                 utc = 0
                 miliSecond = 0
                 timeZone = 0
                 dstFlag = 0
                 errorCount = 0
                 struct = numpy.dtype([
                                       ('nSize','<u4'),
                                       ('nVersion','<u2'),
                                       ('nDataBlockId','<u4'),
                                       ('nUtime','<u4'),
                                       ('nMilsec','<u2'),
                                       ('nTimezone','<i2'),
                                       ('nDstflag','<i2'),
                                       ('nErrorCount','<u4')
                                       ])
                 def __init__(self):
                     pass
                 def read(self, fp):
                     header = numpy.fromfile(fp, self.struct,1)
                     self.size = header['nSize'][0]
                     self.version = header['nVersion'][0]
                     self.dataBlock = header['nDataBlockId'][0]
                     self.utc = header['nUtime'][0]
                     self.miliSecond = header['nMilsec'][0]
                     self.timeZone = header['nTimezone'][0]
                     self.dstFlag = header['nDstflag'][0]
                     self.errorCount = header['nErrorCount'][0]
                     return 1
             class SystemHeader:
                 size = 0
                 numSamples = 0
                 numProfiles = 0
                 numChannels = 0
                 adcResolution = 0
                 pciDioBusWidth = 0
                 struct = numpy.dtype([
                                     ('nSize','<u4'),
                                     ('nNumSamples','<u4'),
                                     ('nNumProfiles','<u4'),
                                     ('nNumChannels','<u4'),
                                     ('nADCResolution','<u4'),
                                     ('nPCDIOBusWidth','<u4'),
                                     ])
                 def __init__(self):
                     pass
                 def read(self, fp):
                     header = numpy.fromfile(fp,self.struct,1)
                     self.size = header['nSize'][0]
                     self.numSamples = header['nNumSamples'][0]
                     self.numProfiles = header['nNumProfiles'][0]
                     self.numChannels = header['nNumChannels'][0]
                     self.adcResolution = header['nADCResolution'][0]
                     self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
                     return 1
             class RadarControllerHeader:
                 size = 0
                 expType = 0
                 nTx = 0
                 ipp = 0
                 txA = 0
                 txB = 0
                 numWindows = 0
                 numTaus = 0
                 codeType = 0
                 line6Function = 0
                 line5Fuction = 0
                 fClock = 0
                 prePulseBefore = 0
                 prePulserAfter = 0
                 rangeIpp = 0
                 rangeTxA = 0
                 rangeTxB = 0
                 struct = numpy.dtype([
                                      ('nSize','<u4'),
                                      ('nExpType','<u4'),
                                      ('nNTx','<u4'),
                                      ('fIpp','<f4'),
                                      ('fTxA','<f4'),
                                      ('fTxB','<f4'),
                                      ('nNumWindows','<u4'),
                                      ('nNumTaus','<u4'),
                                      ('nCodeType','<u4'),
                                      ('nLine6Function','<u4'),
                                      ('nLine5Function','<u4'),
                                      ('fClock','<f4'),
                                      ('nPrePulseBefore','<u4'),
                                      ('nPrePulseAfter','<u4'),
                                      ('sRangeIPP','<a20'),
                                      ('sRangeTxA','<a20'),
                                      ('sRangeTxB','<a20'),
                                      ])
                 def __init__(self):
                     pass
                 def read(self, fp):
                     header = numpy.fromfile(fp,self.struct,1)
                     self.size = header['nSize'][0]
                     self.expType = header['nExpType'][0]
                     self.nTx = header['nNTx'][0]
                     self.ipp = header['fIpp'][0]
                     self.txA = header['fTxA'][0]
                     self.txB = header['fTxB'][0]
                     self.numWindows = header['nNumWindows'][0]
                     self.numTaus = header['nNumTaus'][0]
                     self.codeType = header['nCodeType'][0]
                     self.line6Function = header['nLine6Function'][0]
                     self.line5Fuction = header['nLine5Function'][0]
                     self.fClock = header['fClock'][0]
                     self.prePulseBefore = header['nPrePulseBefore'][0]
                     self.prePulserAfter = header['nPrePulseAfter'][0]
                     self.rangeIpp = header['sRangeIPP'][0]
                     self.rangeTxA = header['sRangeTxA'][0]
                     self.rangeTxB = header['sRangeTxB'][0]
                     # jump Dynamic Radar Controller Header
                     jumpHeader = self.size - 116
                     fp.seek(fp.tell() + jumpHeader)
                     return 1
             class ProcessingHeader:
                 size = 0
                 dataType = 0
                 blockSize = 0
                 profilesPerBlock = 0
                 dataBlocksPerFile = 0
                 numWindows = 0
                 processFlags = 0
                 coherentInt = 0
                 incoherentInt = 0
                 totalSpectra = 0
                 struct = numpy.dtype([
                                        ('nSize','<u4'),
                                        ('nDataType','<u4'),
                                        ('nSizeOfDataBlock','<u4'),
                                        ('nProfilesperBlock','<u4'),
                                        ('nDataBlocksperFile','<u4'),
                                        ('nNumWindows','<u4'),
                                        ('nProcessFlags','<u4'),
                                        ('nCoherentIntegrations','<u4'),
                                        ('nIncoherentIntegrations','<u4'),
                                        ('nTotalSpectra','<u4')
                                        ])
                 samplingWindow = 0
                 structSamplingWindow = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')])
                 numHeights = 0
                 firstHeight = 0
                 deltaHeight = 0
                 samplesWin = 0
                 spectraComb = 0
                 numCode = 0
                 codes = 0
                 numBaud = 0
                 def __init__(self):
                     pass
                 def read(self, fp):
                     header = numpy.fromfile(fp,self.struct,1)
                     self.size = header['nSize'][0]
                     self.dataType = header['nDataType'][0]
                     self.blockSize = header['nSizeOfDataBlock'][0]
                     self.profilesPerBlock = header['nProfilesperBlock'][0]
                     self.dataBlocksPerFile = header['nDataBlocksperFile'][0]
                     self.numWindows = header['nNumWindows'][0]
                     self.processFlags = header['nProcessFlags']
                     self.coherentInt = header['nCoherentIntegrations'][0]
                     self.incoherentInt = header['nIncoherentIntegrations'][0]
                     self.totalSpectra = header['nTotalSpectra'][0]
                     self.samplingWindow = numpy.fromfile(fp,self.structSamplingWindow,self.numWindows)
                     self.numHeights = numpy.sum(self.samplingWindow['nsa'])
                     self.firstHeight = self.samplingWindow['h0']
                     self.deltaHeight = self.samplingWindow['dh']
                     self.samplesWin = self.samplingWindow['nsa']
                     self.spectraComb = numpy.fromfile(fp,'u1',2*self.totalSpectra)
                     if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
                         self.numCode = numpy.fromfile(fp,'<u4',1)
                         self.numBaud = numpy.fromfile(fp,'<u4',1)
                         self.codes = numpy.fromfile(fp,'<f4',self.numCode*self.numBaud).reshape(self.numBaud,self.numCode)
                     return 1

schainpy/IO/Voltage.py +40 -30

             '''
             Created on 23/01/2012
             @author: danielangelsuarezmunoz
             '''
             import numpy
             import os.path
             import glob
             import fnmatch
             import time
             import datetime
             from Header import *
             from Data import DataReader
             from Data import DataWriter
             class VoltageReader(DataReader):
-                # Este flag indica que la data leida no es continua
-                __jumpDataFlag = False
                 __idFile = 0
                 __fp = 0
                 __startDateTime = 0
                 __endDateTime = 0
                 __dataType = 0
                 __sizeOfFileByHeader = 0
                 __pathList = []
                 filenameList = []
                 __lastUTTime = 0
                 __maxTimeStep = 5
-                __flagResetProcessing = 0
+                flagResetProcessing = 0
                 __flagIsNewFile = 0
                 noMoreFiles = 0
                 online = 0
                 filename = ''
                 fileSize = 0
                 firstHeaderSize = 0
                 basicHeaderSize = 24
-                objBasicHeader = BasicHeader()
+                basicHeaderObj = BasicHeader()
-                objSystemHeader = SystemHeader()
+                systemHeaderObj = SystemHeader()
-                objRadarControllerHeader = RadarControllerHeader()
+                radarControllerHeaderObj = RadarControllerHeader()
-                objProcessingHeader = ProcessingHeader()
+                processingHeaderObj = ProcessingHeader()
                 __buffer = 0
                 __buffer_id = 9999
-                m_Voltage= Voltage()
+                #m_Voltage= Voltage()
                 def __init__(self):
                     pass
                 def __rdSystemHeader(self,fp=None):
                     if fp == None:
                         fp = self.__fp
-                    self.objSystemHeader.read(fp)
+                    self.systemHeaderObj.read(fp)
                 def __rdRadarControllerHeader(self,fp=None):
                     if fp == None:
                         fp = self.__fp
-                    self.objRadarControllerHeader.read(fp)
+                    self.radarControllerHeaderObj.read(fp)
                 def __rdProcessingHeader(self,fp=None):
                     if fp == None:
                         fp = self.__fp
-                    self.objProcessingHeader.read(fp)
+                    self.processingHeaderObj.read(fp)
                 def __searchFiles(self,path, startDateTime, endDateTime, set=None, expLabel = "", ext = "*.r"):
                     startUtSeconds = time.mktime(startDateTime.timetuple())
                     endUtSeconds = time.mktime(endDateTime.timetuple())
                     startYear = startDateTime.timetuple().tm_year
                     endYear = endDateTime.timetuple().tm_year
                     startDoy = startDateTime.timetuple().tm_yday
                     endDoy = endDateTime.timetuple().tm_yday
                     rangeOfYears = range(startYear,endYear+1)
                     listOfListDoys = []
                     if startYear == endYear:
                         doyList = range(startDoy,endDoy+1)
                     else:
                         for year in rangeOfYears:
                             if (year == startYear):
                                 listOfListDoys.append(range(startDoy,365+1))
                             elif (year == endYear):
                                 listOfListDoys.append(range(1,endDoy+1))
                             else:
                                 listOfListDoys.append(range(1,365+1))
                         doyList = []
                         for list in listOfListDoys:
                             doyList = doyList + list
                     folders = []
                     for thisPath in os.listdir(path):
                         if os.path.isdir(os.path.join(path,thisPath)):
                             #folders.append(os.path.join(path,thisPath))
                             folders.append(thisPath)
                     pathList = []
                     dicOfPath = {}
                     for year in rangeOfYears:
                         for doy in doyList:
                             tmp = fnmatch.filter(folders, 'D' + '%4.4d%3.3d' % (year,doy))
                             if len(tmp) == 0:
                                 continue
                             if expLabel == '':
                                 pathList.append(os.path.join(path,tmp[0]))
                                 dicOfPath.setdefault(os.path.join(path,tmp[0]))
                                 dicOfPath[os.path.join(path,tmp[0])] = []
                             else:
                                 pathList.append(os.path.join(path,os.path.join(tmp[0],expLabel)))
                                 dicOfPath.setdefault(os.path.join(path,os.path.join(tmp[0],expLabel)))
                                 dicOfPath[os.path.join(path,os.path.join(tmp[0],expLabel))] = []
                     filenameList = []
                     for thisPath in pathList:
                         fileList = glob.glob1(thisPath, ext)
                         #dicOfPath[thisPath].append(fileList)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             if self.isThisFileinRange(filename, startUtSeconds, endUtSeconds):
                                 filenameList.append(filename)
                     self.filenameList = filenameList
                     return pathList, filenameList
                 def isThisFileinRange(self, filename, startUTSeconds=None, endUTSeconds=None):
                     try:
                         fp = open(filename,'rb')
                     except:
                         raise IOError, "The file %s can't be opened" %(filename)
                     if startUTSeconds==None:
                         startUTSeconds = self.startUTCSeconds
                     if endUTSeconds==None:
                         endUTSeconds = self.endUTCSeconds
-                    objBasicHeader = BasicHeader()
+                    basicHeaderObj = BasicHeader()
-                    if not(objBasicHeader.read(fp)):
+                    if not(basicHeaderObj.read(fp)):
                         return 0
                     fp.close()
-                    if not ((startUTSeconds <= objBasicHeader.utc) and (endUTSeconds >= objBasicHeader.utc)):
+                    if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds >= basicHeaderObj.utc)):
                         return 0
                     return 1
                 def __readBasicHeader(self, fp=None):
                     if fp == None:
                         fp = self.__fp
-                    self.objBasicHeader.read(fp)
+                    self.basicHeaderObj.read(fp)
                 def __readFirstHeader(self):
                     self.__readBasicHeader()
                     self.__rdSystemHeader()
                     self.__rdRadarControllerHeader()
                     self.__rdProcessingHeader()
-                    self.firstHeaderSize = self.objBasicHeader.size
+                    self.firstHeaderSize = self.basicHeaderObj.size
-                    data_type=int(numpy.log2((self.objProcessingHeader.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
+                    data_type=int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if data_type == 0:
                         tmp=numpy.dtype([('real','<i1'),('imag','<i1')])
                     elif data_type == 1:
                         tmp=numpy.dtype([('real','<i2'),('imag','<i2')])
                     elif data_type == 2:
                         tmp=numpy.dtype([('real','<i4'),('imag','<i4')])
                     elif data_type == 3:
                         tmp=numpy.dtype([('real','<i8'),('imag','<i8')])
                     elif data_type == 4:
                         tmp=numpy.dtype([('real','<f4'),('imag','<f4')])
                     elif data_type == 5:
                         tmp=numpy.dtype([('real','<f8'),('imag','<f8')])
                     else:
                         print 'no define data type'
                         tmp = 0
-                    self.__flagIsNewFile = 0
                     self.__dataType = tmp
-                    self.__sizeOfFileByHeader = self.objProcessingHeader.dataBlocksPerFile * self.objProcessingHeader.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.objProcessingHeader.dataBlocksPerFile - 1)
+                    self.__sizeOfFileByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
                 def __setNextFileOnline(self):
                     return 0
                 def __setNextFileOffline(self):
                     idFile = self.__idFile
                     while(True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.noMoreFiles = 1
                             return 0
                         filename = self.filenameList[idFile]
                         fileSize = os.path.getsize(filename)
                         fp = open(filename,'rb')
                         currentSize = fileSize - fp.tell()
-                        neededSize = self.objProcessingHeader.blockSize + self.firstHeaderSize
+                        neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                         if (currentSize < neededSize):
                             continue
                         break
                     self.__flagIsNewFile = 1
                     self.__idFile = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.__fp = fp
                     print 'Setting the file: %s'%self.filename
                     return 1
                 def __setNextFile(self):
                     if self.online:
                         return self.__setNextFileOnline()
                     else:
                         return self.__setNextFileOffline()
                 def __setNewBlock(self):
+                    if self.__flagIsNewFile:
+                        return 1
                     currentSize = self.fileSize - self.__fp.tell()
-                    neededSize = self.objProcessingHeader.blockSize + self.basicHeaderSize
+                    neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     # Bloque Completo
                     if (currentSize >= neededSize):
                         self.__readBasicHeader()
                         return 1
-                    self.__setNextFile()
+                    if not(self.__setNextFile()):
+                        return 0
                     self.__readFirstHeader()
-                    deltaTime = self.objBasicHeader.utc - self.__lastUTTime # check this
+                    deltaTime = self.basicHeaderObj.utc - self.__lastUTTime # check this
+                    self.flagResetProcessing = 0
                     if deltaTime > self.__maxTimeStep:
-                        self.__flagResetProcessing = 1
+                        self.flagResetProcessing = 1
                     return 1
                 def __readBlock(self):
                     """Lee el bloque de datos desde la posicion actual del puntero del archivo y
                     actualiza todos los parametros relacionados al bloque de datos (data, time,
                     etc). La data leida es almacenada en el buffer y el contador de datos leidos es
                     seteado a 0
                     """
-                    pts2read = self.objProcessingHeader.profilesPerBlock*self.objProcessingHeader.numHeights*self.objSystemHeader.numChannels
+                    pts2read = self.processingHeaderObj.profilesPerBlock*self.processingHeaderObj.numHeights*self.systemHeaderObj.numChannels
                     data = numpy.fromfile(self.__fp,self.__dataType,pts2read)
-                    data = data.reshape((self.objProcessingHeader.profilesPerBlock, self.objProcessingHeader.numHeights, self.objSystemHeader.numChannels))
+                    data = data.reshape((self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.numHeights, self.systemHeaderObj.numChannels))
+                    self.__flagIsNewFile = 0
                     self.__buffer = data
                     self.__buffer_id = 0
                 def readNextBlock(self):
-                    self.__setNewBlock()
+                    if not(self.__setNewBlock()):
+                        return 0
                     self.__readBlock()
-                    self.__lastUTTime = self.objBasicHeader.utc
+                    self.__lastUTTime = self.basicHeaderObj.utc
+                    return 1
                 def __hasNotDataInBuffer(self):
-                    if self.__buffer_id >= self.objProcessingHeader.profilesPerBlock:
+                    if self.__buffer_id >= self.processingHeaderObj.profilesPerBlock:
                         return 1
                     return 0
                 def getData(self):
                     """Obtiene un unidad de datos del buffer de lectura y es copiada a la clase "Data"
                     con todos los parametros asociados a este. cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos
                     "__readBlock"
                     """
+                    self.flagResetProcessing = 0
                     if self.__hasNotDataInBuffer():
                         self.readNextBlock()
                     if self.noMoreFiles == 1:
                         print 'read finished'
                         return None
                     data = self.__buffer[self.__buffer_id,:,:]
+                    #time = timeblock + n*ipp
                     #print self.__buffer_id
                     self.__buffer_id += 1
                     #call setData - to Data Object
                     return data
                 def setup(self, path, startDateTime, endDateTime, set=None, expLabel = "", ext = ".r", online = 0):
                     if online == 0:
                         pathList, filenameList = self.__searchFiles(path, startDateTime, endDateTime, set, expLabel, ext)
                         if len(filenameList) == 0:
                             print 'Do not exist files in range: %s - %s'%(startDateTime.ctime(), endDateTime.ctime())
                             return 0
             #        for thisFile in filenameList:
             #            print thisFile
                     self.__idFile = -1
                     if not(self.__setNextFile()):
                         print "No more files"
                         return 0
                     self.__readFirstHeader()
                     self.startUTCSeconds = time.mktime(startDateTime.timetuple())
                     self.endUTCSeconds = time.mktime(endDateTime.timetuple())
                     self.startYear = startDateTime.timetuple().tm_year
                     self.endYear = endDateTime.timetuple().tm_year
                     self.startDoy = startDateTime.timetuple().tm_yday
                     self.endDoy = endDateTime.timetuple().tm_yday
                     #call fillHeaderValues() - to Data Object
                     self.__pathList = pathList
                     self.filenameList = filenameList
                     self.online = online
             class VoltageWriter(DataWriter):
                 def __init__(self):
                     pass
   No newline at end of file

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages