diff --git a/schainpy2/Data/JROData.py b/schainpy2/Data/JROData.py new file mode 100644 index 0000000..1ef2c1b --- /dev/null +++ b/schainpy2/Data/JROData.py @@ -0,0 +1,48 @@ + +import copy +import numpy +from JROHeader import BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader + +class Data: + def __init__(self): + pass + + def copy(self, inputObj=None): + if inputObj == None: + return copy.deepcopy(self) + + for key in inputObj.__dict__.keys(): + self.__dict__[key] = inputObj.__dict__[key] + + def deepcopy(self): + return copy.deepcopy(self) + +class JROData(Data): + data = None +# m_BasicHeader = BasicHeader() +# m_SystemHeader = SystemHeader() +# m_RadarControllerHeader = RadarControllerHeader() +# m_ProcessingHeader = ProcessingHeader() + type = None + dataType = None + heightList = None + channelIndexList = None + channelList = None + nHeights = None + nProfiles = None + nBlocksPerFile = None + nChannels = None + flagNoData = False + flagResetProcessing = False + + def __init__(self): + raise ValueError, "This class has not been implemented" + +# def updateHeaderFromObj(self): +# pass +# +# +# def updateObjFromHeader(self): +# pass + + diff --git a/schainpy2/Data/Voltage.py b/schainpy2/Data/Voltage.py new file mode 100644 index 0000000..3f5f5f5 --- /dev/null +++ b/schainpy2/Data/Voltage.py @@ -0,0 +1,27 @@ +import numpy +from JROData import JROData +# No deberia importar los Headers +class Voltage(JROData): + data = None + nCohInt = None + + def __init__(self): + self.type = "Voltage" + + def updateObjFromHeader(self): + xi = self.m_ProcessingHeader.firstHeight + step = self.m_ProcessingHeader.deltaHeight + xf = xi + self.m_ProcessingHeader.numHeights*step + + self.heightList = numpy.arange(xi, xf, step) + self.channelIndexList = numpy.arange(self.m_SystemHeader.numChannels) + self.channelList = numpy.arange(self.m_SystemHeader.numChannels) + + self.nHeights = len(self.heightList) + self.nChannels = len(self.channelList) + self.nProfiles = self.m_ProcessingHeader.profilesPerBlock + self.nBlocksPerFile = self.m_ProcessingHeader.dataBlocksPerFile + self.nCohInt = self.m_ProcessingHeader.coherentInt + + def updateHeaderFromObj(self): + pass \ No newline at end of file diff --git a/schainpy2/Data/__init__.py b/schainpy2/Data/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/schainpy2/Data/__init__.py diff --git a/schainpy2/IO/JRODataIO.py b/schainpy2/IO/JRODataIO.py new file mode 100644 index 0000000..2903c91 --- /dev/null +++ b/schainpy2/IO/JRODataIO.py @@ -0,0 +1,378 @@ +import os, sys +import glob +import time +import numpy +import fnmatch +import time, datetime + +path = os.path.split(os.getcwd())[0] +sys.path.append(path) + +from Model.JROHeader import * +from Model.JROData import JROData + +def isThisFileinRange(filename, startUTSeconds, endUTSeconds): + """ + Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado. + + Inputs: + filename : nombre completo del archivo de datos en formato Jicamarca (.r) + + startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en + segundos contados desde 01/01/1970. + endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en + segundos contados desde 01/01/1970. + + Return: + Boolean : Retorna True si el archivo de datos contiene datos en el rango de + fecha especificado, de lo contrario retorna False. + + Excepciones: + Si el archivo no existe o no puede ser abierto + Si la cabecera no puede ser leida. + + """ + m_BasicHeader = BasicHeader() + + try: + fp = open(filename,'rb') + except: + raise IOError, "The file %s can't be opened" %(filename) + + sts = m_BasicHeader.read(fp) + fp.close() + + if not(sts): + print "Skipping the file %s because it has not a valid header" %(filename) + return 0 + + if not ((startUTSeconds <= m_BasicHeader.utc) and (endUTSeconds > m_BasicHeader.utc)): + return 0 + + return 1 + + + + +class JRODataIO: + c = 3E8 + m_BasicHeader = BasicHeader() + m_SystemHeader = SystemHeader() + m_RadarControllerHeader = RadarControllerHeader() + m_ProcessingHeader = ProcessingHeader() + online = 0 + dataType = None + pathList = [] + filenameList = [] + filename = None + ext = None + fileIndex = None + flagNoMoreFiles = 0 + flagIsNewFile = 1 + flagResetProcessing = 0 + flagIsNewBlock = 0 + fp = None + firstHeaderSize = 0 + basicHeaderSize = 24 + fileSize = None + ippSeconds = None + fileSizeByHeader = None + def __init__(self): + pass + +class JRODataReader(JRODataIO): + def __init__(self): + pass + + def createObjByDefault(self): + """ + + """ + raise ValueError, "This method has not been implemented" + + def getBlockDimension(self): + + raise ValueError, "No implemented" + + def __searchFilesOffLine(self, + path, + startDate, + endDate, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + set=None, + expLabel="", + ext=".r"): + dirList = [] + for thisPath in os.listdir(path): + if os.path.isdir(os.path.join(path,thisPath)): + dirList.append(thisPath) + + if not(dirList): + return None, None + + pathList = [] + dateList = [] + + thisDate = startDate + + while(thisDate <= endDate): + year = thisDate.timetuple().tm_year + doy = thisDate.timetuple().tm_yday + + match = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy)) + if len(match) == 0: + thisDate += datetime.timedelta(1) + continue + + pathList.append(os.path.join(path,match[0],expLabel)) + dateList.append(thisDate) + thisDate += datetime.timedelta(1) + + filenameList = [] + for index in range(len(pathList)): + + thisPath = pathList[index] + fileList = glob.glob1(thisPath, "*%s" %ext) + fileList.sort() + + #Busqueda de datos en el rango de horas indicados + thisDate = dateList[index] + startDT = datetime.datetime.combine(thisDate, startTime) + endDT = datetime.datetime.combine(thisDate, endTime) + + startUtSeconds = time.mktime(startDT.timetuple()) + endUtSeconds = time.mktime(endDT.timetuple()) + + for file in fileList: + + filename = os.path.join(thisPath,file) + + if isThisFileinRange(filename, startUtSeconds, endUtSeconds): + filenameList.append(filename) + + if not(filenameList): + return None, None + + self.filenameList = filenameList + + return pathList, filenameList + + def setup(self,dataOutObj=None, + path=None,startDate=None, + endDate=None, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + set=0, + expLabel = "", + ext = None, + online = 0): + + if path == None: + raise ValueError, "The path is not valid" + + if ext == None: + ext = self.ext + + if dataOutObj == None: + dataOutObj = self.createObjByDefault() + + self.dataOutObj = dataOutObj + + if online: + pass + + else: + print "Searching file in offline mode" + pathList, filenameList = self.__searchFilesOffLine(path, startDate, endDate, startTime, endTime, set, expLabel, ext) + if not(pathList): + print "No files in range: %s - %s"%(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()) + return None + self.fileIndex = -1 + self.pathList = pathList + self.filenameList = filenameList + + self.online = online + ext = ext.lower() + self.ext = ext + + if not(self.setNextFile()): + if (startDate!=None) and (endDate!=None): + print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()) + elif startDate != None: + print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime()) + else: + print "No files" + + return None + + self.updateDataHeader() + + return self.dataOutObj + + def __setNextFileOffline(self): + idFile = self.fileIndex + + while (True): + idFile += 1 + if not(idFile < len(self.filenameList)): + self.flagNoMoreFiles = 1 + print "No more Files" + return 0 + + filename = self.filenameList[idFile] + + if not(self.__verifyFile(filename)): + continue + + fileSize = os.path.getsize(filename) + fp = open(filename,'rb') + break + + self.flagIsNewFile = 1 + self.fileIndex = idFile + self.filename = filename + self.fileSize = fileSize + self.fp = fp + + print "Setting the file: %s"%self.filename + + return 1 + + + + def setNextFile(self): + if self.fp != None: + self.fp.close() + + if self.online: + newFile = self.__setNextFileOnline() + else: + newFile = self.__setNextFileOffline() + + if not(newFile): + return 0 + + self.__readFirstHeader() + self.nReadBlocks = 0 + return 1 + + def __rdProcessingHeader(self, fp=None): + if fp == None: + fp = self.fp + + self.m_ProcessingHeader.read(fp) + + def __rdRadarControllerHeader(self, fp=None): + if fp == None: + fp = self.fp + + self.m_RadarControllerHeader.read(fp) + + def __rdSystemHeader(self, fp=None): + if fp == None: + fp = self.fp + + self.m_SystemHeader.read(fp) + + def __rdBasicHeader(self, fp=None): + if fp == None: + fp = self.fp + + self.m_BasicHeader.read(fp) + + + def __readFirstHeader(self): + self.__rdBasicHeader() + self.__rdSystemHeader() + self.__rdRadarControllerHeader() + self.__rdProcessingHeader() + + self.firstHeaderSize = self.m_BasicHeader.size + + datatype = int(numpy.log2((self.m_ProcessingHeader.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR)) + if datatype == 0: + datatype_str = numpy.dtype([('real',' 0: + fp.seek(jumpFp) + + except: + return 0 + + return 1 + + def write(self, fp): + headerTuple = (self.size, + self.expType, + self.nTx, + self.ipp, + self.txA, + self.txB, + self.numWindows, + self.numTaus, + self.codeType, + self.line6Function, + self.line5Function, + self.fClock, + self.prePulseBefore, + self.prePulserAfter, + self.rangeIpp, + self.rangeTxA, + self.rangeTxB) + + header = numpy.array(headerTuple,self.struct) + header.tofile(fp) + + dynamic = self.dynamic + dynamic.tofile(fp) + + return 1 + + + +class ProcessingHeader(Header): + + size = None + dataType = None + blockSize = None + profilesPerBlock = None + dataBlocksPerFile = None + numWindows = None + processFlags = None + coherentInt = None + incoherentInt = None + totalSpectra = None + struct = None + flag_dc = None + flag_cspc = None + + def __init__(self): + self.size = 0 + self.dataType = 0 + self.blockSize = 0 + self.profilesPerBlock = 0 + self.dataBlocksPerFile = 0 + self.numWindows = 0 + self.processFlags = 0 + self.coherentInt = 0 + self.incoherentInt = 0 + self.totalSpectra = 0 + self.struct = numpy.dtype([ + ('nSize',' 0: + self.flag_cspc = True + + except: + return 0 + + return 1 + + def write(self, fp): + headerTuple = (self.size, + self.dataType, + self.blockSize, + self.profilesPerBlock, + self.dataBlocksPerFile, + self.numWindows, + self.processFlags, + self.coherentInt, + self.incoherentInt, + self.totalSpectra) + + header = numpy.array(headerTuple,self.struct) + header.tofile(fp) + + if self.numWindows != 0: + sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin) + samplingWindow = numpy.array(sampleWindowTuple,self.structSamplingWindow) + samplingWindow.tofile(fp) + + + if self.totalSpectra != 0: + spectraComb = numpy.array([],numpy.dtype('u1')) + spectraComb = self.spectraComb + spectraComb.tofile(fp) + + + if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE: + numCode = self.numCode + numCode.tofile(fp) + + numBaud = self.numBaud + numBaud.tofile(fp) + + code = self.code.reshape(numCode*numBaud) + code.tofile(fp) + + return 1 + +class RCfunction: + NONE=0 + FLIP=1 + CODE=2 + SAMPLING=3 + LIN6DIV256=4 + SYNCHRO=5 + +class nCodeType: + NONE=0 + USERDEFINE=1 + BARKER2=2 + BARKER3=3 + BARKER4=4 + BARKER5=5 + BARKER7=6 + BARKER11=7 + BARKER13=8 + AC128=9 + COMPLEMENTARYCODE2=10 + COMPLEMENTARYCODE4=11 + COMPLEMENTARYCODE8=12 + COMPLEMENTARYCODE16=13 + COMPLEMENTARYCODE32=14 + COMPLEMENTARYCODE64=15 + COMPLEMENTARYCODE128=16 + CODE_BINARY28=17 + +class PROCFLAG: + COHERENT_INTEGRATION = numpy.uint32(0x00000001) + DECODE_DATA = numpy.uint32(0x00000002) + SPECTRA_CALC = numpy.uint32(0x00000004) + INCOHERENT_INTEGRATION = numpy.uint32(0x00000008) + POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010) + SHIFT_FFT_DATA = numpy.uint32(0x00000020) + + DATATYPE_CHAR = numpy.uint32(0x00000040) + DATATYPE_SHORT = numpy.uint32(0x00000080) + DATATYPE_LONG = numpy.uint32(0x00000100) + DATATYPE_INT64 = numpy.uint32(0x00000200) + DATATYPE_FLOAT = numpy.uint32(0x00000400) + DATATYPE_DOUBLE = numpy.uint32(0x00000800) + + DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000) + DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000) + DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000) + + SAVE_CHANNELS_DC = numpy.uint32(0x00008000) + DEFLIP_DATA = numpy.uint32(0x00010000) + DEFINE_PROCESS_CODE = numpy.uint32(0x00020000) + + ACQ_SYS_NATALIA = numpy.uint32(0x00040000) + ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000) + ACQ_SYS_ADRXD = numpy.uint32(0x000C0000) + ACQ_SYS_JULIA = numpy.uint32(0x00100000) + ACQ_SYS_XXXXXX = numpy.uint32(0x00140000) + + EXP_NAME_ESP = numpy.uint32(0x00200000) + CHANNEL_NAMES_ESP = numpy.uint32(0x00400000) + + OPERATION_MASK = numpy.uint32(0x0000003F) + DATATYPE_MASK = numpy.uint32(0x00000FC0) + DATAARRANGE_MASK = numpy.uint32(0x00007000) + ACQ_SYS_MASK = numpy.uint32(0x001C0000) \ No newline at end of file diff --git a/schainpy2/IO/VoltageIO.py b/schainpy2/IO/VoltageIO.py new file mode 100644 index 0000000..9495b0d --- /dev/null +++ b/schainpy2/IO/VoltageIO.py @@ -0,0 +1,41 @@ + +import os, sys +import numpy +import glob +import fnmatch +import time, datetime + +path = os.path.split(os.getcwd())[0] +sys.path.append(path) + +from Model.JROHeader import * +from Model.Voltage import Voltage + +from IO.JRODataIO import JRODataReader + + +class VoltageReader(JRODataReader): + dataOutObj = None + datablock = None + ext = ".r" + optchar = "D" + + def __init__(self, dataOutObj=None): + self.datablock = None + #Por herencia no necesito instanciar nuevamente estos objetos + #self.m_BasicHeader = BasicHeader() + #self.m_SystemHeader = SystemHeader() + #self.m_RadarControllerHeader = RadarControllerHeader() + #self.m_ProcessingHeader = ProcessingHeader() + self.online = 0 + + def createObjByDefault(self): + dataObj = Voltage() + + return dataObj + + def getBlockDimension(self): + pts2read = self.m_ProcessingHeader.profilesPerBlock * self.m_ProcessingHeader.numHeights * self.m_SystemHeader.numChannels + self.blocksize = pts2read + + diff --git a/schainpy2/IO/__init__.py b/schainpy2/IO/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/schainpy2/IO/__init__.py diff --git a/schainpy2/Processing/VoltageProcessor.py b/schainpy2/Processing/VoltageProcessor.py new file mode 100644 index 0000000..5eb7191 --- /dev/null +++ b/schainpy2/Processing/VoltageProcessor.py @@ -0,0 +1,133 @@ +import os +import sys +import numpy + +path = os.path.split(os.getcwd())[0] +sys.path.append(path) + +from Model.Voltage import Voltage + + +class VoltageProcessor: + dataInObj = None + dataOutObj = None + integratorObjIndex = None + writerObjIndex = None + integratorObjList = None + writerObjList = None + + def __init__(self): + self.integratorObjIndex = None + self.writerObjIndex = None + self.integratorObjList = [] + self.writerObjList = [] + + def setup(self,dataInObj=None,dataOutObj=None): + self.dataInObj = dataInObj + + if self.dataOutObj == None: + dataOutObj = Voltage() + + self.dataOutObj = dataOutObj + + return self.dataOutObj + + def init(self): + self.integratorObjIndex = 0 + self.writerObjIndex = 0 + # No necesita copiar en cada init() los atributos de dataInObj + # la copia deberia hacerse por cada nuevo bloque de datos + + def addIntegrator(self,N,timeInterval): + objCohInt = CoherentIntegrator(N,timeInterval) + self.integratorObjList.append(objCohInt) + + def addWriter(self): + pass + + def integrator(self, N=None, timeInterval=None): + if self.dataOutObj.flagNoData: + return 0 + if len(self.integratorObjList) <= self.integratorObjIndex: + self.addIntegrator(N,timeInterval) + + myCohIntObj = self.integratorObjList[self.integratorObjIndex] + myCohIntObj.exe(data=self.dataOutObj.data,timeOfData=None) + + pass + + def writeData(self): + pass + +class CoherentIntegrator: + + integ_counter = None + data = None + navg = None + buffer = None + nCohInt = None + + def __init__(self, N=None,timeInterval=None): + + self.data = None + self.navg = None + self.buffer = None + self.timeOut = None + self.exitCondition = False + self.isReady = False + self.nCohInt = N + self.integ_counter = 0 + if timeInterval!=None: + self.timeIntervalInSeconds = timeInterval * 60. #if (type(timeInterval)!=integer) -> change this line + + if ((timeInterval==None) and (N==None)): + raise ValueError, "N = None ; timeInterval = None" + + if timeInterval == None: + self.timeFlag = False + else: + self.timeFlag = True + + def exe(self, data, timeOfData): + + if self.timeFlag: + if self.timeOut == None: + self.timeOut = timeOfData + self.timeIntervalInSeconds + + if timeOfData < self.timeOut: + if self.buffer == None: + self.buffer = data + else: + self.buffer = self.buffer + data + self.integ_counter += 1 + else: + self.exitCondition = True + + else: + if self.integ_counter < self.nCohInt: + if self.buffer == None: + self.buffer = data + else: + self.buffer = self.buffer + data + + self.integ_counter += 1 + + if self.integ_counter == self.nCohInt: + self.exitCondition = True + + if self.exitCondition: + self.data = self.buffer + self.navg = self.integ_counter + self.isReady = True + self.buffer = None + self.timeOut = None + self.integ_counter = 0 + self.exitCondition = False + + if self.timeFlag: + self.buffer = data + self.timeOut = timeOfData + self.timeIntervalInSeconds + else: + self.isReady = False + + diff --git a/schainpy2/Processing/__init__.py b/schainpy2/Processing/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/schainpy2/Processing/__init__.py diff --git a/schainpy2/testSchainExp.py b/schainpy2/testSchainExp.py new file mode 100644 index 0000000..8ac9164 --- /dev/null +++ b/schainpy2/testSchainExp.py @@ -0,0 +1,64 @@ + +import os, sys +import time, datetime + +path = os.path.split(os.getcwd())[0] +sys.path.append(path) + +from Data.Voltage import Voltage +from IO.VoltageIO import * + +from Processing.VoltageProcessor import * + + + +class TestSChain(): + + def __init__(self): + self.setValues() + self.createObjects() + self.testSChain() + + def setValues(self): + self.path = "/Users/jro/Documents/RadarData/MST_ISR/MST" + + self.wrpath = "/Users/jro/Documents/RadarData/wr_data" + + self.startDate = datetime.date(2009,1,17) + self.endDate = datetime.date(2009,1,17) + + self.startTime = datetime.time(0,0,0) + self.endTime = datetime.time(14,1,1) + + def createObjects(self): + self.readerObj = VoltageReader() + self.voltProcObj = VoltageProcessor() + + self.voltObj1 = self.readerObj.setup( + path = self.path, + startDate = self.startDate, + endDate = self.endDate, + startTime = self.startTime, + endTime = self.endTime, + expLabel = '', + online = 0) + + + + def testSChain(self): + + ini = time.time() + + while(True): + self.readerObj.getData() + + if self.readerObj.flagNoMoreFiles: + break + + if self.readerObj.flagIsNewBlock: + print 'Block No %04d, Time: %s' %(self.readerObj.nTotalBlocks, + datetime.datetime.fromtimestamp(self.readerObj.m_BasicHeader.utc),) + + +if __name__ == '__main__': + TestSChain() \ No newline at end of file