diff --git a/schainpy/model/data/jrodata.py b/schainpy/model/data/jrodata.py index 4baddc5..1efb847 100644 --- a/schainpy/model/data/jrodata.py +++ b/schainpy/model/data/jrodata.py @@ -17,9 +17,9 @@ import json import schainpy.admin from schainpy.utils import log -from .jroheaderIO import SystemHeader, RadarControllerHeader +from .jroheaderIO import SystemHeader, RadarControllerHeader,ProcessingHeader from schainpy.model.data import _noise - +SPEED_OF_LIGHT = 3e8 def getNumpyDtype(dataTypeCode): @@ -190,7 +190,7 @@ class JROData(GenericData): blocknow = None azimuth = None zenith = None - beam = Beam() + profileIndex = None error = None data = None @@ -200,10 +200,17 @@ class JROData(GenericData): azimuthList = [] elevationList = [] last_noise = None - radar_ipp = None + __ipp = None + __ippSeconds = None sampled_heightsFFT = None pulseLength_TxA = None deltaHeight = None + code = None + nCode = None + nBaud = None + unitsDescription = "The units of the parameters are according to the International System of units (Seconds, Meter, Hertz, ...), except \ +the parameters related to distances such as heightList, or heightResolution wich are in Km" + def __str__(self): @@ -268,7 +275,7 @@ class JROData(GenericData): return fmax def getFmax(self): - PRF = 1. / (self.ippSeconds * self.nCohInt) + PRF = 1. / (self.__ippSeconds * self.nCohInt) fmax = PRF return fmax @@ -285,61 +292,67 @@ class JROData(GenericData): def ippSeconds(self): ''' ''' - return self.radarControllerHeaderObj.ippSeconds + #return self.radarControllerHeaderObj.ippSeconds + return self.__ippSeconds @ippSeconds.setter def ippSeconds(self, ippSeconds): ''' ''' - self.radarControllerHeaderObj.ippSeconds = ippSeconds - - @property - def code(self): - ''' - ''' - return self.radarControllerHeaderObj.code - - @code.setter - def code(self, code): - ''' - ''' - self.radarControllerHeaderObj.code = code + #self.radarControllerHeaderObj.ippSeconds = ippSeconds + self.__ippSeconds = ippSeconds + self.__ipp = ippSeconds*SPEED_OF_LIGHT/2000.0 - @property - def nCode(self): - ''' - ''' - return self.radarControllerHeaderObj.nCode - - @nCode.setter - def nCode(self, ncode): - ''' - ''' - self.radarControllerHeaderObj.nCode = ncode - - @property - def nBaud(self): - ''' - ''' - return self.radarControllerHeaderObj.nBaud - @nBaud.setter - def nBaud(self, nbaud): - ''' - ''' - self.radarControllerHeaderObj.nBaud = nbaud + # @property + # def code(self): + # ''' + # ''' + # return self.radarControllerHeaderObj.code + # + # @code.setter + # def code(self, code): + # ''' + # ''' + # self.radarControllerHeaderObj.code = code + # + # @property + # def nCode(self): + # ''' + # ''' + # return self.radarControllerHeaderObj.nCode + # + # @nCode.setter + # def nCode(self, ncode): + # ''' + # ''' + # self.radarControllerHeaderObj.nCode = ncode + # + # @property + # def nBaud(self): + # ''' + # ''' + # return self.radarControllerHeaderObj.nBaud + # + # @nBaud.setter + # def nBaud(self, nbaud): + # ''' + # ''' + # self.radarControllerHeaderObj.nBaud = nbaud @property def ipp(self): ''' ''' - return self.radarControllerHeaderObj.ipp + return self.__ipp + #return self.radarControllerHeaderObj.ipp @ipp.setter def ipp(self, ipp): ''' ''' - self.radarControllerHeaderObj.ipp = ipp + self.__ipp = ipp + #self.radarControllerHeaderObj.ipp = ipp @property def metadata(self): @@ -364,6 +377,7 @@ class Voltage(JROData): self.useLocalTime = True self.radarControllerHeaderObj = RadarControllerHeader() self.systemHeaderObj = SystemHeader() + self.processingHeaderObj = ProcessingHeader() self.type = "Voltage" self.data = None self.nProfiles = None @@ -456,6 +470,7 @@ class Spectra(JROData): self.useLocalTime = True self.radarControllerHeaderObj = RadarControllerHeader() self.systemHeaderObj = SystemHeader() + self.processingHeaderObj = ProcessingHeader() self.type = "Spectra" self.timeZone = 0 self.nProfiles = None @@ -924,12 +939,15 @@ class Parameters(Spectra): noise_estimation = None GauSPC = None # Fit gaussian SPC + + def __init__(self): ''' Constructor ''' self.radarControllerHeaderObj = RadarControllerHeader() self.systemHeaderObj = SystemHeader() + self.processingHeaderObj = ProcessingHeader() self.type = "Parameters" self.timeZone = 0 diff --git a/schainpy/model/data/jroheaderIO.py b/schainpy/model/data/jroheaderIO.py index 048b5e8..28468da 100644 --- a/schainpy/model/data/jroheaderIO.py +++ b/schainpy/model/data/jroheaderIO.py @@ -279,6 +279,7 @@ class SystemHeader(Header): class RadarControllerHeader(Header): expType = None + dtype = "" nTx = None ipp = None txA = None @@ -291,19 +292,32 @@ class RadarControllerHeader(Header): fClock = None prePulseBefore = None prePulseAfter = None - rangeIpp = None + rangeIpp = None #variables innecesarias? rangeTxA = None rangeTxB = None structure = RADAR_STRUCTURE __size = None + ################################################ + ippSeconds = None + frequency = None + sampleRate = None + nOsamp = None + channelList = [] + azimuthList = [] + elevationList =[] + codeList = [] + nChannels = 1 + heightList = [] + heightResolution = None + def __init__(self, expType=2, nTx=1, ipp=None, txA=0, txB=0, nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None, numTaus=0, line6Function=0, line5Function=0, fClock=None, prePulseBefore=0, prePulseAfter=0, - codeType=0, nCode=0, nBaud=0, code=[], - flip1=0, flip2=0): + codeType=0, nCode=0, nBaud=0, code=[],nOsamp = None, frequency = None,sampleRate=None, + flip1=0, flip2=0, nChannels=1): # self.size = 116 self.expType = expType @@ -314,7 +328,8 @@ class RadarControllerHeader(Header): self.rangeIpp = ipp self.rangeTxA = txA self.rangeTxB = txB - + self.frequency = frequency + self.sampleRate = sampleRate self.nWindows = nWindows self.numTaus = numTaus self.codeType = codeType @@ -332,6 +347,7 @@ class RadarControllerHeader(Header): self.nCode = nCode self.nBaud = nBaud self.code = code + self.nOsamp = nOsamp self.flip1 = flip1 self.flip2 = flip2 @@ -578,6 +594,7 @@ class RadarControllerHeader(Header): class ProcessingHeader(Header): # size = None + dtype = None blockSize = None profilesPerBlock = None @@ -590,6 +607,21 @@ class ProcessingHeader(Header): structure = PROCESSING_STRUCTURE flag_dc = None flag_cspc = None + ######################################################### + nFFTPoints = None + nSamplesFFT = None + channelList = [] + azimuthList = [] + elevationList =[] + codeList = [] + nChannels = 1 + heightList = [] + ipp = None + ippSeconds = None + timeIncohInt = None + ################# + rangeIpp = None + heightResolution = None def __init__(self, dtype=0, blockSize=0, profilesPerBlock=0, dataBlocksPerFile=0, nWindows=0, processFlags=0, nCohInt=0, nIncohInt=0, totalSpectra=0, nHeights=0, firstHeight=0, deltaHeight=0, samplesWin=0, spectraComb=0, nCode=0, @@ -623,6 +655,7 @@ class ProcessingHeader(Header): self.flag_deflip = False self.length = 0 + def read(self, fp): self.length = 0 try: diff --git a/schainpy/model/graphics/jroplot_spectra.py b/schainpy/model/graphics/jroplot_spectra.py index 2a836e0..32d538c 100644 --- a/schainpy/model/graphics/jroplot_spectra.py +++ b/schainpy/model/graphics/jroplot_spectra.py @@ -77,7 +77,7 @@ class SpectraPlot(Plot): spc = 10*numpy.log10(z) data['spc'] = spc - print(spc[0].shape) + #print(spc[0].shape) data['rti'] = spc.mean(axis=1) data['noise'] = noise meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1)) @@ -309,16 +309,19 @@ class RTIPlot(Plot): try: if self.channelList != None: if len(self.elevationList) > 0 and len(self.azimuthList) > 0: - self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format( + self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format( self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList] else: - self.titles = ['{} Channel {}'.format( + self.titles = ['{} Channel {}'.format( self.CODE.upper(), x) for x in self.channelList] except: if self.channelList.any() != None: - - self.titles = ['{} Channel {}'.format( - self.CODE.upper(), x) for x in self.channelList] + if len(self.elevationList) > 0 and len(self.azimuthList) > 0: + self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format( + self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList] + else: + self.titles = ['{} Channel {}'.format( + self.CODE.upper(), x) for x in self.channelList] if self.decimation is None: x, y, z = self.fill_gaps(self.x, self.y, self.z) @@ -1002,6 +1005,7 @@ class NoiselessRTIPlot(Plot): def update(self, dataOut): if len(self.channelList) == 0: self.update_list(dataOut) + data = {} meta = {} #print(dataOut.max_nIncohInt, dataOut.nIncohInt) @@ -1037,9 +1041,12 @@ class NoiselessRTIPlot(Plot): self.CODE.upper(), x) for x in self.channelList] except: if self.channelList.any() != None: - - self.titles = ['{} Channel {}'.format( - self.CODE.upper(), x) for x in self.channelList] + if len(self.elevationList) > 0 and len(self.azimuthList) > 0: + self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format( + self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList] + else: + self.titles = ['{} Channel {}'.format( + self.CODE.upper(), x) for x in self.channelList] if self.decimation is None: diff --git a/schainpy/model/io/jroIO_kamisr.py b/schainpy/model/io/jroIO_kamisr.py index c30a082..ee2dc76 100644 --- a/schainpy/model/io/jroIO_kamisr.py +++ b/schainpy/model/io/jroIO_kamisr.py @@ -21,7 +21,7 @@ try: except: from time import sleep -from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader +from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader,ProcessingHeader from schainpy.model.data.jrodata import Voltage from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator from numpy import imag @@ -94,10 +94,11 @@ class AMISRReader(ProcessingUnit): code = None, nCode = 1, nBaud = 0, + nOsamp = None, online=False, old_beams=False, margin_days=1, - nFFT = 1, + nFFT = None, nChannels = None, ): @@ -110,6 +111,7 @@ class AMISRReader(ProcessingUnit): self.code = code self.nCode = int(nCode) self.nBaud = int(nBaud) + self.nOsamp = int(nOsamp) self.margin_days = margin_days self.__sampleRate = None @@ -154,7 +156,7 @@ class AMISRReader(ProcessingUnit): header = 'Raw11/Data/RadacHeader' if self.nChannels == None: - expFile = fp['Setup/ExperimentFile'][()].decode() + expFile = fp['Setup/Experimentfile'][()].decode() linesExp = expFile.split("\n") a = [line for line in linesExp if "nbeamcodes" in line] self.nChannels = int(a[0][11:]) @@ -164,6 +166,8 @@ class AMISRReader(ProcessingUnit): self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode() self.trueBeams = self.beamcodeFile.split("\n") self.trueBeams.pop()#remove last + if self.nFFT == None: + log.error("FFT or number of repetitions per channels is needed",self.name) beams_idx = [k*self.nFFT for k in range(self.nChannels)] beams = [self.trueBeams[b] for b in beams_idx] self.beamCode = [int(x, 16) for x in beams] @@ -191,7 +195,7 @@ class AMISRReader(ProcessingUnit): self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS self.rangeFromFile = fp.get('Raw11/Data/Samples/Range') self.frequency = fp.get('Rx/Frequency') - txAus = fp.get('Raw11/Data/Pulsewidth') + txAus = fp.get('Raw11/Data/Pulsewidth') #seconds self.baud = fp.get('Raw11/Data/TxBaud') sampleRate = fp.get('Rx/SampleRate') self.__sampleRate = sampleRate[()] @@ -207,7 +211,9 @@ class AMISRReader(ProcessingUnit): #filling radar controller header parameters self.__ippKm = self.ippSeconds *.15*1e6 # in km - self.__txA = (txAus[()])*.15 #(ipp[us]*.15km/1us) in km + #self.__txA = txAus[()]*.15 #(ipp[us]*.15km/1us) in km + self.__txA = txAus[()] #seconds + self.__txAKm = self.__txA*1e6*.15 self.__txB = 0 nWindows=1 self.__nSamples = self.nsa @@ -224,6 +230,7 @@ class AMISRReader(ProcessingUnit): self.__codeType = 1 self.__nCode = self.nCode self.__nBaud = self.nBaud + self.__baudTX = self.__txA/(self.nBaud) #self.__code = 0 #filling system header parameters @@ -493,10 +500,12 @@ class AMISRReader(ProcessingUnit): new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64") ############################################ profPerCH = int(self.profPerBlockRAW / (self.nFFT* self.nChannels)) - + #profPerCH = int(self.profPerBlockRAW / self.nChannels) for thisChannel in range(nchan): - idx_ch = [thisChannel+(k*nchan*self.nFFT) for k in range(profPerCH)] + + idx_ch = [self.nFFT*(thisChannel + nchan*k) for k in range(profPerCH)] + #print(idx_ch) if self.nFFT > 1: aux = [numpy.arange(i, i+self.nFFT) for i in idx_ch] idx_ch = None @@ -522,17 +531,7 @@ class AMISRReader(ProcessingUnit): def fillJROHeader(self): #fill radar controller header - self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm, - txA=self.__txA, - txB=0, - nWindows=1, - nHeights=self.__nSamples, - firstHeight=self.__firstHeight, - deltaHeight=self.__deltaHeight, - codeType=self.__codeType, - nCode=self.__nCode, nBaud=self.__nBaud, - code = self.__code, - fClock=self.__sampleRate) + #fill system header self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples, nProfiles=self.newProfiles, @@ -552,6 +551,7 @@ class AMISRReader(ProcessingUnit): ranges = numpy.reshape(self.rangeFromFile[()],(-1)) self.dataOut.heightList = ranges/1000.0 #km self.dataOut.channelList = self.__channelList + self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights # self.dataOut.channelIndexList = None @@ -560,6 +560,10 @@ class AMISRReader(ProcessingUnit): self.dataOut.azimuthList = numpy.array(self.azimuthList) self.dataOut.elevationList = numpy.array(self.elevationList) self.dataOut.codeList = numpy.array(self.beamCode) + + + + #print(self.dataOut.elevationList) self.dataOut.flagNoData = True @@ -581,14 +585,37 @@ class AMISRReader(ProcessingUnit): self.dataOut.flagDeflipData = False #asumo que la data esta sin flip self.dataOut.flagShiftFFT = False self.dataOut.ippSeconds = self.ippSeconds - self.dataOut.radar_ipp = self.ippSeconds - self.dataOut.pulseLength_TxA = self.__txA/0.15 - self.dataOut.deltaHeight = self.__deltaHeight - #Time interval between profiles - #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt + self.dataOut.ipp = self.__ippKm + self.dataOut.frequency = self.__frequency self.dataOut.realtime = self.online + + self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm, + txA=self.__txAKm, + txB=0, + nWindows=1, + nHeights=self.__nSamples, + firstHeight=self.__firstHeight, + codeType=self.__codeType, + nCode=self.__nCode, nBaud=self.__nBaud, + code = self.__code, + nOsamp=self.nOsamp, + frequency = self.__frequency, + sampleRate= self.__sampleRate, + fClock=self.__sampleRate) + + + self.dataOut.radarControllerHeaderObj.heightList = ranges/1000.0 #km + self.dataOut.radarControllerHeaderObj.heightResolution = self.__deltaHeight + self.dataOut.radarControllerHeaderObj.rangeIpp = self.__ippKm #km + self.dataOut.radarControllerHeaderObj.rangeTxA = self.__txA*1e6*.15 #km + self.dataOut.radarControllerHeaderObj.nChannels = self.nchannels + self.dataOut.radarControllerHeaderObj.channelList = self.__channelList + self.dataOut.radarControllerHeaderObj.azimuthList = self.azimuthList + self.dataOut.radarControllerHeaderObj.elevationList = self.elevationList + self.dataOut.radarControllerHeaderObj.dtype = "Voltage" + self.dataOut.ippSeconds = self.ippSeconds pass def readNextFile(self,online=False): diff --git a/schainpy/model/io/jroIO_param.py b/schainpy/model/io/jroIO_param.py index 716a6e6..3eb811a 100644 --- a/schainpy/model/io/jroIO_param.py +++ b/schainpy/model/io/jroIO_param.py @@ -94,7 +94,7 @@ class HDFReader(Reader, ProcessingUnit): self.filefmt = "*%Y%j***" self.folderfmt = "*%Y%j" self.utcoffset = 0 - + self.flagUpdateDataOut = False self.dataOut = Parameters() self.dataOut.error=False ## NOTE: Importante definir esto antes inicio self.dataOut.flagNoData = True @@ -147,19 +147,57 @@ class HDFReader(Reader, ProcessingUnit): + # def readFirstHeader(self): + # '''Read metadata and data''' + # + # self.__readMetadata2() + # self.__readData() + # self.__setBlockList() + # + # for attr in self.meta: + # setattr(self.dataOut, attr, self.meta[attr]) + # self.blockIndex = 0 + # + # return + def readFirstHeader(self): '''Read metadata and data''' - self.__readMetadata() + self.__readMetadata2() self.__readData() self.__setBlockList() - for attr in self.meta: - setattr(self.dataOut, attr, self.meta[attr]) + if "processingHeaderObj" in attr: + self.flagUpdateDataOut=True + at = attr.split('.') + if len(at) > 1: + setattr(eval("self.dataOut."+at[0]),at[1], self.meta[attr]) + else: + setattr(self.dataOut, attr, self.meta[attr]) self.blockIndex = 0 + if self.flagUpdateDataOut: + self.updateDataOut() + return + def updateDataOut(self): + self.dataOut.azimuthList = self.dataOut.processingHeaderObj.azimuthList + self.dataOut.elevationList = self.dataOut.processingHeaderObj.elevationList + self.dataOut.heightList = self.dataOut.processingHeaderObj.heightList + self.dataOut.ippSeconds = self.dataOut.processingHeaderObj.ipp + self.dataOut.elevationList = self.dataOut.processingHeaderObj.elevationList + self.dataOut.channelList = self.dataOut.processingHeaderObj.channelList + self.dataOut.nCohInt = self.dataOut.processingHeaderObj.nCohInt + self.dataOut.nFFTPoints = self.dataOut.processingHeaderObj.nFFTPoints + self.flagUpdateDataOut = False + self.dataOut.frequency = self.dataOut.radarControllerHeaderObj.frequency + #self.dataOut.heightList = self.dataOut.processingHeaderObj.heightList + + + return + + def __setBlockList(self): ''' Selects the data within the times defined @@ -214,6 +252,38 @@ class HDFReader(Reader, ProcessingUnit): return + def __readMetadata2(self): + ''' + Reads Metadata + ''' + + meta = {} + + if self.description: + for key, value in self.description['Metadata'].items(): + meta[key] = self.fp[value][()] + else: + grp = self.fp['Metadata'] + for item in grp.values(): + name = item.name + if isinstance(item, h5py.Dataset): + name = name.split("/")[-1] + meta[name] = item[()] + else: + grp2 = self.fp[name] + Obj = name.split("/")[-1] + + for item2 in grp2.values(): + name2 = Obj+"."+item2.name.split("/")[-1] + meta[name2] = item2[()] + + if self.extras: + for key, value in self.extras.items(): + meta[key] = value + self.meta = meta + #print(meta) + return + def checkForRealPath(self, nextFile, nextDay): @@ -406,6 +476,8 @@ class HDFWriter(Operation): if self.metadataList is None: self.metadataList = self.dataOut.metadata_list + self.metadataList = list(set(self.metadataList)) + tableList = [] dsList = [] @@ -587,6 +659,56 @@ class HDFWriter(Operation): grp.create_dataset(self.getLabel(self.metadataList[i]), data=value) return + def writeMetadata2(self, fp): + + if self.description: + if 'Metadata' in self.description: + grp = fp.create_group('Metadata') + else: + grp = fp + else: + grp = fp.create_group('Metadata') + + + for i in range(len(self.metadataList)): + + attribute = self.metadataList[i] + attr = attribute.split('.') + if len(attr) > 1: + if not hasattr(eval("self.dataOut."+attr[0]),attr[1]): + log.warning('Metadata: {}.{} not found'.format(attr[0],attr[1]), self.name) + continue + value = getattr(eval("self.dataOut."+attr[0]),attr[1]) + if isinstance(value, bool): + if value is True: + value = 1 + else: + value = 0 + if isinstance(value,type(None)): + log.warning("Invalid value detected, {} is None".format(attribute), self.name) + value = 0 + grp2 = None + if not 'Metadata/'+attr[0] in fp: + grp2 = fp.create_group('Metadata/'+attr[0]) + else: + grp2 = fp['Metadata/'+attr[0]] + #print("attribute: ", attribute, value) + grp2.create_dataset(attr[1], data=value) + + else: + if not hasattr(self.dataOut, attr[0] ): + log.warning('Metadata: `{}` not found'.format(attribute), self.name) + continue + value = getattr(self.dataOut, attr[0]) + if isinstance(value, bool): + if value is True: + value = 1 + else: + value = 0 + grp.create_dataset(self.getLabel(attribute), data=value) + + return + def writeData(self, fp): if self.description: @@ -649,7 +771,7 @@ class HDFWriter(Operation): #Setting HDF5 File self.fp = h5py.File(self.filename, 'w') #write metadata - self.writeMetadata(self.fp) + self.writeMetadata2(self.fp) #Write data self.writeData(self.fp) log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name) diff --git a/schainpy/model/io/utilsIO.py b/schainpy/model/io/utilsIO.py index 83ddc54..ce36287 100644 --- a/schainpy/model/io/utilsIO.py +++ b/schainpy/model/io/utilsIO.py @@ -123,6 +123,12 @@ class MergeH5(object): self.filefmt = "*%Y%j***" self.folderfmt = "*%Y%j" + self.flag_spc = False + self.flag_pow = False + self.flag_snr = False + self.flag_nIcoh = False + self.flagProcessingHeader = False + self.flagControllerHeader = False def setup(self): @@ -187,12 +193,25 @@ class MergeH5(object): '''Read metadata and data''' - self.readMetadata(fp) + self.readMetadata(fp,ch) + #print(self.metadataList) data = self.readData(fp) for attr in self.meta: - setattr(self.ch_dataIn[ch], attr, self.meta[attr]) + + if "processingHeaderObj" in attr: + self.flagProcessingHeader=True + + if "radarControllerHeaderObj" in attr: + self.flagControllerHeader=True + + at = attr.split('.') + #print("AT ", at) + if len(at) > 1: + setattr(eval("self.ch_dataIn[ch]."+at[0]),at[1], self.meta[attr]) + else: + setattr(self.ch_dataIn[ch], attr, self.meta[attr]) self.fill_dataIn(data, self.ch_dataIn[ch]) @@ -200,33 +219,57 @@ class MergeH5(object): return - def readMetadata(self, fp): + def readMetadata(self, fp, ch): ''' Reads Metadata ''' meta = {} self.metadataList = [] grp = fp['Metadata'] - for name in grp: - meta[name] = grp[name][()] - self.metadataList.append(name) + for item in grp.values(): + name = item.name + + if isinstance(item, h5py.Dataset): + name = name.split("/")[-1] + if 'List' in name: + meta[name] = item[()].tolist() + else: + meta[name] = item[()] + self.metadataList.append(name) + else: + grp2 = fp[name] + Obj = name.split("/")[-1] + #print(Obj) + for item2 in grp2.values(): + name2 = Obj+"."+item2.name.split("/")[-1] + if 'List' in name2: + meta[name2] = item2[()].tolist() + else: + meta[name2] = item2[()] + self.metadataList.append(name2) + + - for k in meta: - if ('List' in k): - meta[k] = meta[k].tolist() if not self.meta: - self.meta = meta - self.meta["channelList"] =[n for n in range(self.nChannels)] + self.meta = meta.copy() + for key in list(self.meta.keys()): + if "channelList" in key: + self.meta["channelList"] =[n for n in range(self.nChannels)] + if "processingHeaderObj" in key: + self.meta["processingHeaderObj.channelList"] =[n for n in range(self.nChannels)] + if "radarControllerHeaderObj" in key: + self.meta["radarControllerHeaderObj.channelList"] =[n for n in range(self.nChannels)] return 1 + else: - if len(self.meta) == len(meta): - for k in meta: - if 'List' in k and 'channel' not in k and "height" not in k: - self.meta[k] += meta[k] - return 1 - else: - return 0 + for k in list(self.meta.keys()): + if 'List' in k and 'channel' not in k and "height" not in k and "radarControllerHeaderObj" not in k: + self.meta[k] += meta[k] + + #print("Metadata: ",self.meta) + return 1 + @@ -237,17 +280,23 @@ class MergeH5(object): setattr(dataIn, attr, data[attr][:]) else: setattr(dataIn, attr, numpy.squeeze(data[attr][:,:])) - print("shape in", dataIn.data_spc.shape, len(dataIn.data_spc)) - if dataIn.data_spc.ndim > 3: - dataIn.data_spc = dataIn.data_spc[0] - #print("shape in", dataIn.data_spc.shape) + #print("shape in", dataIn.data_spc.shape, len(dataIn.data_spc)) + if self.flag_spc: + if dataIn.data_spc.ndim > 3: + dataIn.data_spc = dataIn.data_spc[0] + #print("shape in", dataIn.data_spc.shape) def getBlocksPerFile(self): b = numpy.zeros(self.nChannels) for i in range(self.nChannels): - b[i] = self.ch_dataIn[i].data_spc.shape[0] #number of blocks + if self.flag_spc: + b[i] = self.ch_dataIn[i].data_spc.shape[0] #number of blocks + elif self.flag_pow: + b[i] = self.ch_dataIn[i].data_pow.shape[0] #number of blocks + elif self.flag_snr: + b[i] = self.ch_dataIn[i].data_snr.shape[0] #number of blocks self.blocksPerFile = int(b.min()) iresh_ch = numpy.where(b > self.blocksPerFile)[0] @@ -307,6 +356,12 @@ class MergeH5(object): grp = fp['Data'] for name in grp: + if "spc" in name: + self.flag_spc = True + if "pow" in name: + self.flag_pow = True + if "snr" in name: + self.flag_snr = True if isinstance(grp[name], h5py.Dataset): array = grp[name][()] elif isinstance(grp[name], h5py.Group): @@ -323,70 +378,153 @@ class MergeH5(object): def getDataOut(self): self.dataOut = self.ch_dataIn[0].copy() #dataIn #blocks, fft, hei for metadata - if self.dataOut.data_spc.ndim < 3: - return 0 + if self.flagProcessingHeader: + self.dataOut.processingHeaderObj = self.ch_dataIn[0].processingHeaderObj.copy() + self.dataOut.heightList = self.dataOut.processingHeaderObj.heightList + self.dataOut.ippSeconds = self.dataOut.processingHeaderObj.ipp + self.dataOut.channelList = self.dataOut.processingHeaderObj.channelList + self.dataOut.nCohInt = self.dataOut.processingHeaderObj.nCohInt + self.dataOut.nFFTPoints = self.dataOut.processingHeaderObj.nFFTPoints + + if self.flagControllerHeader: + self.dataOut.radarControllerHeaderObj = self.ch_dataIn[0].radarControllerHeaderObj.copy() + self.dataOut.frequency = self.dataOut.radarControllerHeaderObj.frequency + #-------------------------------------------------------------------- + + + #-------------------------------------------------------------------- + if self.flag_spc: + if self.dataOut.data_spc.ndim < 3: + print("shape spc in: ",self.dataOut.data_spc.shape ) + return 0 + if self.flag_pow: + if self.dataOut.data_pow.ndim < 2: + print("shape pow in: ",self.dataOut.data_pow.shape ) + return 0 + if self.flag_snr: + if self.dataOut.data_snr.ndim < 2: + print("shape snr in: ",self.dataOut.data_snr.shape ) + return 0 + self.dataOut.data_spc = None + self.dataOut.data_cspc = None + self.dataOut.data_pow = None + self.dataOut.data_snr = None self.dataOut.utctim = None self.dataOut.nIncohInt = None - #print(self.ch_dataIn[0].data_spc.shape) - spc = [data.data_spc for data in self.ch_dataIn] - self.dataOut.data_spc = numpy.stack(spc, axis=1) #blocks, ch, fft, hei + #-------------------------------------------------------------------- + if self.flag_spc: + spc = [data.data_spc for data in self.ch_dataIn] + self.dataOut.data_spc = numpy.stack(spc, axis=1) #blocks, ch, fft, hei + #-------------------------------------------------------------------- + if self.flag_pow: + pow = [data.data_pow for data in self.ch_dataIn] + self.dataOut.data_pow = numpy.stack(pow, axis=1) #blocks, ch, fft, hei + #-------------------------------------------------------------------- + if self.flag_snr: + snr = [data.data_snr for data in self.ch_dataIn] + self.dataOut.data_snr = numpy.stack(snr, axis=1) #blocks, ch, fft, hei + + #-------------------------------------------------------------------- time = [data.utctime for data in self.ch_dataIn] time = numpy.asarray(time).mean(axis=0) - #time = numpy.reshape(time, (len(time),1)) time = numpy.squeeze(time) self.dataOut.utctime = time - ints = [data.nIncohInt for data in self.ch_dataIn] - self.dataOut.nIncohInt = numpy.stack(ints, axis=1) - - - if self.dataOut.nIncohInt.ndim > 3: - aux = self.dataOut.nIncohInt - self.dataOut.nIncohInt = None - self.dataOut.nIncohInt = aux[0] - + #-------------------------------------------------------------------- + if self.flag_nIcoh: + ints = [data.nIncohInt for data in self.ch_dataIn] + self.dataOut.nIncohInt = numpy.stack(ints, axis=1) + + if self.dataOut.nIncohInt.ndim > 3: + aux = self.dataOut.nIncohInt + self.dataOut.nIncohInt = None + self.dataOut.nIncohInt = aux[0] + + if self.dataOut.nIncohInt.ndim < 3: + nIncohInt = numpy.repeat(self.dataOut.nIncohInt, self.dataOut.nHeights).reshape(self.blocksPerFile,self.nChannels, self.dataOut.nHeights) + #nIncohInt = numpy.reshape(nIncohInt, (self.blocksPerFile,self.nChannels, self.dataOut.nHeights)) + self.dataOut.nIncohInt = None + self.dataOut.nIncohInt = nIncohInt + + if (self.dataOut.nIncohInt.shape)[0]==self.nChannels: ## ch,blocks, hei + self.dataOut.nIncohInt = numpy.swapaxes(self.dataOut.nIncohInt, 0, 1) ## blocks,ch, hei + #-------------------------------------------------------------------- + #print("utcTime: ", time.shape) + #print("data_spc ",self.dataOut.data_spc.shape) + if "data_cspc" in self.dataList: + pairsList = [pair for pair in itertools.combinations(self.channelList, 2)] + #print("PairsList: ", pairsList) + self.dataOut.pairsList = pairsList + cspc = [] - if self.dataOut.nIncohInt.ndim < 3: - nIncohInt = numpy.repeat(self.dataOut.nIncohInt, self.dataOut.nHeights).reshape(self.blocksPerFile,self.nChannels, self.dataOut.nHeights) - #nIncohInt = numpy.reshape(nIncohInt, (self.blocksPerFile,self.nChannels, self.dataOut.nHeights)) - self.dataOut.nIncohInt = None - self.dataOut.nIncohInt = nIncohInt + for i, j in pairsList: + cspc.append(self.ch_dataIn[i].data_spc*numpy.conjugate(self.ch_dataIn[j].data_spc)) #blocks, fft, hei - if (self.dataOut.nIncohInt.shape)[0]==self.nChannels: ## ch,blocks, hei - self.dataOut.nIncohInt = numpy.swapaxes(self.dataOut.nIncohInt, 0, 1) ## blocks,ch, hei + cspc = numpy.asarray(cspc) # # pairs, blocks, fft, hei + #print("cspc: ",cspc.shape) + self.dataOut.data_cspc = numpy.swapaxes(cspc, 0, 1) ## blocks, pairs, fft, hei + #print("dataOut.data_cspc: ",self.dataOut.data_cspc.shape) + #if "data_pow" in self.dataList: - #print("utcTime: ", time.shape) - #print("data_spc ",self.dataOut.data_spc.shape) - pairsList = [pair for pair in itertools.combinations(self.channelList, 2)] - #print("PairsList: ", pairsList) - self.dataOut.pairsList = pairsList - cspc = [] - - for i, j in pairsList: - cspc.append(self.ch_dataIn[i].data_spc*numpy.conjugate(self.ch_dataIn[j].data_spc)) #blocks, fft, hei - - cspc = numpy.asarray(cspc) # # pairs, blocks, fft, hei - #print("cspc: ",cspc.shape) - self.dataOut.data_cspc = numpy.swapaxes(cspc, 0, 1) ## blocks, pairs, fft, hei - #print("dataOut.data_cspc: ",self.dataOut.data_cspc.shape) return 1 + # def writeMetadata(self, fp): + # + # + # grp = fp.create_group('Metadata') + # + # for i in range(len(self.metadataList)): + # if not hasattr(self.dataOut, self.metadataList[i]): + # print('Metadata: `{}` not found'.format(self.metadataList[i])) + # continue + # value = getattr(self.dataOut, self.metadataList[i]) + # if isinstance(value, bool): + # if value is True: + # value = 1 + # else: + # value = 0 + # grp.create_dataset(self.getLabel(self.metadataList[i]), data=value) + # return def writeMetadata(self, fp): grp = fp.create_group('Metadata') for i in range(len(self.metadataList)): - if not hasattr(self.dataOut, self.metadataList[i]): - print('Metadata: `{}` not found'.format(self.metadataList[i])) - continue - value = getattr(self.dataOut, self.metadataList[i]) - if isinstance(value, bool): - if value is True: - value = 1 + attribute = self.metadataList[i] + attr = attribute.split('.') + if '' in attr: + attr.remove('') + #print(attr) + if len(attr) > 1: + if not hasattr(eval("self.dataOut."+attr[0]),attr[1]): + print('Metadata: {}.{} not found'.format(attr[0],attr[1])) + continue + value = getattr(eval("self.dataOut."+attr[0]),attr[1]) + if isinstance(value, bool): + if value is True: + value = 1 + else: + value = 0 + grp2 = None + if not 'Metadata/'+attr[0] in fp: + grp2 = fp.create_group('Metadata/'+attr[0]) else: - value = 0 - grp.create_dataset(self.getLabel(self.metadataList[i]), data=value) + grp2 = fp['Metadata/'+attr[0]] + + grp2.create_dataset(attr[1], data=value) + + else: + if not hasattr(self.dataOut, attr[0] ): + print('Metadata: `{}` not found'.format(attribute)) + continue + value = getattr(self.dataOut, attr[0]) + if isinstance(value, bool): + if value is True: + value = 1 + else: + value = 0 + grp.create_dataset(self.getLabel(attribute), data=value) return def getDsList(self): diff --git a/schainpy/model/proc/jroproc_parameters.py b/schainpy/model/proc/jroproc_parameters.py index cc474f9..2c7e469 100755 --- a/schainpy/model/proc/jroproc_parameters.py +++ b/schainpy/model/proc/jroproc_parameters.py @@ -69,14 +69,14 @@ class ParametersProc(ProcessingUnit): self.dataOut.dstFlag = self.dataIn.dstFlag self.dataOut.errorCount = self.dataIn.errorCount self.dataOut.useLocalTime = self.dataIn.useLocalTime - self.dataOut.radar_ipp = self.dataIn.radar_ipp + self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy() - #self.dataOut.radarControllerHeaderTxt = self.dataOut.radarControllerHeaderObj.toString() + self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy() self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy() self.dataOut.channelList = self.dataIn.channelList self.dataOut.heightList = self.dataIn.heightList - self.dataOut.sampled_heightsFFT = self.dataIn.sampled_heightsFFT - self.dataOut.pulseLength_TxA = self.dataIn.pulseLength_TxA + self.dataOut.ipp = self.dataIn.ipp + self.dataOut.ippSeconds = self.dataIn.ippSeconds self.dataOut.deltaHeight = self.dataIn.deltaHeight self.dataOut.dtype = numpy.dtype([('real','