jroproc_spectra.py
1736 lines
| 60.8 KiB
| text/x-python
|
PythonLexer
| r1338 | # Copyright (c) 2012-2020 Jicamarca Radio Observatory | ||
| # All rights reserved. | |||
| # | |||
| # Distributed under the terms of the BSD 3-clause license. | |||
| """Spectra processing Unit and operations | |||
| Here you will find the processing unit `SpectraProc` and several operations | |||
| to work with Spectra data type | |||
| """ | |||
Juan C. Espinoza
|
r1287 | import time | |
|
Juan C. Espinoza
|
r1062 | import itertools | |
Alexander Valdez
|
r1699 | ||
Daniel Valdez
|
r487 | import numpy | |
|
Alexander Valdez
|
r1699 | ||
George Yong
|
r1171 | from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation | |
Miguel Valdez
|
r568 | from schainpy.model.data.jrodata import Spectra | |
| from schainpy.model.data.jrodata import hildebrand_sekhon | |||
| r1739 | from schainpy.model.data import _noise | ||
|
George Yong
|
r1171 | from schainpy.utils import log | |
| r1739 | import matplotlib.pyplot as plt | ||
| from schainpy.model.io.utilsIO import getHei_index | |||
| import datetime | |||
|
Juan C. Espinoza
|
r1287 | ||
|
Daniel Valdez
|
r487 | class SpectraProc(ProcessingUnit): | |
José Chávez
|
r897 | ||
|
George Yong
|
r1179 | def __init__(self): | |
|
George Yong
|
r1171 | ||
|
George Yong
|
r1179 | ProcessingUnit.__init__(self) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.buffer = None | |
| self.firstdatatime = None | |||
| self.profIndex = 0 | |||
| self.dataOut = Spectra() | |||
| r1749 | self.dataOut.error=False | ||
|
Daniel Valdez
|
r495 | self.id_min = None | |
| self.id_max = None | |||
|
George Yong
|
r1171 | self.setupReq = False #Agregar a todas las unidades de proc | |
| r1739 | self.nsamplesFFT = 0 | ||
|
Daniel Valdez
|
r487 | ||
|
Miguel Valdez
|
r623 | def __updateSpecFromVoltage(self): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.dataOut.timeZone = self.dataIn.timeZone | |
| self.dataOut.dstFlag = self.dataIn.dstFlag | |||
| self.dataOut.errorCount = self.dataIn.errorCount | |||
| self.dataOut.useLocalTime = self.dataIn.useLocalTime | |||
|
José Chávez
|
r1120 | try: | |
| self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy() | |||
| except: | |||
| pass | |||
|
Daniel Valdez
|
r487 | self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy() | |
| r1739 | self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy() | ||
| self.dataOut.ippSeconds = self.dataIn.ippSeconds | |||
| self.dataOut.ipp = self.dataIn.ipp | |||
|
Daniel Valdez
|
r487 | self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy() | |
| self.dataOut.channelList = self.dataIn.channelList | |||
| self.dataOut.heightList = self.dataIn.heightList | |||
|
José Chávez
|
r1120 | self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')]) | |
|
Daniel Valdez
|
r487 | self.dataOut.nProfiles = self.dataOut.nFFTPoints | |
|
Miguel Valdez
|
r568 | self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock | |
|
Daniel Valdez
|
r487 | self.dataOut.utctime = self.firstdatatime | |
|
José Chávez
|
r1120 | self.dataOut.flagDecodeData = self.dataIn.flagDecodeData | |
| self.dataOut.flagDeflipData = self.dataIn.flagDeflipData | |||
|
Miguel Valdez
|
r623 | self.dataOut.flagShiftFFT = False | |
|
Daniel Valdez
|
r487 | self.dataOut.nCohInt = self.dataIn.nCohInt | |
| self.dataOut.nIncohInt = 1 | |||
| r1739 | self.dataOut.deltaHeight = self.dataIn.deltaHeight | ||
|
Daniel Valdez
|
r487 | self.dataOut.windowOfFilter = self.dataIn.windowOfFilter | |
| self.dataOut.frequency = self.dataIn.frequency | |||
| self.dataOut.realtime = self.dataIn.realtime | |||
|
Daniel Valdez
|
r499 | self.dataOut.azimuth = self.dataIn.azimuth | |
| self.dataOut.zenith = self.dataIn.zenith | |||
| r1739 | self.dataOut.codeList = self.dataIn.codeList | ||
| self.dataOut.azimuthList = self.dataIn.azimuthList | |||
| self.dataOut.elevationList = self.dataIn.elevationList | |||
| self.dataOut.code = self.dataIn.code | |||
| self.dataOut.nCode = self.dataIn.nCode | |||
| self.dataOut.flagProfilesByRange = self.dataIn.flagProfilesByRange | |||
| self.dataOut.nProfilesByRange = self.dataIn.nProfilesByRange | |||
|
Alexander Valdez
|
r1675 | self.dataOut.runNextUnit = self.dataIn.runNextUnit | |
| try: | |||
| self.dataOut.step = self.dataIn.step | |||
| except: | |||
| pass | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def __getFft(self): | |
| """ | |||
| Convierte valores de Voltaje a Spectra | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | Affected: | |
| self.dataOut.data_spc | |||
| self.dataOut.data_cspc | |||
| self.dataOut.data_dc | |||
| self.dataOut.heightList | |||
|
José Chávez
|
r897 | self.profIndex | |
|
Daniel Valdez
|
r487 | self.buffer | |
| self.dataOut.flagNoData | |||
| """ | |||
|
José Chávez
|
r1120 | fft_volt = numpy.fft.fft( | |
| self.buffer, n=self.dataOut.nFFTPoints, axis=1) | |||
|
Daniel Valdez
|
r487 | fft_volt = fft_volt.astype(numpy.dtype('complex')) | |
|
José Chávez
|
r1120 | dc = fft_volt[:, 0, :] | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # calculo de self-spectra | |
| fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,)) | |||
|
Daniel Valdez
|
r487 | spc = fft_volt * numpy.conjugate(fft_volt) | |
| spc = spc.real | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | blocksize = 0 | |
| blocksize += dc.size | |||
| blocksize += spc.size | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | cspc = None | |
| pairIndex = 0 | |||
| if self.dataOut.pairsList != None: | |||
|
José Chávez
|
r1120 | # calculo de cross-spectra | |
| cspc = numpy.zeros( | |||
| (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex') | |||
|
Daniel Valdez
|
r487 | for pair in self.dataOut.pairsList: | |
|
Miguel Valdez
|
r587 | if pair[0] not in self.dataOut.channelList: | |
|
George Yong
|
r1167 | raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % ( | |
| str(pair), str(self.dataOut.channelList))) | |||
|
Miguel Valdez
|
r587 | if pair[1] not in self.dataOut.channelList: | |
|
George Yong
|
r1167 | raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % ( | |
| str(pair), str(self.dataOut.channelList))) | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \ | |
| numpy.conjugate(fft_volt[pair[1], :, :]) | |||
|
Daniel Valdez
|
r487 | pairIndex += 1 | |
| blocksize += cspc.size | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.dataOut.data_spc = spc | |
| self.dataOut.data_cspc = cspc | |||
| self.dataOut.data_dc = dc | |||
| self.dataOut.blockSize = blocksize | |||
| r1270 | self.dataOut.flagShiftFFT = False | ||
|
José Chávez
|
r897 | ||
| r1739 | def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False, | ||
| r1749 | zeroPad=False, zeroPoints=0, runNextUnit=0): | ||
|
Alexander Valdez
|
r1675 | self.dataIn.runNextUnit = runNextUnit | |
| r1739 | try: | ||
| type = self.dataIn.type.decode("utf-8") | |||
| self.dataIn.type = type | |||
| r1749 | except Exception as e: | ||
| # print("spc -> ",e) | |||
| r1739 | pass | ||
|
Daniel Valdez
|
r487 | if self.dataIn.type == "Spectra": | |
| r1749 | #print("AQUI") | ||
| r1739 | try: | ||
| self.dataOut.copy(self.dataIn) | |||
| self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy() | |||
| self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy() | |||
| self.dataOut.nProfiles = self.dataOut.nFFTPoints | |||
| #self.dataOut.nHeights = len(self.dataOut.heightList) | |||
| except Exception as e: | |||
| print("Error dataIn ",e) | |||
| r1132 | if shift_fft: | ||
| #desplaza a la derecha en el eje 2 determinadas posiciones | |||
| shift = int(self.dataOut.nFFTPoints/2) | |||
| self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1) | |||
| if self.dataOut.data_cspc is not None: | |||
| #desplaza a la derecha en el eje 2 determinadas posiciones | |||
| r1151 | self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1) | ||
| r1341 | if pairsList: | ||
| self.__selectPairs(pairsList) | |||
|
George Yong
|
r1171 | ||
| r1338 | elif self.dataIn.type == "Voltage": | ||
|
José Chávez
|
r897 | ||
|
George Yong
|
r1183 | self.dataOut.flagNoData = True | |
| r1739 | self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy() | ||
| self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy() | |||
|
George Yong
|
r1183 | ||
|
Daniel Valdez
|
r487 | if nFFTPoints == None: | |
|
George Yong
|
r1167 | raise ValueError("This SpectraProc.run() need nFFTPoints input variable") | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r495 | if nProfiles == None: | |
|
Miguel Valdez
|
r568 | nProfiles = nFFTPoints | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if ippFactor == None: | |
| r1739 | self.dataOut.ippFactor = self.dataIn.ippFactor | ||
| else: | |||
| self.dataOut.ippFactor = ippFactor | |||
| r1338 | |||
| r1739 | if self.buffer is None: | ||
| if not zeroPad: | |||
| self.buffer = numpy.zeros((self.dataIn.nChannels, | |||
| nProfiles, | |||
| self.dataIn.nHeights), | |||
| dtype='complex') | |||
| zeroPoints = 0 | |||
| else: | |||
| self.buffer = numpy.zeros((self.dataIn.nChannels, | |||
| nFFTPoints+int(zeroPoints), | |||
| self.dataIn.nHeights), | |||
| dtype='complex') | |||
|
Daniel Valdez
|
r487 | self.dataOut.nFFTPoints = nFFTPoints | |
|
Daniel Valdez
|
r495 | ||
|
Miguel Valdez
|
r611 | if self.buffer is None: | |
|
José Chávez
|
r1120 | self.buffer = numpy.zeros((self.dataIn.nChannels, | |
| nProfiles, | |||
| self.dataIn.nHeights), | |||
|
Miguel Valdez
|
r623 | dtype='complex') | |
|
Daniel Valdez
|
r487 | ||
|
Miguel Valdez
|
r623 | if self.dataIn.flagDataAsBlock: | |
|
Miguel Valdez
|
r720 | nVoltProfiles = self.dataIn.data.shape[1] | |
| r1739 | zeroPoints = 0 | ||
| if nVoltProfiles == nProfiles or zeroPad: | |||
|
Daniel Valdez
|
r495 | self.buffer = self.dataIn.data.copy() | |
|
Miguel Valdez
|
r720 | self.profIndex = nVoltProfiles | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r720 | elif nVoltProfiles < nProfiles: | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | if self.profIndex == 0: | |
| self.id_min = 0 | |||
|
Miguel Valdez
|
r720 | self.id_max = nVoltProfiles | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | self.buffer[:, self.id_min:self.id_max, | |
| :] = self.dataIn.data | |||
|
Miguel Valdez
|
r720 | self.profIndex += nVoltProfiles | |
| self.id_min += nVoltProfiles | |||
| self.id_max += nVoltProfiles | |||
|
Alexander Valdez
|
r1668 | elif nVoltProfiles > nProfiles: | |
| self.reader.bypass = True | |||
| if self.profIndex == 0: | |||
| self.id_min = 0 | |||
| self.id_max = nProfiles | |||
| self.buffer = self.dataIn.data[:, self.id_min:self.id_max,:] | |||
| self.profIndex += nProfiles | |||
| self.id_min += nProfiles | |||
| self.id_max += nProfiles | |||
| if self.id_max == nVoltProfiles: | |||
| self.reader.bypass = False | |||
|
Daniel Valdez
|
r495 | else: | |
|
George Yong
|
r1167 | raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % ( | |
| self.dataIn.type, self.dataIn.data.shape[1], nProfiles)) | |||
|
Daniel Valdez
|
r495 | self.dataOut.flagNoData = True | |
|
José Chávez
|
r897 | else: | |
|
José Chávez
|
r1120 | self.buffer[:, self.profIndex, :] = self.dataIn.data.copy() | |
|
Miguel Valdez
|
r623 | self.profIndex += 1 | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if self.firstdatatime == None: | |
| self.firstdatatime = self.dataIn.utctime | |||
|
José Chávez
|
r897 | ||
| r1739 | if self.profIndex == nProfiles or (zeroPad and zeroPoints==0): | ||
|
Miguel Valdez
|
r623 | self.__updateSpecFromVoltage() | |
| r1338 | if pairsList == None: | ||
| self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)] | |||
| r1341 | else: | ||
| self.dataOut.pairsList = pairsList | |||
|
Daniel Valdez
|
r487 | self.__getFft() | |
| self.dataOut.flagNoData = False | |||
| self.firstdatatime = None | |||
| r1739 | self.nsamplesFFT = self.profIndex | ||
|
Alexander Valdez
|
r1677 | #if not self.reader.bypass: | |
| self.profIndex = 0 | |||
| r1739 | #update Processing Header: | ||
| self.dataOut.processingHeaderObj.dtype = "Spectra" | |||
| self.dataOut.processingHeaderObj.nFFTPoints = self.dataOut.nFFTPoints | |||
| self.dataOut.processingHeaderObj.nSamplesFFT = self.nsamplesFFT | |||
| self.dataOut.processingHeaderObj.nIncohInt = 1 | |||
| elif self.dataIn.type == "Parameters": #when get data from h5 spc file | |||
| self.dataOut.data_spc = self.dataIn.data_spc | |||
| self.dataOut.data_cspc = self.dataIn.data_cspc | |||
| self.dataOut.data_outlier = self.dataIn.data_outlier | |||
| self.dataOut.nProfiles = self.dataIn.nProfiles | |||
| self.dataOut.nIncohInt = self.dataIn.nIncohInt | |||
| self.dataOut.nFFTPoints = self.dataIn.nFFTPoints | |||
| self.dataOut.ippFactor = self.dataIn.ippFactor | |||
| self.dataOut.max_nIncohInt = self.dataIn.max_nIncohInt | |||
| self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy() | |||
| self.dataOut.ProcessingHeader = self.dataIn.ProcessingHeader.copy() | |||
| self.dataOut.ippSeconds = self.dataIn.ippSeconds | |||
| self.dataOut.ipp = self.dataIn.ipp | |||
| #self.dataOut.abscissaList = self.dataIn.getVelRange(1) | |||
| #self.dataOut.spc_noise = self.dataIn.getNoise() | |||
| #self.dataOut.spc_range = (self.dataIn.getFreqRange(1) , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1)) | |||
| # self.dataOut.normFactor = self.dataIn.normFactor | |||
| if hasattr(self.dataIn, 'channelList'): | |||
| self.dataOut.channelList = self.dataIn.channelList | |||
| if hasattr(self.dataIn, 'pairsList'): | |||
| self.dataOut.pairsList = self.dataIn.pairsList | |||
| self.dataOut.groupList = self.dataIn.pairsList | |||
| self.dataOut.flagNoData = False | |||
| if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels | |||
| self.dataOut.ChanDist = self.dataIn.ChanDist | |||
| else: self.dataOut.ChanDist = None | |||
| #if hasattr(self.dataIn, 'VelRange'): #Velocities range | |||
| # self.dataOut.VelRange = self.dataIn.VelRange | |||
| #else: self.dataOut.VelRange = None | |||
| r1338 | else: | ||
| raise ValueError("The type of input object '%s' is not valid".format( | |||
| self.dataIn.type)) | |||
| r1749 | # print("SPC done") | ||
|
Alexander Valdez
|
r1699 | ||
|
Miguel Valdez
|
r730 | def __selectPairs(self, pairsList): | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | if not pairsList: | |
|
Miguel Valdez
|
r730 | return | |
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1062 | pairs = [] | |
| pairsIndex = [] | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1062 | for pair in pairsList: | |
| if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList: | |||
|
Miguel Valdez
|
r730 | continue | |
|
Juan C. Espinoza
|
r1062 | pairs.append(pair) | |
| pairsIndex.append(pairs.index(pair)) | |||
|
José Chávez
|
r1120 | ||
|
Juan C. Espinoza
|
r1062 | self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex] | |
| self.dataOut.pairsList = pairs | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r730 | return | |
|
Alexander Valdez
|
r1675 | ||
| def selectFFTs(self, minFFT, maxFFT ): | |||
ebocanegra
|
r1123 | """ | |
|
Alexander Valdez
|
r1675 | Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango | |
|
ebocanegra
|
r1123 | minFFT<= FFT <= maxFFT | |
| """ | |||
|
Alexander Valdez
|
r1675 | ||
|
ebocanegra
|
r1123 | if (minFFT > maxFFT): | |
|
Juan C. Espinoza
|
r1188 | raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT)) | |
|
ebocanegra
|
r1123 | ||
| if (minFFT < self.dataOut.getFreqRange()[0]): | |||
| minFFT = self.dataOut.getFreqRange()[0] | |||
|
Daniel Valdez
|
r487 | ||
|
ebocanegra
|
r1123 | if (maxFFT > self.dataOut.getFreqRange()[-1]): | |
| maxFFT = self.dataOut.getFreqRange()[-1] | |||
| minIndex = 0 | |||
| maxIndex = 0 | |||
| FFTs = self.dataOut.getFreqRange() | |||
| inda = numpy.where(FFTs >= minFFT) | |||
| indb = numpy.where(FFTs <= maxFFT) | |||
|
Daniel Valdez
|
r487 | ||
|
ebocanegra
|
r1123 | try: | |
| minIndex = inda[0][0] | |||
| except: | |||
| minIndex = 0 | |||
| try: | |||
| maxIndex = indb[0][-1] | |||
| except: | |||
| maxIndex = len(FFTs) | |||
| self.selectFFTsByIndex(minIndex, maxIndex) | |||
| return 1 | |||
|
Alexander Valdez
|
r1675 | ||
|
José Chávez
|
r1120 | def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None): | |
| newheis = numpy.where( | |||
| self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex]) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if hei_ref != None: | |
|
José Chávez
|
r1120 | newheis = numpy.where(self.dataOut.heightList > hei_ref) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | minIndex = min(newheis[0]) | |
| maxIndex = max(newheis[0]) | |||
|
José Chávez
|
r1120 | data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1] | |
| heightList = self.dataOut.heightList[minIndex:maxIndex + 1] | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | # determina indices | |
|
José Chávez
|
r1120 | nheis = int(self.dataOut.radarControllerHeaderObj.txB / | |
| (self.dataOut.heightList[1] - self.dataOut.heightList[0])) | |||
| avg_dB = 10 * \ | |||
| numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0)) | |||
|
Daniel Valdez
|
r487 | beacon_dB = numpy.sort(avg_dB)[-nheis:] | |
| beacon_heiIndexList = [] | |||
| for val in avg_dB.tolist(): | |||
| if val >= beacon_dB[0]: | |||
| beacon_heiIndexList.append(avg_dB.tolist().index(val)) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | data_cspc = None | |
|
Miguel Valdez
|
r612 | if self.dataOut.data_cspc is not None: | |
|
José Chávez
|
r1120 | data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1] | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | data_dc = None | |
|
Miguel Valdez
|
r612 | if self.dataOut.data_dc is not None: | |
|
José Chávez
|
r1120 | data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1] | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.dataOut.data_spc = data_spc | |
| self.dataOut.data_cspc = data_cspc | |||
| self.dataOut.data_dc = data_dc | |||
| self.dataOut.heightList = heightList | |||
| self.dataOut.beacon_heiIndexList = beacon_heiIndexList | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return 1 | |
|
José Chávez
|
r897 | ||
|
ebocanegra
|
r1123 | def selectFFTsByIndex(self, minIndex, maxIndex): | |
| """ | |||
|
Alexander Valdez
|
r1675 | ||
|
ebocanegra
|
r1123 | """ | |
| if (minIndex < 0) or (minIndex > maxIndex): | |||
|
Juan C. Espinoza
|
r1188 | raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex)) | |
|
ebocanegra
|
r1123 | ||
| if (maxIndex >= self.dataOut.nProfiles): | |||
| maxIndex = self.dataOut.nProfiles-1 | |||
| #Spectra | |||
| data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:] | |||
| data_cspc = None | |||
| if self.dataOut.data_cspc is not None: | |||
| data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:] | |||
| data_dc = None | |||
| if self.dataOut.data_dc is not None: | |||
| data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:] | |||
| self.dataOut.data_spc = data_spc | |||
| self.dataOut.data_cspc = data_cspc | |||
| self.dataOut.data_dc = data_dc | |||
|
Alexander Valdez
|
r1675 | ||
|
ebocanegra
|
r1123 | self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1]) | |
| self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1] | |||
| self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1] | |||
| return 1 | |||
|
Juan C. Espinoza
|
r1287 | def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None): | |
| # validacion de rango | |||
| if minHei == None: | |||
| minHei = self.dataOut.heightList[0] | |||
|
ebocanegra
|
r1123 | ||
|
Juan C. Espinoza
|
r1287 | if maxHei == None: | |
| maxHei = self.dataOut.heightList[-1] | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei): | |
| print('minHei: %.2f is out of the heights range' % (minHei)) | |||
| print('minHei is setting to %.2f' % (self.dataOut.heightList[0])) | |||
| minHei = self.dataOut.heightList[0] | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei): | |
| print('maxHei: %.2f is out of the heights range' % (maxHei)) | |||
| print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1])) | |||
| maxHei = self.dataOut.heightList[-1] | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | # validacion de velocidades | |
| velrange = self.dataOut.getVelRange(1) | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | if minVel == None: | |
| minVel = velrange[0] | |||
| if maxVel == None: | |||
| maxVel = velrange[-1] | |||
| if (minVel < velrange[0]) or (minVel > maxVel): | |||
| print('minVel: %.2f is out of the velocity range' % (minVel)) | |||
| print('minVel is setting to %.2f' % (velrange[0])) | |||
| minVel = velrange[0] | |||
| if (maxVel > velrange[-1]) or (maxVel < minVel): | |||
| print('maxVel: %.2f is out of the velocity range' % (maxVel)) | |||
| print('maxVel is setting to %.2f' % (velrange[-1])) | |||
| maxVel = velrange[-1] | |||
| # seleccion de indices para rango | |||
| minIndex = 0 | |||
| maxIndex = 0 | |||
| heights = self.dataOut.heightList | |||
| inda = numpy.where(heights >= minHei) | |||
| indb = numpy.where(heights <= maxHei) | |||
| try: | |||
| minIndex = inda[0][0] | |||
| except: | |||
| minIndex = 0 | |||
| try: | |||
| maxIndex = indb[0][-1] | |||
| except: | |||
| maxIndex = len(heights) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if (minIndex < 0) or (minIndex > maxIndex): | |
|
Juan C. Espinoza
|
r1287 | raise ValueError("some value in (%d,%d) is not valid" % ( | |
|
George Yong
|
r1167 | minIndex, maxIndex)) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if (maxIndex >= self.dataOut.nHeights): | |
|
José Chávez
|
r1120 | maxIndex = self.dataOut.nHeights - 1 | |
|
Daniel Valdez
|
r487 | ||
|
Juan C. Espinoza
|
r1287 | # seleccion de indices para velocidades | |
| indminvel = numpy.where(velrange >= minVel) | |||
| indmaxvel = numpy.where(velrange <= maxVel) | |||
| try: | |||
| minIndexVel = indminvel[0][0] | |||
| except: | |||
| minIndexVel = 0 | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | try: | |
| maxIndexVel = indmaxvel[0][-1] | |||
| except: | |||
| maxIndexVel = len(velrange) | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | # seleccion del espectro | |
| data_spc = self.dataOut.data_spc[:, | |||
| minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1] | |||
| # estimacion de ruido | |||
| noise = numpy.zeros(self.dataOut.nChannels) | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | for channel in range(self.dataOut.nChannels): | |
| daux = data_spc[channel, :, :] | |||
| sortdata = numpy.sort(daux, axis=None) | |||
| noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt) | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | self.dataOut.noise_estimation = noise.copy() | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return 1 | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r1684 | class GetSNR(Operation): | |
| ''' | |||
| Written by R. Flores | |||
| ''' | |||
| """Operation to get SNR. | |||
| Parameters: | |||
| ----------- | |||
| Example | |||
| -------- | |||
| op = proc_unit.addOperation(name='GetSNR', optype='other') | |||
| """ | |||
| def __init__(self, **kwargs): | |||
| Operation.__init__(self, **kwargs) | |||
| def run(self,dataOut): | |||
| noise = dataOut.getNoise(ymin_index=-10) #Región superior donde solo debería de haber ruido | |||
| dataOut.data_snr = (dataOut.data_spc.sum(axis=1)-noise[:,None]*dataOut.nFFTPoints)/(noise[:,None]*dataOut.nFFTPoints) #It works apparently | |||
| dataOut.snl = numpy.log10(dataOut.data_snr) | |||
| dataOut.snl = numpy.where(dataOut.data_snr<.01, numpy.nan, dataOut.snl) | |||
| return dataOut | |||
|
Juan C. Espinoza
|
r1287 | class removeDC(Operation): | |
| def run(self, dataOut, mode=2): | |||
| self.dataOut = dataOut | |||
|
Daniel Valdez
|
r487 | jspectra = self.dataOut.data_spc | |
| jcspectra = self.dataOut.data_cspc | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | num_chan = jspectra.shape[0] | |
| num_hei = jspectra.shape[2] | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r612 | if jcspectra is not None: | |
|
Daniel Valdez
|
r487 | jcspectraExist = True | |
| num_pairs = jcspectra.shape[0] | |||
|
José Chávez
|
r1120 | else: | |
| jcspectraExist = False | |||
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | freq_dc = int(jspectra.shape[1] / 2) | |
|
José Chávez
|
r1120 | ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc | |
|
George Yong
|
r1167 | ind_vel = ind_vel.astype(int) | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | if ind_vel[0] < 0: | |
|
George Yong
|
r1167 | ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof | |
|
José Chávez
|
r897 | ||
| if mode == 1: | |||
|
José Chávez
|
r1120 | jspectra[:, freq_dc, :] = ( | |
| jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if jcspectraExist: | |
|
José Chávez
|
r1120 | jcspectra[:, freq_dc, :] = ( | |
| jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if mode == 2: | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | vel = numpy.array([-2, -1, 1, 2]) | |
| xx = numpy.zeros([4, 4]) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for fil in range(4): | |
|
George Yong
|
r1167 | xx[fil, :] = vel[fil]**numpy.asarray(list(range(4))) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | xx_inv = numpy.linalg.inv(xx) | |
|
José Chávez
|
r1120 | xx_aux = xx_inv[0, :] | |
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r1675 | for ich in range(num_chan): | |
|
José Chávez
|
r1120 | yy = jspectra[ich, ind_vel, :] | |
| jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy) | |||
|
Daniel Valdez
|
r487 | ||
|
José Chávez
|
r1120 | junkid = jspectra[ich, freq_dc, :] <= 0 | |
|
Daniel Valdez
|
r487 | cjunkid = sum(junkid) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if cjunkid.any(): | |
|
José Chávez
|
r1120 | jspectra[ich, freq_dc, junkid.nonzero()] = ( | |
| jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if jcspectraExist: | |
| for ip in range(num_pairs): | |||
|
José Chávez
|
r1120 | yy = jcspectra[ip, ind_vel, :] | |
| jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.dataOut.data_spc = jspectra | |
| self.dataOut.data_cspc = jcspectra | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | return self.dataOut | |
| r1739 | class getNoiseB(Operation): | ||
| """ | |||
| Get noise from custom heights and frequency ranges, | |||
| offset for additional manual correction | |||
| J. Apaza -> developed to amisr isr spectra | |||
| """ | |||
| __slots__ =('offset','warnings', 'isConfig', 'minIndex','maxIndex','minIndexFFT','maxIndexFFT') | |||
| def __init__(self): | |||
| Operation.__init__(self) | |||
| self.isConfig = False | |||
| def setup(self, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False): | |||
| self.warnings = warnings | |||
| if minHei == None: | |||
| minHei = self.dataOut.heightList[0] | |||
| if maxHei == None: | |||
| maxHei = self.dataOut.heightList[-1] | |||
| if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei): | |||
| if self.warnings: | |||
| print('minHei: %.2f is out of the heights range' % (minHei)) | |||
| print('minHei is setting to %.2f' % (self.dataOut.heightList[0])) | |||
| minHei = self.dataOut.heightList[0] | |||
| if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei): | |||
| if self.warnings: | |||
| print('maxHei: %.2f is out of the heights range' % (maxHei)) | |||
| print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1])) | |||
| maxHei = self.dataOut.heightList[-1] | |||
| #indices relativos a los puntos de fft, puede ser de acuerdo a velocidad o frecuencia | |||
| minIndexFFT = 0 | |||
| maxIndexFFT = 0 | |||
| # validacion de velocidades | |||
| indminPoint = None | |||
| indmaxPoint = None | |||
| if self.dataOut.type == 'Spectra': | |||
| if minVel == None and maxVel == None : | |||
| freqrange = self.dataOut.getFreqRange(1) | |||
| if minFreq == None: | |||
| minFreq = freqrange[0] | |||
| if maxFreq == None: | |||
| maxFreq = freqrange[-1] | |||
| if (minFreq < freqrange[0]) or (minFreq > maxFreq): | |||
| if self.warnings: | |||
| print('minFreq: %.2f is out of the frequency range' % (minFreq)) | |||
| print('minFreq is setting to %.2f' % (freqrange[0])) | |||
| minFreq = freqrange[0] | |||
| if (maxFreq > freqrange[-1]) or (maxFreq < minFreq): | |||
| if self.warnings: | |||
| print('maxFreq: %.2f is out of the frequency range' % (maxFreq)) | |||
| print('maxFreq is setting to %.2f' % (freqrange[-1])) | |||
| maxFreq = freqrange[-1] | |||
| indminPoint = numpy.where(freqrange >= minFreq) | |||
| indmaxPoint = numpy.where(freqrange <= maxFreq) | |||
| else: | |||
| velrange = self.dataOut.getVelRange(1) | |||
| if minVel == None: | |||
| minVel = velrange[0] | |||
| if maxVel == None: | |||
| maxVel = velrange[-1] | |||
| if (minVel < velrange[0]) or (minVel > maxVel): | |||
| if self.warnings: | |||
| print('minVel: %.2f is out of the velocity range' % (minVel)) | |||
| print('minVel is setting to %.2f' % (velrange[0])) | |||
| minVel = velrange[0] | |||
| if (maxVel > velrange[-1]) or (maxVel < minVel): | |||
| if self.warnings: | |||
| print('maxVel: %.2f is out of the velocity range' % (maxVel)) | |||
| print('maxVel is setting to %.2f' % (velrange[-1])) | |||
| maxVel = velrange[-1] | |||
| indminPoint = numpy.where(velrange >= minVel) | |||
| indmaxPoint = numpy.where(velrange <= maxVel) | |||
| # seleccion de indices para rango REEMPLAZAR FOR FUNCION EXTERNA LUEGO | |||
| # minIndex = 0 | |||
| # maxIndex = 0 | |||
| # heights = self.dataOut.heightList | |||
| # inda = numpy.where(heights >= minHei) | |||
| # indb = numpy.where(heights <= maxHei) | |||
| # try: | |||
| # minIndex = inda[0][0] | |||
| # except: | |||
| # minIndex = 0 | |||
| # try: | |||
| # maxIndex = indb[0][-1] | |||
| # except: | |||
| # maxIndex = len(heights) | |||
| # if (minIndex < 0) or (minIndex > maxIndex): | |||
| # raise ValueError("some value in (%d,%d) is not valid" % ( | |||
| # minIndex, maxIndex)) | |||
| # if (maxIndex >= self.dataOut.nHeights): | |||
| # maxIndex = self.dataOut.nHeights - 1 | |||
| minIndex, maxIndex = getHei_index(minHei,maxHei,self.dataOut.heightList) | |||
| #############################################################3 | |||
| # seleccion de indices para velocidades | |||
| if self.dataOut.type == 'Spectra': | |||
| try: | |||
| minIndexFFT = indminPoint[0][0] | |||
| except: | |||
| minIndexFFT = 0 | |||
| try: | |||
| maxIndexFFT = indmaxPoint[0][-1] | |||
| except: | |||
| maxIndexFFT = len( self.dataOut.getFreqRange(1)) | |||
| self.minIndex, self.maxIndex, self.minIndexFFT, self.maxIndexFFT = minIndex, maxIndex, minIndexFFT, maxIndexFFT | |||
| self.isConfig = True | |||
| self.offset = 1 | |||
| if offset!=None: | |||
| self.offset = 10**(offset/10) | |||
| def run(self, dataOut, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False): | |||
| self.dataOut = dataOut | |||
| if not self.isConfig: | |||
| self.setup(offset, minHei, maxHei,minVel, maxVel, minFreq, maxFreq, warnings) | |||
| self.dataOut.noise_estimation = None | |||
| noise = None | |||
| if self.dataOut.type == 'Voltage': | |||
| noise = self.dataOut.getNoise(ymin_index=self.minIndex, ymax_index=self.maxIndex) | |||
| elif self.dataOut.type == 'Spectra': | |||
| noise = numpy.zeros( self.dataOut.nChannels) | |||
| norm = 1 | |||
| for channel in range( self.dataOut.nChannels): | |||
| if not hasattr(self.dataOut.nIncohInt,'__len__'): | |||
| norm = 1 | |||
| else: | |||
| norm = self.dataOut.max_nIncohInt[channel]/self.dataOut.nIncohInt[channel, self.minIndex:self.maxIndex] | |||
| daux = self.dataOut.data_spc[channel,self.minIndexFFT:self.maxIndexFFT, self.minIndex:self.maxIndex] | |||
| daux = numpy.multiply(daux, norm) | |||
| sortdata = numpy.sort(daux, axis=None) | |||
| noise[channel] = _noise.hildebrand_sekhon(sortdata, self.dataOut.max_nIncohInt[channel])/self.offset | |||
| else: | |||
| noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex) | |||
| self.dataOut.noise_estimation = noise.copy() # dataOut.noise | |||
| return self.dataOut | |||
| def getNoiseByMean(self,data): | |||
| #data debe estar ordenado | |||
| data = numpy.mean(data,axis=1) | |||
| sortdata = numpy.sort(data, axis=None) | |||
| pnoise = None | |||
| j = 0 | |||
| mean = numpy.mean(sortdata) | |||
| min = numpy.min(sortdata) | |||
| delta = mean - min | |||
| indexes = numpy.where(sortdata > (mean+delta))[0] #only array of indexes | |||
| #print(len(indexes)) | |||
| if len(indexes)==0: | |||
| pnoise = numpy.mean(sortdata) | |||
| else: | |||
| j = indexes[0] | |||
| pnoise = numpy.mean(sortdata[0:j]) | |||
| return pnoise | |||
| def getNoiseByHS(self,data, navg): | |||
| #data debe estar ordenado | |||
| #data = numpy.mean(data,axis=1) | |||
| sortdata = numpy.sort(data, axis=None) | |||
| lenOfData = len(sortdata) | |||
| nums_min = lenOfData*0.2 | |||
| if nums_min <= 5: | |||
| nums_min = 5 | |||
| sump = 0. | |||
| sumq = 0. | |||
| j = 0 | |||
| cont = 1 | |||
| while((cont == 1)and(j < lenOfData)): | |||
| sump += sortdata[j] | |||
| sumq += sortdata[j]**2 | |||
| #sumq -= sump**2 | |||
| if j > nums_min: | |||
| rtest = float(j)/(j-1) + 1.0/navg | |||
| #if ((sumq*j) > (sump**2)): | |||
| if ((sumq*j) > (rtest*sump**2)): | |||
| j = j - 1 | |||
| sump = sump - sortdata[j] | |||
| sumq = sumq - sortdata[j]**2 | |||
| cont = 0 | |||
| j += 1 | |||
|
Juan C. Espinoza
|
r1287 | ||
| r1739 | lnoise = sump / j | ||
| return lnoise | |||
|
Juan C. Espinoza
|
r1287 | class removeInterference(Operation): | |
|
José Chávez
|
r897 | ||
|
ebocanegra
|
r1123 | def removeInterference2(self): | |
|
Alexander Valdez
|
r1675 | ||
|
ebocanegra
|
r1123 | cspc = self.dataOut.data_cspc | |
| spc = self.dataOut.data_spc | |||
|
Alexander Valdez
|
r1675 | Heights = numpy.arange(cspc.shape[2]) | |
|
ebocanegra
|
r1123 | realCspc = numpy.abs(cspc) | |
|
Alexander Valdez
|
r1675 | ||
|
ebocanegra
|
r1123 | for i in range(cspc.shape[0]): | |
| LinePower= numpy.sum(realCspc[i], axis=0) | |||
| Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)] | |||
|
Alexander Valdez
|
r1685 | SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ] | |
| InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 ) | |||
|
ebocanegra
|
r1123 | InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)] | |
| InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)] | |||
|
Alexander Valdez
|
r1685 | ||
| InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) ) | |||
|
ebocanegra
|
r1123 | #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax])) | |
| if len(InterferenceRange)<int(cspc.shape[1]*0.3): | |||
| cspc[i,InterferenceRange,:] = numpy.NaN | |||
|
Alexander Valdez
|
r1685 | ||
|
ebocanegra
|
r1123 | self.dataOut.data_cspc = cspc | |
|
Alexander Valdez
|
r1668 | def removeInterference(self, interf=2, hei_interf=None, nhei_interf=None, offhei_interf=None): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | jspectra = self.dataOut.data_spc | |
| jcspectra = self.dataOut.data_cspc | |||
| jnoise = self.dataOut.getNoise() | |||
| num_incoh = self.dataOut.nIncohInt | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | num_channel = jspectra.shape[0] | |
| num_prof = jspectra.shape[1] | |||
| num_hei = jspectra.shape[2] | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # hei_interf | |
|
Miguel Valdez
|
r612 | if hei_interf is None: | |
|
Juan C. Espinoza
|
r1197 | count_hei = int(num_hei / 2) | |
|
George Yong
|
r1167 | hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei | |
|
Daniel Valdez
|
r487 | hei_interf = numpy.asarray(hei_interf)[0] | |
|
José Chávez
|
r1120 | # nhei_interf | |
|
Daniel Valdez
|
r487 | if (nhei_interf == None): | |
| nhei_interf = 5 | |||
| if (nhei_interf < 1): | |||
|
José Chávez
|
r897 | nhei_interf = 1 | |
|
Daniel Valdez
|
r487 | if (nhei_interf > count_hei): | |
| nhei_interf = count_hei | |||
|
José Chávez
|
r897 | if (offhei_interf == None): | |
|
Daniel Valdez
|
r487 | offhei_interf = 0 | |
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | ind_hei = list(range(num_hei)) | |
|
José Chávez
|
r897 | # mask_prof = numpy.asarray(range(num_prof - 2)) + 1 | |
|
Daniel Valdez
|
r487 | # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1 | |
|
George Yong
|
r1167 | mask_prof = numpy.asarray(list(range(num_prof))) | |
|
Daniel Valdez
|
r487 | num_mask_prof = mask_prof.size | |
|
José Chávez
|
r1120 | comp_mask_prof = [0, num_prof / 2] | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal | |
|
Daniel Valdez
|
r487 | if (jnoise.size < num_channel or numpy.isnan(jnoise).any()): | |
| jnoise = numpy.nan | |||
| noise_exist = jnoise[0] < numpy.Inf | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # Subrutina de Remocion de la Interferencia | |
|
Daniel Valdez
|
r487 | for ich in range(num_channel): | |
|
José Chávez
|
r1120 | # Se ordena los espectros segun su potencia (menor a mayor) | |
| power = jspectra[ich, mask_prof, :] | |||
| power = power[:, hei_interf] | |||
| power = power.sum(axis=0) | |||
|
Daniel Valdez
|
r487 | psort = power.ravel().argsort() | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # Se estima la interferencia promedio en los Espectros de Potencia empleando | |
|
George Yong
|
r1167 | junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range( | |
| offhei_interf, nhei_interf + offhei_interf))]]] | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if noise_exist: | |
|
José Chávez
|
r1120 | # tmp_noise = jnoise[ich] / num_prof | |
|
Daniel Valdez
|
r487 | tmp_noise = jnoise[ich] | |
| junkspc_interf = junkspc_interf - tmp_noise | |||
| #junkspc_interf[:,comp_mask_prof] = 0 | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf | |
|
Daniel Valdez
|
r487 | jspc_interf = jspc_interf.transpose() | |
|
José Chávez
|
r1120 | # Calculando el espectro de interferencia promedio | |
| noiseid = numpy.where( | |||
| jspc_interf <= tmp_noise / numpy.sqrt(num_incoh)) | |||
|
Daniel Valdez
|
r487 | noiseid = noiseid[0] | |
| cnoiseid = noiseid.size | |||
|
José Chávez
|
r1120 | interfid = numpy.where( | |
| jspc_interf > tmp_noise / numpy.sqrt(num_incoh)) | |||
|
Daniel Valdez
|
r487 | interfid = interfid[0] | |
| cinterfid = interfid.size | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | if (cnoiseid > 0): | |
| jspc_interf[noiseid] = 0 | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # Expandiendo los perfiles a limpiar | |
|
Daniel Valdez
|
r487 | if (cinterfid > 0): | |
|
José Chávez
|
r1120 | new_interfid = ( | |
| numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof | |||
|
José Chávez
|
r897 | new_interfid = numpy.asarray(new_interfid) | |
|
Daniel Valdez
|
r487 | new_interfid = {x for x in new_interfid} | |
| new_interfid = numpy.array(list(new_interfid)) | |||
| new_cinterfid = new_interfid.size | |||
|
José Chávez
|
r1120 | else: | |
| new_cinterfid = 0 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for ip in range(new_cinterfid): | |
|
José Chávez
|
r1120 | ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort() | |
| jspc_interf[new_interfid[ip] | |||
|
George Yong
|
r1194 | ] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]] | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | jspectra[ich, :, ind_hei] = jspectra[ich, :, | |
| ind_hei] - jspc_interf # Corregir indices | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # Removiendo la interferencia del punto de mayor interferencia | |
|
Daniel Valdez
|
r487 | ListAux = jspc_interf[mask_prof].tolist() | |
| maxid = ListAux.index(max(ListAux)) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if cinterfid > 0: | |
|
José Chávez
|
r1120 | for ip in range(cinterfid * (interf == 2) - 1): | |
| ind = (jspectra[ich, interfid[ip], :] < tmp_noise * | |||
| (1 + 1 / numpy.sqrt(num_incoh))).nonzero() | |||
|
Daniel Valdez
|
r487 | cind = len(ind) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if (cind > 0): | |
|
José Chávez
|
r1120 | jspectra[ich, interfid[ip], ind] = tmp_noise * \ | |
| (1 + (numpy.random.uniform(cind) - 0.5) / | |||
| numpy.sqrt(num_incoh)) | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | ind = numpy.array([-2, -1, 1, 2]) | |
| xx = numpy.zeros([4, 4]) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for id1 in range(4): | |
|
George Yong
|
r1167 | xx[:, id1] = ind[id1]**numpy.asarray(list(range(4))) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | xx_inv = numpy.linalg.inv(xx) | |
|
José Chávez
|
r1120 | xx = xx_inv[:, 0] | |
| ind = (ind + maxid + num_mask_prof) % num_mask_prof | |||
| yy = jspectra[ich, mask_prof[ind], :] | |||
| jspectra[ich, mask_prof[maxid], :] = numpy.dot( | |||
| yy.transpose(), xx) | |||
| indAux = (jspectra[ich, :, :] < tmp_noise * | |||
| (1 - 1 / numpy.sqrt(num_incoh))).nonzero() | |||
| jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \ | |||
| (1 - 1 / numpy.sqrt(num_incoh)) | |||
| # Remocion de Interferencia en el Cross Spectra | |||
| if jcspectra is None: | |||
| return jspectra, jcspectra | |||
|
Juan C. Espinoza
|
r1197 | num_pairs = int(jcspectra.size / (num_prof * num_hei)) | |
|
Daniel Valdez
|
r487 | jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for ip in range(num_pairs): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | #------------------------------------------- | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | cspower = numpy.abs(jcspectra[ip, mask_prof, :]) | |
| cspower = cspower[:, hei_interf] | |||
| cspower = cspower.sum(axis=0) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | cspsort = cspower.ravel().argsort() | |
|
George Yong
|
r1167 | junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range( | |
| offhei_interf, nhei_interf + offhei_interf))]]] | |||
|
Daniel Valdez
|
r487 | junkcspc_interf = junkcspc_interf.transpose() | |
|
José Chávez
|
r1120 | jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort() | |
|
José Chávez
|
r897 | ||
|
George Yong
|
r1194 | median_real = int(numpy.median(numpy.real( | |
| junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :]))) | |||
| median_imag = int(numpy.median(numpy.imag( | |||
| junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :]))) | |||
| comp_mask_prof = [int(e) for e in comp_mask_prof] | |||
| r1739 | junkcspc_interf[comp_mask_prof, :] = numpy.complex_( | ||
|
José Chávez
|
r1120 | median_real, median_imag) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for iprof in range(num_prof): | |
|
José Chávez
|
r1120 | ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort() | |
|
George Yong
|
r1194 | jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]] | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # Removiendo la Interferencia | |
| jcspectra[ip, :, ind_hei] = jcspectra[ip, | |||
| :, ind_hei] - jcspc_interf | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist() | |
| maxid = ListAux.index(max(ListAux)) | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | ind = numpy.array([-2, -1, 1, 2]) | |
| xx = numpy.zeros([4, 4]) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for id1 in range(4): | |
|
George Yong
|
r1167 | xx[:, id1] = ind[id1]**numpy.asarray(list(range(4))) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | xx_inv = numpy.linalg.inv(xx) | |
|
José Chávez
|
r1120 | xx = xx_inv[:, 0] | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | ind = (ind + maxid + num_mask_prof) % num_mask_prof | |
| yy = jcspectra[ip, mask_prof[ind], :] | |||
| jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx) | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # Guardar Resultados | |
|
Daniel Valdez
|
r487 | self.dataOut.data_spc = jspectra | |
| self.dataOut.data_cspc = jcspectra | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return 1 | |
| r1739 | |||
|
José Chávez
|
r897 | ||
|
Alexander Valdez
|
r1668 | def run(self, dataOut, interf=2,hei_interf=None, nhei_interf=None, offhei_interf=None, mode=1): | |
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | self.dataOut = dataOut | |
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | if mode == 1: | |
|
Alexander Valdez
|
r1668 | self.removeInterference(interf=2,hei_interf=None, nhei_interf=None, offhei_interf=None) | |
|
Juan C. Espinoza
|
r1287 | elif mode == 2: | |
| self.removeInterference2() | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | return self.dataOut | |
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1177 | ||
|
Alexander Valdez
|
r1668 | class deflip(Operation): | |
| def run(self, dataOut): | |||
| # arreglo 1: (num_chan, num_profiles, num_heights) | |||
| self.dataOut = dataOut | |||
| # JULIA-oblicua, indice 2 | |||
| # arreglo 2: (num_profiles, num_heights) | |||
| jspectra = self.dataOut.data_spc[2] | |||
| jspectra_tmp=numpy.zeros(jspectra.shape) | |||
| num_profiles=jspectra.shape[0] | |||
| freq_dc = int(num_profiles / 2) | |||
| # Flip con for | |||
| for j in range(num_profiles): | |||
| jspectra_tmp[num_profiles-j-1]= jspectra[j] | |||
| # Intercambio perfil de DC con perfil inmediato anterior | |||
| jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1] | |||
| jspectra_tmp[freq_dc]= jspectra[freq_dc] | |||
| # canal modificado es re-escrito en el arreglo de canales | |||
| self.dataOut.data_spc[2] = jspectra_tmp | |||
| return self.dataOut | |||
|
José Chávez
|
r1120 | class IncohInt(Operation): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | __profIndex = 0 | |
|
José Chávez
|
r1120 | __withOverapping = False | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | __byTime = False | |
| __initime = None | |||
| __lastdatatime = None | |||
| __integrationtime = None | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | __buffer_spc = None | |
| __buffer_cspc = None | |||
| __buffer_dc = None | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | __dataReady = False | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | __timeInterval = None | |
| r1739 | incohInt = 0 | ||
| nOutliers = 0 | |||
|
Daniel Valdez
|
r487 | n = None | |
| r1739 | |||
| _flagProfilesByRange = False | |||
| _nProfilesByRange = 0 | |||
|
George Yong
|
r1179 | def __init__(self): | |
|
George Yong
|
r1171 | ||
|
George Yong
|
r1179 | Operation.__init__(self) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def setup(self, n=None, timeInterval=None, overlapping=False): | |
| """ | |||
| Set the parameters of the integration class. | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | Inputs: | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | n : Number of coherent integrations | |
| timeInterval : Time of integration. If the parameter "n" is selected this one does not work | |||
|
José Chávez
|
r897 | overlapping : | |
|
Daniel Valdez
|
r487 | """ | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.__initime = None | |
| self.__lastdatatime = 0 | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | self.__buffer_spc = 0 | |
| self.__buffer_cspc = 0 | |||
| self.__buffer_dc = 0 | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | self.__profIndex = 0 | |
| self.__dataReady = False | |||
| self.__byTime = False | |||
| r1739 | self.incohInt = 0 | ||
| self.nOutliers = 0 | |||
|
Miguel Valdez
|
r623 | if n is None and timeInterval is None: | |
|
George Yong
|
r1167 | raise ValueError("n or timeInterval should be specified ...") | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | if n is not None: | |
| self.n = int(n) | |||
|
Daniel Valdez
|
r487 | else: | |
|
Alexander Valdez
|
r1675 | ||
|
José Chávez
|
r1120 | self.__integrationtime = int(timeInterval) | |
|
Miguel Valdez
|
r623 | self.n = None | |
|
Daniel Valdez
|
r487 | self.__byTime = True | |
|
José Chávez
|
r897 | ||
| r1739 | |||
|
Daniel Valdez
|
r487 | def putData(self, data_spc, data_cspc, data_dc): | |
| """ | |||
| Add a profile to the __buffer_spc and increase in one the __profileIndex | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | """ | |
| r1739 | if data_spc.all() == numpy.nan : | ||
| print("nan ") | |||
| return | |||
|
Miguel Valdez
|
r623 | self.__buffer_spc += data_spc | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | if data_cspc is None: | |
| self.__buffer_cspc = None | |||
| else: | |||
| self.__buffer_cspc += data_cspc | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | if data_dc is None: | |
| self.__buffer_dc = None | |||
| else: | |||
| self.__buffer_dc += data_dc | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | self.__profIndex += 1 | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def pushData(self): | |
| """ | |||
| Return the sum of the last profiles and the profiles used in the sum. | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | Affected: | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.__profileIndex | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | """ | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | data_spc = self.__buffer_spc | |
| data_cspc = self.__buffer_cspc | |||
| data_dc = self.__buffer_dc | |||
|
Daniel Valdez
|
r487 | n = self.__profIndex | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | self.__buffer_spc = 0 | |
| self.__buffer_cspc = 0 | |||
| self.__buffer_dc = 0 | |||
| r1739 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return data_spc, data_cspc, data_dc, n | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def byProfiles(self, *args): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.__dataReady = False | |
|
Miguel Valdez
|
r624 | avgdata_spc = None | |
| avgdata_cspc = None | |||
| avgdata_dc = None | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.putData(*args) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if self.__profIndex == self.n: | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData() | |
|
Miguel Valdez
|
r623 | self.n = n | |
|
Daniel Valdez
|
r487 | self.__dataReady = True | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return avgdata_spc, avgdata_cspc, avgdata_dc | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def byTime(self, datatime, *args): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.__dataReady = False | |
|
Miguel Valdez
|
r624 | avgdata_spc = None | |
| avgdata_cspc = None | |||
| avgdata_dc = None | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.putData(*args) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if (datatime - self.__initime) >= self.__integrationtime: | |
| avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData() | |||
| self.n = n | |||
| self.__dataReady = True | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return avgdata_spc, avgdata_cspc, avgdata_dc | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def integrate(self, datatime, *args): | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | if self.__profIndex == 0: | |
|
Daniel Valdez
|
r487 | self.__initime = datatime | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if self.__byTime: | |
|
José Chávez
|
r1120 | avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime( | |
| datatime, *args) | |||
|
Daniel Valdez
|
r487 | else: | |
| avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args) | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | if not self.__dataReady: | |
|
Daniel Valdez
|
r487 | return None, None, None, None | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def run(self, dataOut, n=None, timeInterval=None, overlapping=False): | |
|
José Chávez
|
r1120 | if n == 1: | |
|
Juan C. Espinoza
|
r1287 | return dataOut | |
|
José Chávez
|
r897 | ||
| r1739 | if dataOut.flagNoData == True: | ||
| return dataOut | |||
| if dataOut.flagProfilesByRange == True: | |||
| self._flagProfilesByRange = True | |||
| dataOut.flagNoData = True | |||
| dataOut.processingHeaderObj.timeIncohInt = timeInterval | |||
|
Daniel Valdez
|
r487 | if not self.isConfig: | |
| r1739 | self._nProfilesByRange = numpy.zeros((1,len(dataOut.heightList))) | ||
|
Daniel Valdez
|
r487 | self.setup(n, timeInterval, overlapping) | |
| self.isConfig = True | |||
|
José Chávez
|
r897 | ||
| r1739 | |||
|
Daniel Valdez
|
r487 | avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime, | |
| dataOut.data_spc, | |||
| dataOut.data_cspc, | |||
| dataOut.data_dc) | |||
|
José Chávez
|
r897 | ||
| r1739 | self.incohInt += dataOut.nIncohInt | ||
| if isinstance(dataOut.data_outlier,numpy.ndarray) or isinstance(dataOut.data_outlier,int) or isinstance(dataOut.data_outlier, float): | |||
| self.nOutliers += dataOut.data_outlier | |||
|
José Chávez
|
r897 | ||
| r1739 | if self._flagProfilesByRange: | ||
| dataOut.flagProfilesByRange = True | |||
| self._nProfilesByRange += dataOut.nProfilesByRange | |||
| if self.__dataReady: | |||
| #print("prof: ",dataOut.max_nIncohInt,self.__profIndex) | |||
|
Daniel Valdez
|
r487 | dataOut.data_spc = avgdata_spc | |
| dataOut.data_cspc = avgdata_cspc | |||
|
Alexander Valdez
|
r1675 | dataOut.data_dc = avgdata_dc | |
| r1739 | dataOut.nIncohInt = self.incohInt | ||
| dataOut.data_outlier = self.nOutliers | |||
|
Daniel Valdez
|
r487 | dataOut.utctime = avgdatatime | |
|
George Yong
|
r1171 | dataOut.flagNoData = False | |
| r1739 | self.incohInt = 0 | ||
| self.nOutliers = 0 | |||
| self.__profIndex = 0 | |||
| dataOut.nProfilesByRange = self._nProfilesByRange | |||
| self._nProfilesByRange = numpy.zeros((1,len(dataOut.heightList))) | |||
| self._flagProfilesByRange = False | |||
| # print("IncohInt Done") | |||
| r1344 | return dataOut | ||
| r1739 | |||
| class IntegrationFaradaySpectra(Operation): | |||
| __profIndex = 0 | |||
| __withOverapping = False | |||
| __byTime = False | |||
| __initime = None | |||
| __lastdatatime = None | |||
| __integrationtime = None | |||
| __buffer_spc = None | |||
| __buffer_cspc = None | |||
| __buffer_dc = None | |||
| __dataReady = False | |||
| __timeInterval = None | |||
| n_ints = None #matriz de numero de integracions (CH,HEI) | |||
| n = None | |||
| minHei_ind = None | |||
| maxHei_ind = None | |||
| navg = 1.0 | |||
| factor = 0.0 | |||
| dataoutliers = None # (CHANNELS, HEIGHTS) | |||
| _flagProfilesByRange = False | |||
| _nProfilesByRange = 0 | |||
| def __init__(self): | |||
| Operation.__init__(self) | |||
| def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None, minHei=None, maxHei=None, avg=1,factor=0.75): | |||
| """ | |||
| Set the parameters of the integration class. | |||
| Inputs: | |||
| n : Number of coherent integrations | |||
| timeInterval : Time of integration. If the parameter "n" is selected this one does not work | |||
| overlapping : | |||
| """ | |||
| self.__initime = None | |||
| self.__lastdatatime = 0 | |||
| self.__buffer_spc = [] | |||
| self.__buffer_cspc = [] | |||
| self.__buffer_dc = 0 | |||
| self.__profIndex = 0 | |||
| self.__dataReady = False | |||
| self.__byTime = False | |||
| self.factor = factor | |||
| self.navg = avg | |||
| #self.ByLags = dataOut.ByLags ###REDEFINIR | |||
| self.ByLags = False | |||
| self.maxProfilesInt = 0 | |||
| self.__nChannels = dataOut.nChannels | |||
| if DPL != None: | |||
| self.DPL=DPL | |||
| else: | |||
| #self.DPL=dataOut.DPL ###REDEFINIR | |||
| self.DPL=0 | |||
| if n is None and timeInterval is None: | |||
| raise ValueError("n or timeInterval should be specified ...") | |||
| if n is not None: | |||
| self.n = int(n) | |||
| else: | |||
| self.__integrationtime = int(timeInterval) | |||
| self.n = None | |||
| self.__byTime = True | |||
| if minHei == None: | |||
| minHei = self.dataOut.heightList[0] | |||
| if maxHei == None: | |||
| maxHei = self.dataOut.heightList[-1] | |||
| if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei): | |||
| print('minHei: %.2f is out of the heights range' % (minHei)) | |||
| print('minHei is setting to %.2f' % (self.dataOut.heightList[0])) | |||
| minHei = self.dataOut.heightList[0] | |||
| if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei): | |||
| print('maxHei: %.2f is out of the heights range' % (maxHei)) | |||
| print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1])) | |||
| maxHei = self.dataOut.heightList[-1] | |||
| ind_list1 = numpy.where(self.dataOut.heightList >= minHei) | |||
| ind_list2 = numpy.where(self.dataOut.heightList <= maxHei) | |||
| self.minHei_ind = ind_list1[0][0] | |||
| self.maxHei_ind = ind_list2[0][-1] | |||
| def putData(self, data_spc, data_cspc, data_dc): | |||
| """ | |||
| Add a profile to the __buffer_spc and increase in one the __profileIndex | |||
| """ | |||
| self.__buffer_spc.append(data_spc) | |||
| if self.__nChannels < 2: | |||
| self.__buffer_cspc = None | |||
| else: | |||
| self.__buffer_cspc.append(data_cspc) | |||
| if data_dc is None: | |||
| self.__buffer_dc = None | |||
| else: | |||
| self.__buffer_dc += data_dc | |||
| self.__profIndex += 1 | |||
| return | |||
| def hildebrand_sekhon_Integration(self,sortdata,navg, factor): | |||
| #data debe estar ordenado | |||
| #sortdata = numpy.sort(data, axis=None) | |||
| #sortID=data.argsort() | |||
| lenOfData = len(sortdata) | |||
| nums_min = lenOfData*factor | |||
| if nums_min <= 5: | |||
| nums_min = 5 | |||
| sump = 0. | |||
| sumq = 0. | |||
| j = 0 | |||
| cont = 1 | |||
| while((cont == 1)and(j < lenOfData)): | |||
| sump += sortdata[j] | |||
| sumq += sortdata[j]**2 | |||
| if j > nums_min: | |||
| rtest = float(j)/(j-1) + 1.0/navg | |||
| if ((sumq*j) > (rtest*sump**2)): | |||
| j = j - 1 | |||
| sump = sump - sortdata[j] | |||
| sumq = sumq - sortdata[j]**2 | |||
| cont = 0 | |||
| j += 1 | |||
| #lnoise = sump / j | |||
| #print("H S done") | |||
| #return j,sortID | |||
| return j | |||
| def pushData(self): | |||
| """ | |||
| Return the sum of the last profiles and the profiles used in the sum. | |||
| Affected: | |||
| self.__profileIndex | |||
| """ | |||
| bufferH=None | |||
| buffer=None | |||
| buffer1=None | |||
| buffer_cspc=None | |||
| #print("aes: ", self.__buffer_cspc) | |||
| self.__buffer_spc=numpy.array(self.__buffer_spc) | |||
| if self.__nChannels > 1 : | |||
| self.__buffer_cspc=numpy.array(self.__buffer_cspc) | |||
| #print("FREQ_DC",self.__buffer_spc.shape,self.__buffer_cspc.shape) | |||
| freq_dc = int(self.__buffer_spc.shape[2] / 2) | |||
| #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights) | |||
| self.dataOutliers = numpy.zeros((self.nChannels,self.nHeights)) # --> almacen de outliers | |||
| for k in range(self.minHei_ind,self.maxHei_ind): | |||
| if self.__nChannels > 1: | |||
| buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k]) | |||
| outliers_IDs_cspc=[] | |||
| cspc_outliers_exist=False | |||
| for i in range(self.nChannels):#dataOut.nChannels): | |||
| buffer1=numpy.copy(self.__buffer_spc[:,i,:,k]) | |||
| indexes=[] | |||
| #sortIDs=[] | |||
| outliers_IDs=[] | |||
| for j in range(self.nProfiles): #frecuencias en el tiempo | |||
| # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0 | |||
| # continue | |||
| # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1 | |||
| # continue | |||
| buffer=buffer1[:,j] | |||
| sortdata = numpy.sort(buffer, axis=None) | |||
| sortID=buffer.argsort() | |||
| index = _noise.hildebrand_sekhon2(sortdata,self.navg) | |||
| #index,sortID=self.hildebrand_sekhon_Integration(buffer,1,self.factor) | |||
| # fig,ax = plt.subplots() | |||
| # ax.set_title(str(k)+" "+str(j)) | |||
| # x=range(len(sortdata)) | |||
| # ax.scatter(x,sortdata) | |||
| # ax.axvline(index) | |||
| # plt.show() | |||
| indexes.append(index) | |||
| #sortIDs.append(sortID) | |||
| outliers_IDs=numpy.append(outliers_IDs,sortID[index:]) | |||
| #print("Outliers: ",outliers_IDs) | |||
| outliers_IDs=numpy.array(outliers_IDs) | |||
| outliers_IDs=outliers_IDs.ravel() | |||
| outliers_IDs=numpy.unique(outliers_IDs) | |||
| outliers_IDs=outliers_IDs.astype(numpy.dtype('int64')) | |||
| indexes=numpy.array(indexes) | |||
| indexmin=numpy.min(indexes) | |||
| #print(indexmin,buffer1.shape[0], k) | |||
| # fig,ax = plt.subplots() | |||
| # ax.plot(sortdata) | |||
| # ax2 = ax.twinx() | |||
| # x=range(len(indexes)) | |||
| # #plt.scatter(x,indexes) | |||
| # ax2.scatter(x,indexes) | |||
| # plt.show() | |||
| if indexmin != buffer1.shape[0]: | |||
| if self.__nChannels > 1: | |||
| cspc_outliers_exist= True | |||
| lt=outliers_IDs | |||
| #avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0) | |||
| for p in list(outliers_IDs): | |||
| #buffer1[p,:]=avg | |||
| buffer1[p,:] = numpy.NaN | |||
| self.dataOutliers[i,k] = len(outliers_IDs) | |||
| self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1) | |||
| if self.__nChannels > 1: | |||
| outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs) | |||
| if self.__nChannels > 1: | |||
| outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64')) | |||
| if cspc_outliers_exist: | |||
| lt=outliers_IDs_cspc | |||
| #avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0) | |||
| for p in list(outliers_IDs_cspc): | |||
| #buffer_cspc[p,:]=avg | |||
| buffer_cspc[p,:] = numpy.NaN | |||
| if self.__nChannels > 1: | |||
| self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc) | |||
| nOutliers = len(outliers_IDs) | |||
| #print("Outliers n: ",self.dataOutliers,nOutliers) | |||
| buffer=None | |||
| bufferH=None | |||
| buffer1=None | |||
| buffer_cspc=None | |||
| buffer=None | |||
| #data_spc = numpy.sum(self.__buffer_spc,axis=0) | |||
| data_spc = numpy.nansum(self.__buffer_spc,axis=0) | |||
| if self.__nChannels > 1: | |||
| #data_cspc = numpy.sum(self.__buffer_cspc,axis=0) | |||
| data_cspc = numpy.nansum(self.__buffer_cspc,axis=0) | |||
| else: | |||
| data_cspc = None | |||
| data_dc = self.__buffer_dc | |||
| #(CH, HEIGH) | |||
| self.maxProfilesInt = self.__profIndex - 1 | |||
| n = self.__profIndex - self.dataOutliers # n becomes a matrix | |||
| self.__buffer_spc = [] | |||
| self.__buffer_cspc = [] | |||
| self.__buffer_dc = 0 | |||
| self.__profIndex = 0 | |||
| #print("cleaned ",data_cspc) | |||
| return data_spc, data_cspc, data_dc, n | |||
| def byProfiles(self, *args): | |||
| self.__dataReady = False | |||
| avgdata_spc = None | |||
| avgdata_cspc = None | |||
| avgdata_dc = None | |||
| self.putData(*args) | |||
| if self.__profIndex >= self.n: | |||
| avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData() | |||
| self.n_ints = n | |||
| self.__dataReady = True | |||
| return avgdata_spc, avgdata_cspc, avgdata_dc | |||
| def byTime(self, datatime, *args): | |||
| self.__dataReady = False | |||
| avgdata_spc = None | |||
| avgdata_cspc = None | |||
| avgdata_dc = None | |||
| self.putData(*args) | |||
| if (datatime - self.__initime) >= self.__integrationtime: | |||
| avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData() | |||
| self.n_ints = n | |||
| self.__dataReady = True | |||
| return avgdata_spc, avgdata_cspc, avgdata_dc | |||
| def integrate(self, datatime, *args): | |||
| if self.__profIndex == 0: | |||
| self.__initime = datatime | |||
| if self.__byTime: | |||
| avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime( | |||
| datatime, *args) | |||
| else: | |||
| avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args) | |||
| if not self.__dataReady: | |||
| return None, None, None, None | |||
| #print("integrate", avgdata_cspc) | |||
| return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc | |||
| def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False, minHei=None, maxHei=None, avg=1, factor=0.75): | |||
| self.dataOut = dataOut | |||
| if n == 1: | |||
| return self.dataOut | |||
| self.dataOut.processingHeaderObj.timeIncohInt = timeInterval | |||
| if dataOut.flagProfilesByRange: | |||
| self._flagProfilesByRange = True | |||
| if self.dataOut.nChannels == 1: | |||
| self.dataOut.data_cspc = None #si es un solo canal no vale la pena acumular DATOS | |||
| #print("IN spc:", self.dataOut.data_spc.shape, self.dataOut.data_cspc) | |||
| if not self.isConfig: | |||
| self.setup(self.dataOut, n, timeInterval, overlapping,DPL ,minHei, maxHei, avg, factor) | |||
| self.isConfig = True | |||
| if not self.ByLags: | |||
| self.nProfiles=self.dataOut.nProfiles | |||
| self.nChannels=self.dataOut.nChannels | |||
| self.nHeights=self.dataOut.nHeights | |||
| avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime, | |||
| self.dataOut.data_spc, | |||
| self.dataOut.data_cspc, | |||
| self.dataOut.data_dc) | |||
| else: | |||
| self.nProfiles=self.dataOut.nProfiles | |||
| self.nChannels=self.dataOut.nChannels | |||
| self.nHeights=self.dataOut.nHeights | |||
| avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime, | |||
| self.dataOut.dataLag_spc, | |||
| self.dataOut.dataLag_cspc, | |||
| self.dataOut.dataLag_dc) | |||
| self.dataOut.flagNoData = True | |||
| if self._flagProfilesByRange: | |||
| dataOut.flagProfilesByRange = True | |||
| self._nProfilesByRange += dataOut.nProfilesByRange | |||
| if self.__dataReady: | |||
| if not self.ByLags: | |||
| if self.nChannels == 1: | |||
| #print("f int", avgdata_spc.shape) | |||
| self.dataOut.data_spc = avgdata_spc | |||
| self.dataOut.data_cspc = None | |||
| else: | |||
| self.dataOut.data_spc = numpy.squeeze(avgdata_spc) | |||
| self.dataOut.data_cspc = numpy.squeeze(avgdata_cspc) | |||
| self.dataOut.data_dc = avgdata_dc | |||
| self.dataOut.data_outlier = self.dataOutliers | |||
| else: | |||
| self.dataOut.dataLag_spc = avgdata_spc | |||
| self.dataOut.dataLag_cspc = avgdata_cspc | |||
| self.dataOut.dataLag_dc = avgdata_dc | |||
| self.dataOut.data_spc=self.dataOut.dataLag_spc[:,:,:,self.dataOut.LagPlot] | |||
| self.dataOut.data_cspc=self.dataOut.dataLag_cspc[:,:,:,self.dataOut.LagPlot] | |||
| self.dataOut.data_dc=self.dataOut.dataLag_dc[:,:,self.dataOut.LagPlot] | |||
| self.dataOut.nIncohInt *= self.n_ints | |||
| self.dataOut.utctime = avgdatatime | |||
| self.dataOut.flagNoData = False | |||
| dataOut.nProfilesByRange = self._nProfilesByRange | |||
| self._nProfilesByRange = numpy.zeros((1,len(dataOut.heightList))) | |||
| self._flagProfilesByRange = False | |||
| return self.dataOut | |||
| r1344 | class dopplerFlip(Operation): | ||
|
Alexander Valdez
|
r1675 | ||
|
Alexander Valdez
|
r1699 | def run(self, dataOut, chann = None): | |
| r1344 | # arreglo 1: (num_chan, num_profiles, num_heights) | ||
|
Alexander Valdez
|
r1675 | self.dataOut = dataOut | |
| r1344 | # JULIA-oblicua, indice 2 | ||
| # arreglo 2: (num_profiles, num_heights) | |||
|
Alexander Valdez
|
r1699 | jspectra = self.dataOut.data_spc[chann] | |
| r1344 | jspectra_tmp = numpy.zeros(jspectra.shape) | ||
| num_profiles = jspectra.shape[0] | |||
| freq_dc = int(num_profiles / 2) | |||
| # Flip con for | |||
| for j in range(num_profiles): | |||
| jspectra_tmp[num_profiles-j-1]= jspectra[j] | |||
| # Intercambio perfil de DC con perfil inmediato anterior | |||
| jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1] | |||
| jspectra_tmp[freq_dc]= jspectra[freq_dc] | |||
| # canal modificado es re-escrito en el arreglo de canales | |||
|
Alexander Valdez
|
r1699 | self.dataOut.data_spc[chann] = jspectra_tmp | |
| r1344 | |||
| return self.dataOut |
