jroproc_spectra.py
2289 lines
| 84.3 KiB
| text/x-python
|
PythonLexer
| r1338 | # Copyright (c) 2012-2020 Jicamarca Radio Observatory | ||
| # All rights reserved. | |||
| # | |||
| # Distributed under the terms of the BSD 3-clause license. | |||
| """Spectra processing Unit and operations | |||
| Here you will find the processing unit `SpectraProc` and several operations | |||
| to work with Spectra data type | |||
| """ | |||
Juan C. Espinoza
|
r1287 | import time | |
|
Juan C. Espinoza
|
r1062 | import itertools | |
Daniel Valdez
|
r487 | import numpy | |
| r1385 | import math | ||
|
Daniel Valdez
|
r487 | ||
George Yong
|
r1171 | from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation | |
Miguel Valdez
|
r568 | from schainpy.model.data.jrodata import Spectra | |
| from schainpy.model.data.jrodata import hildebrand_sekhon | |||
| r1475 | from schainpy.model.data import _noise | ||
|
George Yong
|
r1171 | from schainpy.utils import log | |
| r1528 | import matplotlib.pyplot as plt | ||
| r1540 | #from scipy.optimize import curve_fit | ||
| r1553 | from schainpy.model.io.utilsIO import getHei_index | ||
| r1570 | import datetime | ||
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | class SpectraProc(ProcessingUnit): | |
|
José Chávez
|
r897 | ||
|
George Yong
|
r1179 | def __init__(self): | |
|
George Yong
|
r1171 | ||
|
George Yong
|
r1179 | ProcessingUnit.__init__(self) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.buffer = None | |
| self.firstdatatime = None | |||
| self.profIndex = 0 | |||
| self.dataOut = Spectra() | |||
|
Daniel Valdez
|
r495 | self.id_min = None | |
| self.id_max = None | |||
|
George Yong
|
r1171 | self.setupReq = False #Agregar a todas las unidades de proc | |
| r1559 | self.nsamplesFFT = 0 | ||
|
Daniel Valdez
|
r487 | ||
|
Miguel Valdez
|
r623 | def __updateSpecFromVoltage(self): | |
|
José Chávez
|
r897 | ||
| r1506 | |||
|
Daniel Valdez
|
r487 | self.dataOut.timeZone = self.dataIn.timeZone | |
| self.dataOut.dstFlag = self.dataIn.dstFlag | |||
| self.dataOut.errorCount = self.dataIn.errorCount | |||
| self.dataOut.useLocalTime = self.dataIn.useLocalTime | |||
| r1559 | |||
| self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy() | |||
|
Daniel Valdez
|
r487 | self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy() | |
| r1559 | self.dataOut.ippSeconds = self.dataIn.ippSeconds | ||
| self.dataOut.ipp = self.dataIn.ipp | |||
|
Daniel Valdez
|
r487 | self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy() | |
| self.dataOut.channelList = self.dataIn.channelList | |||
| self.dataOut.heightList = self.dataIn.heightList | |||
|
José Chávez
|
r1120 | self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')]) | |
|
Daniel Valdez
|
r487 | self.dataOut.nProfiles = self.dataOut.nFFTPoints | |
|
Miguel Valdez
|
r568 | self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock | |
|
Daniel Valdez
|
r487 | self.dataOut.utctime = self.firstdatatime | |
|
José Chávez
|
r1120 | self.dataOut.flagDecodeData = self.dataIn.flagDecodeData | |
| self.dataOut.flagDeflipData = self.dataIn.flagDeflipData | |||
|
Miguel Valdez
|
r623 | self.dataOut.flagShiftFFT = False | |
|
Daniel Valdez
|
r487 | self.dataOut.nCohInt = self.dataIn.nCohInt | |
| self.dataOut.nIncohInt = 1 | |||
| r1553 | self.dataOut.deltaHeight = self.dataIn.deltaHeight | ||
|
Daniel Valdez
|
r487 | self.dataOut.windowOfFilter = self.dataIn.windowOfFilter | |
| self.dataOut.frequency = self.dataIn.frequency | |||
| self.dataOut.realtime = self.dataIn.realtime | |||
|
Daniel Valdez
|
r499 | self.dataOut.azimuth = self.dataIn.azimuth | |
| self.dataOut.zenith = self.dataIn.zenith | |||
| r1387 | self.dataOut.codeList = self.dataIn.codeList | ||
| self.dataOut.azimuthList = self.dataIn.azimuthList | |||
| self.dataOut.elevationList = self.dataIn.elevationList | |||
| r1560 | self.dataOut.code = self.dataIn.code | ||
| self.dataOut.nCode = self.dataIn.nCode | |||
| r1579 | self.dataOut.flagProfilesByRange = self.dataIn.flagProfilesByRange | ||
| self.dataOut.nProfilesByRange = self.dataIn.nProfilesByRange | |||
|
José Chávez
|
r897 | ||
| r1404 | |||
|
Daniel Valdez
|
r487 | def __getFft(self): | |
| r1547 | # print("fft donw") | ||
|
Daniel Valdez
|
r487 | """ | |
| Convierte valores de Voltaje a Spectra | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | Affected: | |
| self.dataOut.data_spc | |||
| self.dataOut.data_cspc | |||
| self.dataOut.data_dc | |||
| self.dataOut.heightList | |||
|
José Chávez
|
r897 | self.profIndex | |
|
Daniel Valdez
|
r487 | self.buffer | |
| self.dataOut.flagNoData | |||
| """ | |||
|
José Chávez
|
r1120 | fft_volt = numpy.fft.fft( | |
| self.buffer, n=self.dataOut.nFFTPoints, axis=1) | |||
|
Daniel Valdez
|
r487 | fft_volt = fft_volt.astype(numpy.dtype('complex')) | |
|
José Chávez
|
r1120 | dc = fft_volt[:, 0, :] | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # calculo de self-spectra | |
| fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,)) | |||
|
Daniel Valdez
|
r487 | spc = fft_volt * numpy.conjugate(fft_volt) | |
| spc = spc.real | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | blocksize = 0 | |
| blocksize += dc.size | |||
| blocksize += spc.size | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | cspc = None | |
| pairIndex = 0 | |||
| if self.dataOut.pairsList != None: | |||
|
José Chávez
|
r1120 | # calculo de cross-spectra | |
| cspc = numpy.zeros( | |||
| (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex') | |||
|
Daniel Valdez
|
r487 | for pair in self.dataOut.pairsList: | |
|
Miguel Valdez
|
r587 | if pair[0] not in self.dataOut.channelList: | |
|
George Yong
|
r1167 | raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % ( | |
| str(pair), str(self.dataOut.channelList))) | |||
|
Miguel Valdez
|
r587 | if pair[1] not in self.dataOut.channelList: | |
|
George Yong
|
r1167 | raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % ( | |
| str(pair), str(self.dataOut.channelList))) | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \ | |
| numpy.conjugate(fft_volt[pair[1], :, :]) | |||
|
Daniel Valdez
|
r487 | pairIndex += 1 | |
| blocksize += cspc.size | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.dataOut.data_spc = spc | |
| self.dataOut.data_cspc = cspc | |||
| self.dataOut.data_dc = dc | |||
| self.dataOut.blockSize = blocksize | |||
| r1270 | self.dataOut.flagShiftFFT = False | ||
|
José Chávez
|
r897 | ||
| r1547 | def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False, zeroPad=False): | ||
| r1559 | |||
| r1553 | |||
| r1540 | try: | ||
| type = self.dataIn.type.decode("utf-8") | |||
| self.dataIn.type = type | |||
| except: | |||
| pass | |||
|
Daniel Valdez
|
r487 | if self.dataIn.type == "Spectra": | |
| r1406 | |||
| try: | |||
| self.dataOut.copy(self.dataIn) | |||
| r1559 | self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy() | ||
| self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy() | |||
| r1553 | self.dataOut.nProfiles = self.dataOut.nFFTPoints | ||
| #self.dataOut.nHeights = len(self.dataOut.heightList) | |||
| r1406 | except Exception as e: | ||
| r1506 | print("Error dataIn ",e) | ||
| r1406 | |||
| r1559 | |||
| r1132 | if shift_fft: | ||
| #desplaza a la derecha en el eje 2 determinadas posiciones | |||
| shift = int(self.dataOut.nFFTPoints/2) | |||
| self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1) | |||
| if self.dataOut.data_cspc is not None: | |||
| #desplaza a la derecha en el eje 2 determinadas posiciones | |||
| r1151 | self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1) | ||
| r1341 | if pairsList: | ||
| self.__selectPairs(pairsList) | |||
|
George Yong
|
r1171 | ||
| r1404 | |||
| r1338 | elif self.dataIn.type == "Voltage": | ||
|
José Chávez
|
r897 | ||
|
George Yong
|
r1183 | self.dataOut.flagNoData = True | |
| r1559 | self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy() | ||
| self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy() | |||
|
Daniel Valdez
|
r487 | if nFFTPoints == None: | |
|
George Yong
|
r1167 | raise ValueError("This SpectraProc.run() need nFFTPoints input variable") | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r495 | if nProfiles == None: | |
|
Miguel Valdez
|
r568 | nProfiles = nFFTPoints | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if ippFactor == None: | |
| r1338 | self.dataOut.ippFactor = 1 | ||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.dataOut.nFFTPoints = nFFTPoints | |
| r1506 | #print(" volts ch,prof, h: ", self.dataIn.data.shape) | ||
|
Miguel Valdez
|
r611 | if self.buffer is None: | |
| r1547 | if not zeroPad: | ||
| self.buffer = numpy.zeros((self.dataIn.nChannels, | |||
|
José Chávez
|
r1120 | nProfiles, | |
| self.dataIn.nHeights), | |||
|
Miguel Valdez
|
r623 | dtype='complex') | |
| r1547 | else: | ||
| self.buffer = numpy.zeros((self.dataIn.nChannels, | |||
| nFFTPoints, | |||
| self.dataIn.nHeights), | |||
| dtype='complex') | |||
|
Daniel Valdez
|
r487 | ||
|
Miguel Valdez
|
r623 | if self.dataIn.flagDataAsBlock: | |
|
Miguel Valdez
|
r720 | nVoltProfiles = self.dataIn.data.shape[1] | |
|
José Chávez
|
r897 | ||
| r1547 | if nVoltProfiles == nProfiles or zeroPad: | ||
|
Daniel Valdez
|
r495 | self.buffer = self.dataIn.data.copy() | |
|
Miguel Valdez
|
r720 | self.profIndex = nVoltProfiles | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r720 | elif nVoltProfiles < nProfiles: | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | if self.profIndex == 0: | |
| self.id_min = 0 | |||
|
Miguel Valdez
|
r720 | self.id_max = nVoltProfiles | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | self.buffer[:, self.id_min:self.id_max, | |
| :] = self.dataIn.data | |||
|
Miguel Valdez
|
r720 | self.profIndex += nVoltProfiles | |
| self.id_min += nVoltProfiles | |||
| self.id_max += nVoltProfiles | |||
|
Daniel Valdez
|
r495 | else: | |
|
George Yong
|
r1167 | raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % ( | |
| self.dataIn.type, self.dataIn.data.shape[1], nProfiles)) | |||
|
Daniel Valdez
|
r495 | self.dataOut.flagNoData = True | |
|
José Chávez
|
r897 | else: | |
|
José Chávez
|
r1120 | self.buffer[:, self.profIndex, :] = self.dataIn.data.copy() | |
|
Miguel Valdez
|
r623 | self.profIndex += 1 | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if self.firstdatatime == None: | |
| self.firstdatatime = self.dataIn.utctime | |||
|
José Chávez
|
r897 | ||
| r1547 | if self.profIndex == nProfiles or zeroPad: | ||
| r1506 | |||
|
Miguel Valdez
|
r623 | self.__updateSpecFromVoltage() | |
| r1528 | |||
| r1338 | if pairsList == None: | ||
| self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)] | |||
| r1341 | else: | ||
| self.dataOut.pairsList = pairsList | |||
|
Daniel Valdez
|
r487 | self.__getFft() | |
| self.dataOut.flagNoData = False | |||
| self.firstdatatime = None | |||
| r1559 | self.nsamplesFFT = self.profIndex | ||
|
Daniel Valdez
|
r487 | self.profIndex = 0 | |
| r1472 | |||
| r1559 | #update Processing Header: | ||
| self.dataOut.processingHeaderObj.dtype = "Spectra" | |||
| self.dataOut.processingHeaderObj.nFFTPoints = self.dataOut.nFFTPoints | |||
| self.dataOut.processingHeaderObj.nSamplesFFT = self.nsamplesFFT | |||
| self.dataOut.processingHeaderObj.nIncohInt = 1 | |||
| r1540 | elif self.dataIn.type == "Parameters": | ||
| self.dataOut.data_spc = self.dataIn.data_spc | |||
| self.dataOut.data_cspc = self.dataIn.data_cspc | |||
| self.dataOut.data_outlier = self.dataIn.data_outlier | |||
| self.dataOut.nProfiles = self.dataIn.nProfiles | |||
| self.dataOut.nIncohInt = self.dataIn.nIncohInt | |||
| self.dataOut.nFFTPoints = self.dataIn.nFFTPoints | |||
| self.dataOut.ippFactor = self.dataIn.ippFactor | |||
| self.dataOut.max_nIncohInt = self.dataIn.max_nIncohInt | |||
| self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy() | |||
| r1559 | self.dataOut.ProcessingHeader = self.dataIn.ProcessingHeader.copy() | ||
| self.dataOut.ippSeconds = self.dataIn.ippSeconds | |||
| r1540 | self.dataOut.ipp = self.dataIn.ipp | ||
| #self.dataOut.abscissaList = self.dataIn.getVelRange(1) | |||
| #self.dataOut.spc_noise = self.dataIn.getNoise() | |||
| #self.dataOut.spc_range = (self.dataIn.getFreqRange(1) , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1)) | |||
| # self.dataOut.normFactor = self.dataIn.normFactor | |||
| if hasattr(self.dataIn, 'channelList'): | |||
| self.dataOut.channelList = self.dataIn.channelList | |||
| if hasattr(self.dataIn, 'pairsList'): | |||
| self.dataOut.pairsList = self.dataIn.pairsList | |||
| self.dataOut.groupList = self.dataIn.pairsList | |||
| self.dataOut.flagNoData = False | |||
| if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels | |||
| self.dataOut.ChanDist = self.dataIn.ChanDist | |||
| else: self.dataOut.ChanDist = None | |||
| #if hasattr(self.dataIn, 'VelRange'): #Velocities range | |||
| # self.dataOut.VelRange = self.dataIn.VelRange | |||
| #else: self.dataOut.VelRange = None | |||
| r1528 | |||
| r1338 | else: | ||
| r1540 | raise ValueError("The type of input object {} is not valid".format( | ||
| r1338 | self.dataIn.type)) | ||
| r1559 | |||
| r1553 | #print("spc proc Done", self.dataOut.data_spc.shape) | ||
| r1559 | #print(self.dataOut.data_spc) | ||
| return | |||
| r1506 | |||
|
Miguel Valdez
|
r730 | def __selectPairs(self, pairsList): | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | if not pairsList: | |
|
Miguel Valdez
|
r730 | return | |
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1062 | pairs = [] | |
| pairsIndex = [] | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1062 | for pair in pairsList: | |
| if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList: | |||
|
Miguel Valdez
|
r730 | continue | |
|
Juan C. Espinoza
|
r1062 | pairs.append(pair) | |
| pairsIndex.append(pairs.index(pair)) | |||
|
José Chávez
|
r1120 | ||
|
Juan C. Espinoza
|
r1062 | self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex] | |
| self.dataOut.pairsList = pairs | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r730 | return | |
|
José Chávez
|
r897 | ||
ebocanegra
|
r1123 | def selectFFTs(self, minFFT, maxFFT ): | |
| """ | |||
| r1279 | Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango | ||
|
ebocanegra
|
r1123 | minFFT<= FFT <= maxFFT | |
| """ | |||
| r1279 | |||
|
ebocanegra
|
r1123 | if (minFFT > maxFFT): | |
|
Juan C. Espinoza
|
r1188 | raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT)) | |
|
ebocanegra
|
r1123 | ||
| if (minFFT < self.dataOut.getFreqRange()[0]): | |||
| minFFT = self.dataOut.getFreqRange()[0] | |||
|
Daniel Valdez
|
r487 | ||
|
ebocanegra
|
r1123 | if (maxFFT > self.dataOut.getFreqRange()[-1]): | |
| maxFFT = self.dataOut.getFreqRange()[-1] | |||
| minIndex = 0 | |||
| maxIndex = 0 | |||
| FFTs = self.dataOut.getFreqRange() | |||
| inda = numpy.where(FFTs >= minFFT) | |||
| indb = numpy.where(FFTs <= maxFFT) | |||
|
Daniel Valdez
|
r487 | ||
|
ebocanegra
|
r1123 | try: | |
| minIndex = inda[0][0] | |||
| except: | |||
| minIndex = 0 | |||
| try: | |||
| maxIndex = indb[0][-1] | |||
| except: | |||
| maxIndex = len(FFTs) | |||
| self.selectFFTsByIndex(minIndex, maxIndex) | |||
| return 1 | |||
| r1279 | |||
|
José Chávez
|
r1120 | def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None): | |
| newheis = numpy.where( | |||
| self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex]) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if hei_ref != None: | |
|
José Chávez
|
r1120 | newheis = numpy.where(self.dataOut.heightList > hei_ref) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | minIndex = min(newheis[0]) | |
| maxIndex = max(newheis[0]) | |||
|
José Chávez
|
r1120 | data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1] | |
| heightList = self.dataOut.heightList[minIndex:maxIndex + 1] | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | # determina indices | |
|
José Chávez
|
r1120 | nheis = int(self.dataOut.radarControllerHeaderObj.txB / | |
| (self.dataOut.heightList[1] - self.dataOut.heightList[0])) | |||
| avg_dB = 10 * \ | |||
| numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0)) | |||
|
Daniel Valdez
|
r487 | beacon_dB = numpy.sort(avg_dB)[-nheis:] | |
| beacon_heiIndexList = [] | |||
| for val in avg_dB.tolist(): | |||
| if val >= beacon_dB[0]: | |||
| beacon_heiIndexList.append(avg_dB.tolist().index(val)) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | #data_spc = data_spc[:,:,beacon_heiIndexList] | |
| data_cspc = None | |||
|
Miguel Valdez
|
r612 | if self.dataOut.data_cspc is not None: | |
|
José Chávez
|
r1120 | data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1] | |
|
Daniel Valdez
|
r487 | #data_cspc = data_cspc[:,:,beacon_heiIndexList] | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | data_dc = None | |
|
Miguel Valdez
|
r612 | if self.dataOut.data_dc is not None: | |
|
José Chávez
|
r1120 | data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1] | |
|
Daniel Valdez
|
r487 | #data_dc = data_dc[:,beacon_heiIndexList] | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.dataOut.data_spc = data_spc | |
| self.dataOut.data_cspc = data_cspc | |||
| self.dataOut.data_dc = data_dc | |||
| self.dataOut.heightList = heightList | |||
| self.dataOut.beacon_heiIndexList = beacon_heiIndexList | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return 1 | |
|
José Chávez
|
r897 | ||
|
ebocanegra
|
r1123 | def selectFFTsByIndex(self, minIndex, maxIndex): | |
| """ | |||
| r1279 | |||
|
ebocanegra
|
r1123 | """ | |
| if (minIndex < 0) or (minIndex > maxIndex): | |||
|
Juan C. Espinoza
|
r1188 | raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex)) | |
|
ebocanegra
|
r1123 | ||
| if (maxIndex >= self.dataOut.nProfiles): | |||
| maxIndex = self.dataOut.nProfiles-1 | |||
| #Spectra | |||
| data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:] | |||
| data_cspc = None | |||
| if self.dataOut.data_cspc is not None: | |||
| data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:] | |||
| data_dc = None | |||
| if self.dataOut.data_dc is not None: | |||
| data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:] | |||
| self.dataOut.data_spc = data_spc | |||
| self.dataOut.data_cspc = data_cspc | |||
| self.dataOut.data_dc = data_dc | |||
| r1279 | |||
|
ebocanegra
|
r1123 | self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1]) | |
| self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1] | |||
| self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1] | |||
| return 1 | |||
|
Juan C. Espinoza
|
r1287 | def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None): | |
| # validacion de rango | |||
| if minHei == None: | |||
| minHei = self.dataOut.heightList[0] | |||
|
ebocanegra
|
r1123 | ||
|
Juan C. Espinoza
|
r1287 | if maxHei == None: | |
| maxHei = self.dataOut.heightList[-1] | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei): | |
| print('minHei: %.2f is out of the heights range' % (minHei)) | |||
| print('minHei is setting to %.2f' % (self.dataOut.heightList[0])) | |||
| minHei = self.dataOut.heightList[0] | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei): | |
| print('maxHei: %.2f is out of the heights range' % (maxHei)) | |||
| print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1])) | |||
| maxHei = self.dataOut.heightList[-1] | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | # validacion de velocidades | |
| velrange = self.dataOut.getVelRange(1) | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | if minVel == None: | |
| minVel = velrange[0] | |||
| if maxVel == None: | |||
| maxVel = velrange[-1] | |||
| if (minVel < velrange[0]) or (minVel > maxVel): | |||
| print('minVel: %.2f is out of the velocity range' % (minVel)) | |||
| print('minVel is setting to %.2f' % (velrange[0])) | |||
| minVel = velrange[0] | |||
| if (maxVel > velrange[-1]) or (maxVel < minVel): | |||
| print('maxVel: %.2f is out of the velocity range' % (maxVel)) | |||
| print('maxVel is setting to %.2f' % (velrange[-1])) | |||
| maxVel = velrange[-1] | |||
| # seleccion de indices para rango | |||
| minIndex = 0 | |||
| maxIndex = 0 | |||
| heights = self.dataOut.heightList | |||
| inda = numpy.where(heights >= minHei) | |||
| indb = numpy.where(heights <= maxHei) | |||
| try: | |||
| minIndex = inda[0][0] | |||
| except: | |||
| minIndex = 0 | |||
| try: | |||
| maxIndex = indb[0][-1] | |||
| except: | |||
| maxIndex = len(heights) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if (minIndex < 0) or (minIndex > maxIndex): | |
|
Juan C. Espinoza
|
r1287 | raise ValueError("some value in (%d,%d) is not valid" % ( | |
|
George Yong
|
r1167 | minIndex, maxIndex)) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if (maxIndex >= self.dataOut.nHeights): | |
|
José Chávez
|
r1120 | maxIndex = self.dataOut.nHeights - 1 | |
|
Daniel Valdez
|
r487 | ||
|
Juan C. Espinoza
|
r1287 | # seleccion de indices para velocidades | |
| indminvel = numpy.where(velrange >= minVel) | |||
| indmaxvel = numpy.where(velrange <= maxVel) | |||
| try: | |||
| minIndexVel = indminvel[0][0] | |||
| except: | |||
| minIndexVel = 0 | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | try: | |
| maxIndexVel = indmaxvel[0][-1] | |||
| except: | |||
| maxIndexVel = len(velrange) | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | # seleccion del espectro | |
| data_spc = self.dataOut.data_spc[:, | |||
| minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1] | |||
| # estimacion de ruido | |||
| noise = numpy.zeros(self.dataOut.nChannels) | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | for channel in range(self.dataOut.nChannels): | |
| daux = data_spc[channel, :, :] | |||
| sortdata = numpy.sort(daux, axis=None) | |||
| noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt) | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | self.dataOut.noise_estimation = noise.copy() | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return 1 | |
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | class removeDC(Operation): | |
| def run(self, dataOut, mode=2): | |||
| self.dataOut = dataOut | |||
|
Daniel Valdez
|
r487 | jspectra = self.dataOut.data_spc | |
| jcspectra = self.dataOut.data_cspc | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | num_chan = jspectra.shape[0] | |
| num_hei = jspectra.shape[2] | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r612 | if jcspectra is not None: | |
|
Daniel Valdez
|
r487 | jcspectraExist = True | |
| num_pairs = jcspectra.shape[0] | |||
|
José Chávez
|
r1120 | else: | |
| jcspectraExist = False | |||
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | freq_dc = int(jspectra.shape[1] / 2) | |
|
José Chávez
|
r1120 | ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc | |
|
George Yong
|
r1167 | ind_vel = ind_vel.astype(int) | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | if ind_vel[0] < 0: | |
|
George Yong
|
r1167 | ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof | |
|
José Chávez
|
r897 | ||
| if mode == 1: | |||
|
José Chávez
|
r1120 | jspectra[:, freq_dc, :] = ( | |
| jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if jcspectraExist: | |
|
José Chávez
|
r1120 | jcspectra[:, freq_dc, :] = ( | |
| jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if mode == 2: | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | vel = numpy.array([-2, -1, 1, 2]) | |
| xx = numpy.zeros([4, 4]) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for fil in range(4): | |
|
George Yong
|
r1167 | xx[fil, :] = vel[fil]**numpy.asarray(list(range(4))) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | xx_inv = numpy.linalg.inv(xx) | |
|
José Chávez
|
r1120 | xx_aux = xx_inv[0, :] | |
|
José Chávez
|
r897 | ||
| r1279 | for ich in range(num_chan): | ||
|
José Chávez
|
r1120 | yy = jspectra[ich, ind_vel, :] | |
| jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy) | |||
|
Daniel Valdez
|
r487 | ||
|
José Chávez
|
r1120 | junkid = jspectra[ich, freq_dc, :] <= 0 | |
|
Daniel Valdez
|
r487 | cjunkid = sum(junkid) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if cjunkid.any(): | |
|
José Chávez
|
r1120 | jspectra[ich, freq_dc, junkid.nonzero()] = ( | |
| jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if jcspectraExist: | |
| for ip in range(num_pairs): | |||
|
José Chávez
|
r1120 | yy = jcspectra[ip, ind_vel, :] | |
| jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.dataOut.data_spc = jspectra | |
| self.dataOut.data_cspc = jcspectra | |||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | return self.dataOut | |
| r1540 | class getNoiseB(Operation): | ||
| __slots__ =('offset','warnings', 'isConfig', 'minIndex','maxIndex','minIndexFFT','maxIndexFFT') | |||
| r1468 | def __init__(self): | ||
| Operation.__init__(self) | |||
| r1540 | self.isConfig = False | ||
| def setup(self, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False): | |||
| r1468 | |||
| r1506 | self.warnings = warnings | ||
| r1468 | if minHei == None: | ||
| minHei = self.dataOut.heightList[0] | |||
| if maxHei == None: | |||
| maxHei = self.dataOut.heightList[-1] | |||
| if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei): | |||
| r1506 | if self.warnings: | ||
| print('minHei: %.2f is out of the heights range' % (minHei)) | |||
| print('minHei is setting to %.2f' % (self.dataOut.heightList[0])) | |||
| r1468 | minHei = self.dataOut.heightList[0] | ||
| if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei): | |||
| r1506 | if self.warnings: | ||
| print('maxHei: %.2f is out of the heights range' % (maxHei)) | |||
| print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1])) | |||
| r1468 | maxHei = self.dataOut.heightList[-1] | ||
| #indices relativos a los puntos de fft, puede ser de acuerdo a velocidad o frecuencia | |||
| minIndexFFT = 0 | |||
| maxIndexFFT = 0 | |||
| # validacion de velocidades | |||
| indminPoint = None | |||
| indmaxPoint = None | |||
| r1528 | if self.dataOut.type == 'Spectra': | ||
| if minVel == None and maxVel == None : | |||
| r1468 | |||
| r1528 | freqrange = self.dataOut.getFreqRange(1) | ||
| r1468 | |||
| r1528 | if minFreq == None: | ||
| minFreq = freqrange[0] | |||
| r1468 | |||
| r1528 | if maxFreq == None: | ||
| maxFreq = freqrange[-1] | |||
| r1468 | |||
| r1528 | if (minFreq < freqrange[0]) or (minFreq > maxFreq): | ||
| if self.warnings: | |||
| print('minFreq: %.2f is out of the frequency range' % (minFreq)) | |||
| print('minFreq is setting to %.2f' % (freqrange[0])) | |||
| minFreq = freqrange[0] | |||
| r1468 | |||
| r1528 | if (maxFreq > freqrange[-1]) or (maxFreq < minFreq): | ||
| if self.warnings: | |||
| print('maxFreq: %.2f is out of the frequency range' % (maxFreq)) | |||
| print('maxFreq is setting to %.2f' % (freqrange[-1])) | |||
| maxFreq = freqrange[-1] | |||
| r1468 | |||
| r1528 | indminPoint = numpy.where(freqrange >= minFreq) | ||
| indmaxPoint = numpy.where(freqrange <= maxFreq) | |||
| r1468 | |||
| r1528 | else: | ||
| r1468 | |||
| r1528 | velrange = self.dataOut.getVelRange(1) | ||
| r1468 | |||
| r1528 | if minVel == None: | ||
| minVel = velrange[0] | |||
| r1468 | |||
| r1528 | if maxVel == None: | ||
| maxVel = velrange[-1] | |||
| r1468 | |||
| r1528 | if (minVel < velrange[0]) or (minVel > maxVel): | ||
| if self.warnings: | |||
| print('minVel: %.2f is out of the velocity range' % (minVel)) | |||
| print('minVel is setting to %.2f' % (velrange[0])) | |||
| minVel = velrange[0] | |||
| r1468 | |||
| r1528 | if (maxVel > velrange[-1]) or (maxVel < minVel): | ||
| if self.warnings: | |||
| print('maxVel: %.2f is out of the velocity range' % (maxVel)) | |||
| print('maxVel is setting to %.2f' % (velrange[-1])) | |||
| maxVel = velrange[-1] | |||
| r1468 | |||
| r1528 | indminPoint = numpy.where(velrange >= minVel) | ||
| indmaxPoint = numpy.where(velrange <= maxVel) | |||
| r1468 | |||
| # seleccion de indices para rango | |||
| minIndex = 0 | |||
| maxIndex = 0 | |||
| heights = self.dataOut.heightList | |||
| inda = numpy.where(heights >= minHei) | |||
| indb = numpy.where(heights <= maxHei) | |||
| try: | |||
| minIndex = inda[0][0] | |||
| except: | |||
| minIndex = 0 | |||
| try: | |||
| maxIndex = indb[0][-1] | |||
| except: | |||
| maxIndex = len(heights) | |||
| if (minIndex < 0) or (minIndex > maxIndex): | |||
| raise ValueError("some value in (%d,%d) is not valid" % ( | |||
| minIndex, maxIndex)) | |||
| if (maxIndex >= self.dataOut.nHeights): | |||
| maxIndex = self.dataOut.nHeights - 1 | |||
| #############################################################3 | |||
| # seleccion de indices para velocidades | |||
| r1528 | if self.dataOut.type == 'Spectra': | ||
| try: | |||
| minIndexFFT = indminPoint[0][0] | |||
| except: | |||
| minIndexFFT = 0 | |||
| r1468 | |||
| r1528 | try: | ||
| maxIndexFFT = indmaxPoint[0][-1] | |||
| except: | |||
| maxIndexFFT = len( self.dataOut.getFreqRange(1)) | |||
| r1468 | |||
| r1540 | self.minIndex, self.maxIndex, self.minIndexFFT, self.maxIndexFFT = minIndex, maxIndex, minIndexFFT, maxIndexFFT | ||
| self.isConfig = True | |||
| r1546 | self.offset = 1 | ||
| r1540 | if offset!=None: | ||
| self.offset = 10**(offset/10) | |||
| r1541 | #print("config getNoiseB Done") | ||
| r1540 | |||
| def run(self, dataOut, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False): | |||
| self.dataOut = dataOut | |||
| if not self.isConfig: | |||
| self.setup(offset, minHei, maxHei,minVel, maxVel, minFreq, maxFreq, warnings) | |||
| r1468 | |||
| r1472 | self.dataOut.noise_estimation = None | ||
| r1528 | noise = None | ||
| r1554 | #print("data type: ",self.dataOut.type, self.dataOut.nIncohInt, self.dataOut.max_nIncohInt) | ||
| r1528 | if self.dataOut.type == 'Voltage': | ||
| r1540 | noise = self.dataOut.getNoise(ymin_index=self.minIndex, ymax_index=self.maxIndex) | ||
| r1541 | #print(minIndex, maxIndex,minIndexVel, maxIndexVel) | ||
| r1528 | elif self.dataOut.type == 'Spectra': | ||
| r1554 | #print(self.dataOut.nChannels, self.minIndex, self.maxIndex,self.minIndexFFT, self.maxIndexFFT, self.dataOut.max_nIncohInt, self.dataOut.nIncohInt) | ||
| r1540 | noise = numpy.zeros( self.dataOut.nChannels) | ||
| r1541 | norm = 1 | ||
| r1547 | |||
| r1540 | for channel in range( self.dataOut.nChannels): | ||
| r1541 | if not hasattr(self.dataOut.nIncohInt,'__len__'): | ||
| norm = 1 | |||
| else: | |||
| r1554 | norm = self.dataOut.max_nIncohInt[channel]/self.dataOut.nIncohInt[channel, self.minIndex:self.maxIndex] | ||
| r1583 | |||
| r1554 | #print("norm nIncoh: ", norm ,self.dataOut.data_spc.shape, self.dataOut.max_nIncohInt) | ||
| r1540 | daux = self.dataOut.data_spc[channel,self.minIndexFFT:self.maxIndexFFT, self.minIndex:self.maxIndex] | ||
| daux = numpy.multiply(daux, norm) | |||
| #print("offset: ", self.offset, 10*numpy.log10(self.offset)) | |||
| r1547 | # noise[channel] = self.getNoiseByMean(daux)/self.offset | ||
| r1541 | #print(daux.shape, daux) | ||
| r1547 | #noise[channel] = self.getNoiseByHS(daux, self.dataOut.max_nIncohInt)/self.offset | ||
| sortdata = numpy.sort(daux, axis=None) | |||
| r1554 | |||
| noise[channel] = _noise.hildebrand_sekhon(sortdata, self.dataOut.max_nIncohInt[channel])/self.offset | |||
| r1566 | #print("noise shape", noise[channel], self.name) | ||
| r1541 | |||
| r1540 | #noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex) | ||
| r1528 | else: | ||
| r1540 | noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex) | ||
| r1583 | |||
| r1472 | self.dataOut.noise_estimation = noise.copy() # dataOut.noise | ||
| r1468 | #print("2: ",10*numpy.log10(self.dataOut.noise_estimation/64)) | ||
| r1548 | #print("2: ",self.dataOut.noise_estimation) | ||
| r1540 | #print(self.dataOut.flagNoData) | ||
| r1583 | #print("getNoise Done", 10*numpy.log10(noise)) | ||
| r1468 | return self.dataOut | ||
| r1540 | def getNoiseByMean(self,data): | ||
| #data debe estar ordenado | |||
| data = numpy.mean(data,axis=1) | |||
| sortdata = numpy.sort(data, axis=None) | |||
| #sortID=data.argsort() | |||
| #print(data.shape) | |||
| pnoise = None | |||
| j = 0 | |||
| mean = numpy.mean(sortdata) | |||
| min = numpy.min(sortdata) | |||
| delta = mean - min | |||
| indexes = numpy.where(sortdata > (mean+delta))[0] #only array of indexes | |||
| #print(len(indexes)) | |||
| if len(indexes)==0: | |||
| pnoise = numpy.mean(sortdata) | |||
| else: | |||
| j = indexes[0] | |||
| pnoise = numpy.mean(sortdata[0:j]) | |||
| # from matplotlib import pyplot as plt | |||
| # plt.plot(sortdata) | |||
| # plt.vlines(j,(pnoise-delta),(pnoise+delta), color='r') | |||
| # plt.show() | |||
| #print("noise: ", 10*numpy.log10(pnoise)) | |||
| return pnoise | |||
| def getNoiseByHS(self,data, navg): | |||
| #data debe estar ordenado | |||
| #data = numpy.mean(data,axis=1) | |||
| sortdata = numpy.sort(data, axis=None) | |||
| lenOfData = len(sortdata) | |||
| r1547 | nums_min = lenOfData*0.2 | ||
| r1540 | |||
| if nums_min <= 5: | |||
| nums_min = 5 | |||
| sump = 0. | |||
| sumq = 0. | |||
| j = 0 | |||
| cont = 1 | |||
| while((cont == 1)and(j < lenOfData)): | |||
| sump += sortdata[j] | |||
| sumq += sortdata[j]**2 | |||
| #sumq -= sump**2 | |||
| if j > nums_min: | |||
| r1547 | rtest = float(j)/(j-1) + 1.0/navg | ||
| r1540 | #if ((sumq*j) > (sump**2)): | ||
| if ((sumq*j) > (rtest*sump**2)): | |||
| j = j - 1 | |||
| sump = sump - sortdata[j] | |||
| sumq = sumq - sortdata[j]**2 | |||
| cont = 0 | |||
| j += 1 | |||
| lnoise = sump / j | |||
| return lnoise | |||
| r1468 | |||
| r1385 | |||
| def fit_func( x, a0, a1, a2): #, a3, a4, a5): | |||
| z = (x - a1) / a2 | |||
| y = a0 * numpy.exp(-z**2 / a2) #+ a3 + a4 * x + a5 * x**2 | |||
| return y | |||
| r1392 | |||
| r1554 | # class CleanRayleigh(Operation): | ||
| # | |||
| # def __init__(self): | |||
| # | |||
| # Operation.__init__(self) | |||
| # self.i=0 | |||
| # self.isConfig = False | |||
| # self.__dataReady = False | |||
| # self.__profIndex = 0 | |||
| # self.byTime = False | |||
| # self.byProfiles = False | |||
| # | |||
| # self.bloques = None | |||
| # self.bloque0 = None | |||
| # | |||
| # self.index = 0 | |||
| # | |||
| # self.buffer = 0 | |||
| # self.buffer2 = 0 | |||
| # self.buffer3 = 0 | |||
| # | |||
| # | |||
| # def setup(self,dataOut,min_hei,max_hei,n, timeInterval,factor_stdv): | |||
| # | |||
| # self.nChannels = dataOut.nChannels | |||
| # self.nProf = dataOut.nProfiles | |||
| # self.nPairs = dataOut.data_cspc.shape[0] | |||
| # self.pairsArray = numpy.array(dataOut.pairsList) | |||
| # self.spectra = dataOut.data_spc | |||
| # self.cspectra = dataOut.data_cspc | |||
| # self.heights = dataOut.heightList #alturas totales | |||
| # self.nHeights = len(self.heights) | |||
| # self.min_hei = min_hei | |||
| # self.max_hei = max_hei | |||
| # if (self.min_hei == None): | |||
| # self.min_hei = 0 | |||
| # if (self.max_hei == None): | |||
| # self.max_hei = dataOut.heightList[-1] | |||
| # self.hval = ((self.max_hei>=self.heights) & (self.heights >= self.min_hei)).nonzero() | |||
| # self.heightsClean = self.heights[self.hval] #alturas filtradas | |||
| # self.hval = self.hval[0] # forma (N,), an solo N elementos -> Indices de alturas | |||
| # self.nHeightsClean = len(self.heightsClean) | |||
| # self.channels = dataOut.channelList | |||
| # self.nChan = len(self.channels) | |||
| # self.nIncohInt = dataOut.nIncohInt | |||
| # self.__initime = dataOut.utctime | |||
| # self.maxAltInd = self.hval[-1]+1 | |||
| # self.minAltInd = self.hval[0] | |||
| # | |||
| # self.crosspairs = dataOut.pairsList | |||
| # self.nPairs = len(self.crosspairs) | |||
| # self.normFactor = dataOut.normFactor | |||
| # self.nFFTPoints = dataOut.nFFTPoints | |||
| # self.ippSeconds = dataOut.ippSeconds | |||
| # self.currentTime = self.__initime | |||
| # self.pairsArray = numpy.array(dataOut.pairsList) | |||
| # self.factor_stdv = factor_stdv | |||
| # | |||
| # if n != None : | |||
| # self.byProfiles = True | |||
| # self.nIntProfiles = n | |||
| # else: | |||
| # self.__integrationtime = timeInterval | |||
| # | |||
| # self.__dataReady = False | |||
| # self.isConfig = True | |||
| # | |||
| # | |||
| # | |||
| # def run(self, dataOut,min_hei=None,max_hei=None, n=None, timeInterval=10,factor_stdv=2.5): | |||
| # #print("runing cleanRayleigh") | |||
| # if not self.isConfig : | |||
| # | |||
| # self.setup(dataOut, min_hei,max_hei,n,timeInterval,factor_stdv) | |||
| # | |||
| # tini=dataOut.utctime | |||
| # | |||
| # if self.byProfiles: | |||
| # if self.__profIndex == self.nIntProfiles: | |||
| # self.__dataReady = True | |||
| # else: | |||
| # if (tini - self.__initime) >= self.__integrationtime: | |||
| # | |||
| # self.__dataReady = True | |||
| # self.__initime = tini | |||
| # | |||
| # #if (tini.tm_min % 2) == 0 and (tini.tm_sec < 5 and self.fint==0): | |||
| # | |||
| # if self.__dataReady: | |||
| # | |||
| # self.__profIndex = 0 | |||
| # jspc = self.buffer | |||
| # jcspc = self.buffer2 | |||
| # #jnoise = self.buffer3 | |||
| # self.buffer = dataOut.data_spc | |||
| # self.buffer2 = dataOut.data_cspc | |||
| # #self.buffer3 = dataOut.noise | |||
| # self.currentTime = dataOut.utctime | |||
| # if numpy.any(jspc) : | |||
| # #print( jspc.shape, jcspc.shape) | |||
| # jspc = numpy.reshape(jspc,(int(len(jspc)/self.nChannels),self.nChannels,self.nFFTPoints,self.nHeights)) | |||
| # try: | |||
| # jcspc= numpy.reshape(jcspc,(int(len(jcspc)/self.nPairs),self.nPairs,self.nFFTPoints,self.nHeights)) | |||
| # except: | |||
| # print("no cspc") | |||
| # self.__dataReady = False | |||
| # #print( jspc.shape, jcspc.shape) | |||
| # dataOut.flagNoData = False | |||
| # else: | |||
| # dataOut.flagNoData = True | |||
| # self.__dataReady = False | |||
| # return dataOut | |||
| # else: | |||
| # #print( len(self.buffer)) | |||
| # if numpy.any(self.buffer): | |||
| # self.buffer = numpy.concatenate((self.buffer,dataOut.data_spc), axis=0) | |||
| # try: | |||
| # self.buffer2 = numpy.concatenate((self.buffer2,dataOut.data_cspc), axis=0) | |||
| # self.buffer3 += dataOut.data_dc | |||
| # except: | |||
| # pass | |||
| # else: | |||
| # self.buffer = dataOut.data_spc | |||
| # self.buffer2 = dataOut.data_cspc | |||
| # self.buffer3 = dataOut.data_dc | |||
| # #print self.index, self.fint | |||
| # #print self.buffer2.shape | |||
| # dataOut.flagNoData = True ## NOTE: ?? revisar LUEGO | |||
| # self.__profIndex += 1 | |||
| # return dataOut ## NOTE: REV | |||
| # | |||
| # | |||
| # #index = tini.tm_hour*12+tini.tm_min/5 | |||
| # ''' | |||
| # #REVISAR | |||
| # ''' | |||
| # # jspc = jspc/self.nFFTPoints/self.normFactor | |||
| # # jcspc = jcspc/self.nFFTPoints/self.normFactor | |||
| # | |||
| # | |||
| # | |||
| # tmp_spectra,tmp_cspectra = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv) | |||
| # dataOut.data_spc = tmp_spectra | |||
| # dataOut.data_cspc = tmp_cspectra | |||
| # | |||
| # #dataOut.data_spc,dataOut.data_cspc = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv) | |||
| # | |||
| # dataOut.data_dc = self.buffer3 | |||
| # dataOut.nIncohInt *= self.nIntProfiles | |||
| # dataOut.max_nIncohInt = self.nIntProfiles | |||
| # dataOut.utctime = self.currentTime #tiempo promediado | |||
| # #print("Time: ",time.localtime(dataOut.utctime)) | |||
| # # dataOut.data_spc = sat_spectra | |||
| # # dataOut.data_cspc = sat_cspectra | |||
| # self.buffer = 0 | |||
| # self.buffer2 = 0 | |||
| # self.buffer3 = 0 | |||
| # | |||
| # return dataOut | |||
| # | |||
| # def cleanRayleigh(self,dataOut,spectra,cspectra,factor_stdv): | |||
| # print("OP cleanRayleigh") | |||
| # #import matplotlib.pyplot as plt | |||
| # #for k in range(149): | |||
| # #channelsProcssd = [] | |||
| # #channelA_ok = False | |||
| # #rfunc = cspectra.copy() #self.bloques | |||
| # rfunc = spectra.copy() | |||
| # #rfunc = cspectra | |||
| # #val_spc = spectra*0.0 #self.bloque0*0.0 | |||
| # #val_cspc = cspectra*0.0 #self.bloques*0.0 | |||
| # #in_sat_spectra = spectra.copy() #self.bloque0 | |||
| # #in_sat_cspectra = cspectra.copy() #self.bloques | |||
| # | |||
| # | |||
| # ###ONLY FOR TEST: | |||
| # raxs = math.ceil(math.sqrt(self.nPairs)) | |||
| # if raxs == 0: | |||
| # raxs = 1 | |||
| # caxs = math.ceil(self.nPairs/raxs) | |||
| # if self.nPairs <4: | |||
| # raxs = 2 | |||
| # caxs = 2 | |||
| # #print(raxs, caxs) | |||
| # fft_rev = 14 #nFFT to plot | |||
| # hei_rev = ((self.heights >= 550) & (self.heights <= 551)).nonzero() #hei to plot | |||
| # hei_rev = hei_rev[0] | |||
| # #print(hei_rev) | |||
| # | |||
| # #print numpy.absolute(rfunc[:,0,0,14]) | |||
| # | |||
| # gauss_fit, covariance = None, None | |||
| # for ih in range(self.minAltInd,self.maxAltInd): | |||
| # for ifreq in range(self.nFFTPoints): | |||
| # ''' | |||
| # ###ONLY FOR TEST: | |||
| # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY | |||
| # fig, axs = plt.subplots(raxs, caxs) | |||
| # fig2, axs2 = plt.subplots(raxs, caxs) | |||
| # col_ax = 0 | |||
| # row_ax = 0 | |||
| # ''' | |||
| # #print(self.nPairs) | |||
| # for ii in range(self.nChan): #PARES DE CANALES SELF y CROSS | |||
| # # if self.crosspairs[ii][1]-self.crosspairs[ii][0] > 1: # APLICAR SOLO EN PARES CONTIGUOS | |||
| # # continue | |||
| # # if not self.crosspairs[ii][0] in channelsProcssd: | |||
| # # channelA_ok = True | |||
| # #print("pair: ",self.crosspairs[ii]) | |||
| # ''' | |||
| # ###ONLY FOR TEST: | |||
| # if (col_ax%caxs==0 and col_ax!=0 and self.nPairs !=1): | |||
| # col_ax = 0 | |||
| # row_ax += 1 | |||
| # ''' | |||
| # func2clean = 10*numpy.log10(numpy.absolute(rfunc[:,ii,ifreq,ih])) #Potencia? | |||
| # #print(func2clean.shape) | |||
| # val = (numpy.isfinite(func2clean)==True).nonzero() | |||
| # | |||
| # if len(val)>0: #limitador | |||
| # min_val = numpy.around(numpy.amin(func2clean)-2) #> (-40) | |||
| # if min_val <= -40 : | |||
| # min_val = -40 | |||
| # max_val = numpy.around(numpy.amax(func2clean)+2) #< 200 | |||
| # if max_val >= 200 : | |||
| # max_val = 200 | |||
| # #print min_val, max_val | |||
| # step = 1 | |||
| # #print("Getting bins and the histogram") | |||
| # x_dist = min_val + numpy.arange(1 + ((max_val-(min_val))/step))*step | |||
| # y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step)) | |||
| # #print(len(y_dist),len(binstep[:-1])) | |||
| # #print(row_ax,col_ax, " ..") | |||
| # #print(self.pairsArray[ii][0],self.pairsArray[ii][1]) | |||
| # mean = numpy.sum(x_dist * y_dist) / numpy.sum(y_dist) | |||
| # sigma = numpy.sqrt(numpy.sum(y_dist * (x_dist - mean)**2) / numpy.sum(y_dist)) | |||
| # parg = [numpy.amax(y_dist),mean,sigma] | |||
| # | |||
| # newY = None | |||
| # | |||
| # try : | |||
| # gauss_fit, covariance = curve_fit(fit_func, x_dist, y_dist,p0=parg) | |||
| # mode = gauss_fit[1] | |||
| # stdv = gauss_fit[2] | |||
| # #print(" FIT OK",gauss_fit) | |||
| # ''' | |||
| # ###ONLY FOR TEST: | |||
| # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY | |||
| # newY = fit_func(x_dist,gauss_fit[0],gauss_fit[1],gauss_fit[2]) | |||
| # axs[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green') | |||
| # axs[row_ax,col_ax].plot(binstep[:-1],newY,color='red') | |||
| # axs[row_ax,col_ax].set_title("CH "+str(self.channels[ii])) | |||
| # ''' | |||
| # except: | |||
| # mode = mean | |||
| # stdv = sigma | |||
| # #print("FIT FAIL") | |||
| # #continue | |||
| # | |||
| # | |||
| # #print(mode,stdv) | |||
| # #Removing echoes greater than mode + std_factor*stdv | |||
| # noval = (abs(func2clean - mode)>=(factor_stdv*stdv)).nonzero() | |||
| # #noval tiene los indices que se van a remover | |||
| # #print("Chan ",ii," novals: ",len(noval[0])) | |||
| # if len(noval[0]) > 0: #forma de array (N,) es igual a longitud (N) | |||
| # novall = ((func2clean - mode) >= (factor_stdv*stdv)).nonzero() | |||
| # #print(novall) | |||
| # #print(" ",self.pairsArray[ii]) | |||
| # #cross_pairs = self.pairsArray[ii] | |||
| # #Getting coherent echoes which are removed. | |||
| # # if len(novall[0]) > 0: | |||
| # # | |||
| # # val_spc[novall[0],cross_pairs[0],ifreq,ih] = 1 | |||
| # # val_spc[novall[0],cross_pairs[1],ifreq,ih] = 1 | |||
| # # val_cspc[novall[0],ii,ifreq,ih] = 1 | |||
| # #print("OUT NOVALL 1") | |||
| # try: | |||
| # pair = (self.channels[ii],self.channels[ii + 1]) | |||
| # except: | |||
| # pair = (99,99) | |||
| # #print("par ", pair) | |||
| # if ( pair in self.crosspairs): | |||
| # q = self.crosspairs.index(pair) | |||
| # #print("está aqui: ", q, (ii,ii + 1)) | |||
| # new_a = numpy.delete(cspectra[:,q,ifreq,ih], noval[0]) | |||
| # cspectra[noval,q,ifreq,ih] = numpy.mean(new_a) #mean CrossSpectra | |||
| # | |||
| # #if channelA_ok: | |||
| # #chA = self.channels.index(cross_pairs[0]) | |||
| # new_b = numpy.delete(spectra[:,ii,ifreq,ih], noval[0]) | |||
| # spectra[noval,ii,ifreq,ih] = numpy.mean(new_b) #mean Spectra Pair A | |||
| # #channelA_ok = False | |||
| # | |||
| # # chB = self.channels.index(cross_pairs[1]) | |||
| # # new_c = numpy.delete(spectra[:,chB,ifreq,ih], noval[0]) | |||
| # # spectra[noval,chB,ifreq,ih] = numpy.mean(new_c) #mean Spectra Pair B | |||
| # # | |||
| # # channelsProcssd.append(self.crosspairs[ii][0]) # save channel A | |||
| # # channelsProcssd.append(self.crosspairs[ii][1]) # save channel B | |||
| # ''' | |||
| # ###ONLY FOR TEST: | |||
| # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY | |||
| # func2clean = 10*numpy.log10(numpy.absolute(spectra[:,ii,ifreq,ih])) | |||
| # y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step)) | |||
| # axs2[row_ax,col_ax].plot(binstep[:-1],newY,color='red') | |||
| # axs2[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green') | |||
| # axs2[row_ax,col_ax].set_title("CH "+str(self.channels[ii])) | |||
| # ''' | |||
| # ''' | |||
| # ###ONLY FOR TEST: | |||
| # col_ax += 1 #contador de ploteo columnas | |||
| # ##print(col_ax) | |||
| # ###ONLY FOR TEST: | |||
| # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY | |||
| # title = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km" | |||
| # title2 = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km CLEANED" | |||
| # fig.suptitle(title) | |||
| # fig2.suptitle(title2) | |||
| # plt.show() | |||
| # ''' | |||
| # ################################################################################################## | |||
| # | |||
| # #print("Getting average of the spectra and cross-spectra from incoherent echoes.") | |||
| # out_spectra = numpy.zeros([self.nChan,self.nFFTPoints,self.nHeights], dtype=float) #+numpy.nan | |||
| # out_cspectra = numpy.zeros([self.nPairs,self.nFFTPoints,self.nHeights], dtype=complex) #+numpy.nan | |||
| # for ih in range(self.nHeights): | |||
| # for ifreq in range(self.nFFTPoints): | |||
| # for ich in range(self.nChan): | |||
| # tmp = spectra[:,ich,ifreq,ih] | |||
| # valid = (numpy.isfinite(tmp[:])==True).nonzero() | |||
| # | |||
| # if len(valid[0]) >0 : | |||
| # out_spectra[ich,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0]) | |||
| # | |||
| # for icr in range(self.nPairs): | |||
| # tmp = numpy.squeeze(cspectra[:,icr,ifreq,ih]) | |||
| # valid = (numpy.isfinite(tmp)==True).nonzero() | |||
| # if len(valid[0]) > 0: | |||
| # out_cspectra[icr,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0]) | |||
| # | |||
| # return out_spectra, out_cspectra | |||
| # | |||
| # def REM_ISOLATED_POINTS(self,array,rth): | |||
| # # import matplotlib.pyplot as plt | |||
| # if rth == None : | |||
| # rth = 4 | |||
| # #print("REM ISO") | |||
| # num_prof = len(array[0,:,0]) | |||
| # num_hei = len(array[0,0,:]) | |||
| # n2d = len(array[:,0,0]) | |||
| # | |||
| # for ii in range(n2d) : | |||
| # #print ii,n2d | |||
| # tmp = array[ii,:,:] | |||
| # #print tmp.shape, array[ii,101,:],array[ii,102,:] | |||
| # | |||
| # # fig = plt.figure(figsize=(6,5)) | |||
| # # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8 | |||
| # # ax = fig.add_axes([left, bottom, width, height]) | |||
| # # x = range(num_prof) | |||
| # # y = range(num_hei) | |||
| # # cp = ax.contour(y,x,tmp) | |||
| # # ax.clabel(cp, inline=True,fontsize=10) | |||
| # # plt.show() | |||
| # | |||
| # #indxs = WHERE(FINITE(tmp) AND tmp GT 0,cindxs) | |||
| # tmp = numpy.reshape(tmp,num_prof*num_hei) | |||
| # indxs1 = (numpy.isfinite(tmp)==True).nonzero() | |||
| # indxs2 = (tmp > 0).nonzero() | |||
| # | |||
| # indxs1 = (indxs1[0]) | |||
| # indxs2 = indxs2[0] | |||
| # #indxs1 = numpy.array(indxs1[0]) | |||
| # #indxs2 = numpy.array(indxs2[0]) | |||
| # indxs = None | |||
| # #print indxs1 , indxs2 | |||
| # for iv in range(len(indxs2)): | |||
| # indv = numpy.array((indxs1 == indxs2[iv]).nonzero()) | |||
| # #print len(indxs2), indv | |||
| # if len(indv[0]) > 0 : | |||
| # indxs = numpy.concatenate((indxs,indxs2[iv]), axis=None) | |||
| # # print indxs | |||
| # indxs = indxs[1:] | |||
| # #print(indxs, len(indxs)) | |||
| # if len(indxs) < 4 : | |||
| # array[ii,:,:] = 0. | |||
| # return | |||
| # | |||
| # xpos = numpy.mod(indxs ,num_hei) | |||
| # ypos = (indxs / num_hei) | |||
| # sx = numpy.argsort(xpos) # Ordering respect to "x" (time) | |||
| # #print sx | |||
| # xpos = xpos[sx] | |||
| # ypos = ypos[sx] | |||
| # | |||
| # # *********************************** Cleaning isolated points ********************************** | |||
| # ic = 0 | |||
| # while True : | |||
| # r = numpy.sqrt(list(numpy.power((xpos[ic]-xpos),2)+ numpy.power((ypos[ic]-ypos),2))) | |||
| # #no_coh = WHERE(FINITE(r) AND (r LE rth),cno_coh) | |||
| # #plt.plot(r) | |||
| # #plt.show() | |||
| # no_coh1 = (numpy.isfinite(r)==True).nonzero() | |||
| # no_coh2 = (r <= rth).nonzero() | |||
| # #print r, no_coh1, no_coh2 | |||
| # no_coh1 = numpy.array(no_coh1[0]) | |||
| # no_coh2 = numpy.array(no_coh2[0]) | |||
| # no_coh = None | |||
| # #print valid1 , valid2 | |||
| # for iv in range(len(no_coh2)): | |||
| # indv = numpy.array((no_coh1 == no_coh2[iv]).nonzero()) | |||
| # if len(indv[0]) > 0 : | |||
| # no_coh = numpy.concatenate((no_coh,no_coh2[iv]), axis=None) | |||
| # no_coh = no_coh[1:] | |||
| # #print len(no_coh), no_coh | |||
| # if len(no_coh) < 4 : | |||
| # #print xpos[ic], ypos[ic], ic | |||
| # # plt.plot(r) | |||
| # # plt.show() | |||
| # xpos[ic] = numpy.nan | |||
| # ypos[ic] = numpy.nan | |||
| # | |||
| # ic = ic + 1 | |||
| # if (ic == len(indxs)) : | |||
| # break | |||
| # #print( xpos, ypos) | |||
| # | |||
| # indxs = (numpy.isfinite(list(xpos))==True).nonzero() | |||
| # #print indxs[0] | |||
| # if len(indxs[0]) < 4 : | |||
| # array[ii,:,:] = 0. | |||
| # return | |||
| # | |||
| # xpos = xpos[indxs[0]] | |||
| # ypos = ypos[indxs[0]] | |||
| # for i in range(0,len(ypos)): | |||
| # ypos[i]=int(ypos[i]) | |||
| # junk = tmp | |||
| # tmp = junk*0.0 | |||
| # | |||
| # tmp[list(xpos + (ypos*num_hei))] = junk[list(xpos + (ypos*num_hei))] | |||
| # array[ii,:,:] = numpy.reshape(tmp,(num_prof,num_hei)) | |||
| # | |||
| # #print array.shape | |||
| # #tmp = numpy.reshape(tmp,(num_prof,num_hei)) | |||
| # #print tmp.shape | |||
| # | |||
| # # fig = plt.figure(figsize=(6,5)) | |||
| # # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8 | |||
| # # ax = fig.add_axes([left, bottom, width, height]) | |||
| # # x = range(num_prof) | |||
| # # y = range(num_hei) | |||
| # # cp = ax.contour(y,x,array[ii,:,:]) | |||
| # # ax.clabel(cp, inline=True,fontsize=10) | |||
| # # plt.show() | |||
| # return array | |||
| # | |||
| r1399 | |||
| class IntegrationFaradaySpectra(Operation): | |||
| __profIndex = 0 | |||
| __withOverapping = False | |||
| __byTime = False | |||
| __initime = None | |||
| __lastdatatime = None | |||
| __integrationtime = None | |||
| __buffer_spc = None | |||
| __buffer_cspc = None | |||
| __buffer_dc = None | |||
| __dataReady = False | |||
| __timeInterval = None | |||
| r1528 | n_ints = None #matriz de numero de integracions (CH,HEI) | ||
| r1399 | n = None | ||
| r1473 | minHei_ind = None | ||
| maxHei_ind = None | |||
| r1506 | navg = 1.0 | ||
| r1473 | factor = 0.0 | ||
| r1528 | dataoutliers = None # (CHANNELS, HEIGHTS) | ||
| r1399 | |||
| r1579 | _flagProfilesByRange = False | ||
| _nProfilesByRange = 0 | |||
| r1399 | def __init__(self): | ||
| Operation.__init__(self) | |||
| r1506 | def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None, minHei=None, maxHei=None, avg=1,factor=0.75): | ||
| r1399 | """ | ||
| Set the parameters of the integration class. | |||
| Inputs: | |||
| n : Number of coherent integrations | |||
| timeInterval : Time of integration. If the parameter "n" is selected this one does not work | |||
| overlapping : | |||
| """ | |||
| self.__initime = None | |||
| self.__lastdatatime = 0 | |||
| self.__buffer_spc = [] | |||
| self.__buffer_cspc = [] | |||
| self.__buffer_dc = 0 | |||
| self.__profIndex = 0 | |||
| self.__dataReady = False | |||
| self.__byTime = False | |||
| r1473 | self.factor = factor | ||
| r1506 | self.navg = avg | ||
| r1399 | #self.ByLags = dataOut.ByLags ###REDEFINIR | ||
| self.ByLags = False | |||
| r1554 | self.maxProfilesInt = 0 | ||
| r1548 | self.__nChannels = dataOut.nChannels | ||
| r1399 | if DPL != None: | ||
| self.DPL=DPL | |||
| else: | |||
| #self.DPL=dataOut.DPL ###REDEFINIR | |||
| self.DPL=0 | |||
| if n is None and timeInterval is None: | |||
| raise ValueError("n or timeInterval should be specified ...") | |||
| if n is not None: | |||
| self.n = int(n) | |||
| else: | |||
| self.__integrationtime = int(timeInterval) | |||
| self.n = None | |||
| self.__byTime = True | |||
| r1559 | |||
| r1473 | if minHei == None: | ||
| minHei = self.dataOut.heightList[0] | |||
| if maxHei == None: | |||
| maxHei = self.dataOut.heightList[-1] | |||
| if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei): | |||
| print('minHei: %.2f is out of the heights range' % (minHei)) | |||
| print('minHei is setting to %.2f' % (self.dataOut.heightList[0])) | |||
| minHei = self.dataOut.heightList[0] | |||
| if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei): | |||
| print('maxHei: %.2f is out of the heights range' % (maxHei)) | |||
| print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1])) | |||
| maxHei = self.dataOut.heightList[-1] | |||
| ind_list1 = numpy.where(self.dataOut.heightList >= minHei) | |||
| ind_list2 = numpy.where(self.dataOut.heightList <= maxHei) | |||
| self.minHei_ind = ind_list1[0][0] | |||
| self.maxHei_ind = ind_list2[0][-1] | |||
| r1559 | |||
| r1399 | def putData(self, data_spc, data_cspc, data_dc): | ||
| """ | |||
| Add a profile to the __buffer_spc and increase in one the __profileIndex | |||
| """ | |||
| self.__buffer_spc.append(data_spc) | |||
| r1548 | if self.__nChannels < 2: | ||
| r1399 | self.__buffer_cspc = None | ||
| else: | |||
| self.__buffer_cspc.append(data_cspc) | |||
| if data_dc is None: | |||
| self.__buffer_dc = None | |||
| else: | |||
| self.__buffer_dc += data_dc | |||
| self.__profIndex += 1 | |||
| return | |||
| r1475 | def hildebrand_sekhon_Integration(self,sortdata,navg, factor): | ||
| #data debe estar ordenado | |||
| #sortdata = numpy.sort(data, axis=None) | |||
| #sortID=data.argsort() | |||
| r1399 | lenOfData = len(sortdata) | ||
| r1473 | nums_min = lenOfData*factor | ||
| r1399 | if nums_min <= 5: | ||
| nums_min = 5 | |||
| sump = 0. | |||
| sumq = 0. | |||
| j = 0 | |||
| cont = 1 | |||
| while((cont == 1)and(j < lenOfData)): | |||
| sump += sortdata[j] | |||
| sumq += sortdata[j]**2 | |||
| if j > nums_min: | |||
| rtest = float(j)/(j-1) + 1.0/navg | |||
| if ((sumq*j) > (rtest*sump**2)): | |||
| j = j - 1 | |||
| sump = sump - sortdata[j] | |||
| sumq = sumq - sortdata[j]**2 | |||
| cont = 0 | |||
| j += 1 | |||
| #lnoise = sump / j | |||
| r1473 | #print("H S done") | ||
| r1475 | #return j,sortID | ||
| return j | |||
| r1399 | |||
| def pushData(self): | |||
| """ | |||
| Return the sum of the last profiles and the profiles used in the sum. | |||
| Affected: | |||
| self.__profileIndex | |||
| """ | |||
| bufferH=None | |||
| buffer=None | |||
| buffer1=None | |||
| buffer_cspc=None | |||
| r1546 | #print("aes: ", self.__buffer_cspc) | ||
| r1399 | self.__buffer_spc=numpy.array(self.__buffer_spc) | ||
| r1548 | if self.__nChannels > 1 : | ||
| r1473 | self.__buffer_cspc=numpy.array(self.__buffer_cspc) | ||
| r1546 | |||
| r1473 | #print("FREQ_DC",self.__buffer_spc.shape,self.__buffer_cspc.shape) | ||
| r1472 | |||
| r1399 | freq_dc = int(self.__buffer_spc.shape[2] / 2) | ||
| #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights) | |||
| r1473 | |||
| r1528 | self.dataOutliers = numpy.zeros((self.nChannels,self.nHeights)) # --> almacen de outliers | ||
| r1473 | for k in range(self.minHei_ind,self.maxHei_ind): | ||
| r1548 | if self.__nChannels > 1: | ||
| r1472 | buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k]) | ||
| r1546 | |||
| r1399 | outliers_IDs_cspc=[] | ||
| cspc_outliers_exist=False | |||
| for i in range(self.nChannels):#dataOut.nChannels): | |||
| buffer1=numpy.copy(self.__buffer_spc[:,i,:,k]) | |||
| indexes=[] | |||
| #sortIDs=[] | |||
| outliers_IDs=[] | |||
| r1528 | for j in range(self.nProfiles): #frecuencias en el tiempo | ||
| r1399 | # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0 | ||
| # continue | |||
| # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1 | |||
| # continue | |||
| buffer=buffer1[:,j] | |||
| r1475 | sortdata = numpy.sort(buffer, axis=None) | ||
| r1528 | |||
| r1475 | sortID=buffer.argsort() | ||
| index = _noise.hildebrand_sekhon2(sortdata,self.navg) | |||
| #index,sortID=self.hildebrand_sekhon_Integration(buffer,1,self.factor) | |||
| r1399 | |||
| r1528 | # fig,ax = plt.subplots() | ||
| # ax.set_title(str(k)+" "+str(j)) | |||
| # x=range(len(sortdata)) | |||
| # ax.scatter(x,sortdata) | |||
| # ax.axvline(index) | |||
| # plt.show() | |||
| r1399 | indexes.append(index) | ||
| #sortIDs.append(sortID) | |||
| outliers_IDs=numpy.append(outliers_IDs,sortID[index:]) | |||
| r1528 | #print("Outliers: ",outliers_IDs) | ||
| r1399 | outliers_IDs=numpy.array(outliers_IDs) | ||
| outliers_IDs=outliers_IDs.ravel() | |||
| outliers_IDs=numpy.unique(outliers_IDs) | |||
| outliers_IDs=outliers_IDs.astype(numpy.dtype('int64')) | |||
| indexes=numpy.array(indexes) | |||
| indexmin=numpy.min(indexes) | |||
| r1506 | |||
| r1528 | |||
| #print(indexmin,buffer1.shape[0], k) | |||
| # fig,ax = plt.subplots() | |||
| # ax.plot(sortdata) | |||
| # ax2 = ax.twinx() | |||
| # x=range(len(indexes)) | |||
| # #plt.scatter(x,indexes) | |||
| # ax2.scatter(x,indexes) | |||
| # plt.show() | |||
| r1399 | if indexmin != buffer1.shape[0]: | ||
| r1548 | if self.__nChannels > 1: | ||
| r1473 | cspc_outliers_exist= True | ||
| r1506 | |||
| r1399 | lt=outliers_IDs | ||
| r1528 | #avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0) | ||
| r1399 | |||
| for p in list(outliers_IDs): | |||
| r1528 | #buffer1[p,:]=avg | ||
| buffer1[p,:] = numpy.NaN | |||
| self.dataOutliers[i,k] = len(outliers_IDs) | |||
| r1399 | |||
| r1546 | |||
| r1399 | self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1) | ||
| r1528 | |||
| r1399 | |||
| r1548 | if self.__nChannels > 1: | ||
| r1546 | outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs) | ||
| r1528 | |||
| r1548 | if self.__nChannels > 1: | ||
| r1546 | outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64')) | ||
| if cspc_outliers_exist: | |||
| r1528 | |||
| r1399 | lt=outliers_IDs_cspc | ||
| r1528 | #avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0) | ||
| r1399 | for p in list(outliers_IDs_cspc): | ||
| r1528 | #buffer_cspc[p,:]=avg | ||
| buffer_cspc[p,:] = numpy.NaN | |||
| r1399 | |||
| r1548 | if self.__nChannels > 1: | ||
| r1473 | self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc) | ||
| r1546 | |||
| r1399 | |||
| r1528 | nOutliers = len(outliers_IDs) | ||
| #print("Outliers n: ",self.dataOutliers,nOutliers) | |||
| r1399 | buffer=None | ||
| bufferH=None | |||
| buffer1=None | |||
| buffer_cspc=None | |||
| buffer=None | |||
| r1528 | |||
| #data_spc = numpy.sum(self.__buffer_spc,axis=0) | |||
| data_spc = numpy.nansum(self.__buffer_spc,axis=0) | |||
| r1548 | if self.__nChannels > 1: | ||
| r1528 | #data_cspc = numpy.sum(self.__buffer_cspc,axis=0) | ||
| data_cspc = numpy.nansum(self.__buffer_cspc,axis=0) | |||
| r1546 | else: | ||
| data_cspc = None | |||
| r1399 | data_dc = self.__buffer_dc | ||
| r1528 | #(CH, HEIGH) | ||
| r1554 | self.maxProfilesInt = self.__profIndex - 1 | ||
| r1540 | n = self.__profIndex - self.dataOutliers # n becomes a matrix | ||
| r1399 | |||
| self.__buffer_spc = [] | |||
| self.__buffer_cspc = [] | |||
| self.__buffer_dc = 0 | |||
| self.__profIndex = 0 | |||
| r1546 | #print("cleaned ",data_cspc) | ||
| r1399 | return data_spc, data_cspc, data_dc, n | ||
| def byProfiles(self, *args): | |||
| self.__dataReady = False | |||
| avgdata_spc = None | |||
| avgdata_cspc = None | |||
| avgdata_dc = None | |||
| self.putData(*args) | |||
| if self.__profIndex == self.n: | |||
| avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData() | |||
| r1528 | self.n_ints = n | ||
| r1399 | self.__dataReady = True | ||
| return avgdata_spc, avgdata_cspc, avgdata_dc | |||
| def byTime(self, datatime, *args): | |||
| self.__dataReady = False | |||
| avgdata_spc = None | |||
| avgdata_cspc = None | |||
| avgdata_dc = None | |||
| self.putData(*args) | |||
| if (datatime - self.__initime) >= self.__integrationtime: | |||
| avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData() | |||
| r1528 | self.n_ints = n | ||
| r1399 | self.__dataReady = True | ||
| return avgdata_spc, avgdata_cspc, avgdata_dc | |||
| def integrate(self, datatime, *args): | |||
| if self.__profIndex == 0: | |||
| self.__initime = datatime | |||
| if self.__byTime: | |||
| avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime( | |||
| datatime, *args) | |||
| else: | |||
| avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args) | |||
| if not self.__dataReady: | |||
| return None, None, None, None | |||
| r1546 | #print("integrate", avgdata_cspc) | ||
| r1399 | return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc | ||
| r1506 | def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False, minHei=None, maxHei=None, avg=1, factor=0.75): | ||
| r1540 | self.dataOut = dataOut | ||
| r1399 | if n == 1: | ||
| r1473 | return self.dataOut | ||
| r1559 | self.dataOut.processingHeaderObj.timeIncohInt = timeInterval | ||
| r1579 | |||
| r1583 | if dataOut.flagProfilesByRange: | ||
| r1579 | self._flagProfilesByRange = True | ||
| r1473 | if self.dataOut.nChannels == 1: | ||
| self.dataOut.data_cspc = None #si es un solo canal no vale la pena acumular DATOS | |||
| r1546 | #print("IN spc:", self.dataOut.data_spc.shape, self.dataOut.data_cspc) | ||
| r1399 | if not self.isConfig: | ||
| r1506 | self.setup(self.dataOut, n, timeInterval, overlapping,DPL ,minHei, maxHei, avg, factor) | ||
| r1399 | self.isConfig = True | ||
| if not self.ByLags: | |||
| r1473 | self.nProfiles=self.dataOut.nProfiles | ||
| self.nChannels=self.dataOut.nChannels | |||
| self.nHeights=self.dataOut.nHeights | |||
| avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime, | |||
| self.dataOut.data_spc, | |||
| self.dataOut.data_cspc, | |||
| self.dataOut.data_dc) | |||
| r1399 | else: | ||
| r1473 | self.nProfiles=self.dataOut.nProfiles | ||
| self.nChannels=self.dataOut.nChannels | |||
| self.nHeights=self.dataOut.nHeights | |||
| avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime, | |||
| self.dataOut.dataLag_spc, | |||
| self.dataOut.dataLag_cspc, | |||
| self.dataOut.dataLag_dc) | |||
| r1540 | self.dataOut.flagNoData = True | ||
| r1579 | |||
| if self._flagProfilesByRange: | |||
| dataOut.flagProfilesByRange = True | |||
| self._nProfilesByRange += dataOut.nProfilesByRange | |||
| r1399 | if self.__dataReady: | ||
| if not self.ByLags: | |||
| r1472 | if self.nChannels == 1: | ||
| r1473 | #print("f int", avgdata_spc.shape) | ||
| self.dataOut.data_spc = avgdata_spc | |||
| r1546 | self.dataOut.data_cspc = None | ||
| r1472 | else: | ||
| r1473 | self.dataOut.data_spc = numpy.squeeze(avgdata_spc) | ||
| self.dataOut.data_cspc = numpy.squeeze(avgdata_cspc) | |||
| self.dataOut.data_dc = avgdata_dc | |||
| r1528 | self.dataOut.data_outlier = self.dataOutliers | ||
| r1579 | |||
| r1472 | |||
| r1399 | else: | ||
| r1473 | self.dataOut.dataLag_spc = avgdata_spc | ||
| self.dataOut.dataLag_cspc = avgdata_cspc | |||
| self.dataOut.dataLag_dc = avgdata_dc | |||
| r1399 | |||
| r1473 | self.dataOut.data_spc=self.dataOut.dataLag_spc[:,:,:,self.dataOut.LagPlot] | ||
| self.dataOut.data_cspc=self.dataOut.dataLag_cspc[:,:,:,self.dataOut.LagPlot] | |||
| self.dataOut.data_dc=self.dataOut.dataLag_dc[:,:,self.dataOut.LagPlot] | |||
| r1399 | |||
| r1528 | self.dataOut.nIncohInt *= self.n_ints | ||
| r1554 | #print("maxProfilesInt: ",self.maxProfilesInt) | ||
| r1473 | self.dataOut.utctime = avgdatatime | ||
| self.dataOut.flagNoData = False | |||
| r1579 | |||
| dataOut.nProfilesByRange = self._nProfilesByRange | |||
| r1583 | self._nProfilesByRange = numpy.zeros(len(dataOut.heightList)) | ||
| r1579 | self._flagProfilesByRange = False | ||
| r1559 | |||
| # #update Processing Header: | |||
| # self.dataOut.processingHeaderObj.nIncohInt = | |||
| # self.dataOut.processingHeaderObj.nFFTPoints = self.dataOut.nFFTPoints | |||
| r1546 | #print("Faraday Integration DONE...", self.dataOut.data_cspc) | ||
| r1540 | #print(self.dataOut.flagNoData) | ||
| r1473 | return self.dataOut | ||
| r1399 | |||
| r1547 | |||
|
Juan C. Espinoza
|
r1287 | class removeInterference(Operation): | |
|
José Chávez
|
r897 | ||
| r1547 | def removeInterference3(self, min_hei = None, max_hei = None): | ||
| jspectra = self.dataOut.data_spc | |||
| #jcspectra = self.dataOut.data_cspc | |||
| jnoise = self.dataOut.getNoise() | |||
| num_incoh = self.dataOut.max_nIncohInt | |||
| #print(jspectra.shape) | |||
| num_channel, num_prof, num_hei = jspectra.shape | |||
| minHei = min_hei | |||
| maxHei = max_hei | |||
| ######################################################################## | |||
| if minHei == None or (minHei < self.dataOut.heightList[0]): | |||
| minHei = self.dataOut.heightList[0] | |||
| if maxHei == None or (maxHei > self.dataOut.heightList[-1]): | |||
| maxHei = self.dataOut.heightList[-1] | |||
| minIndex = 0 | |||
| maxIndex = 0 | |||
| heights = self.dataOut.heightList | |||
| inda = numpy.where(heights >= minHei) | |||
| indb = numpy.where(heights <= maxHei) | |||
| try: | |||
| minIndex = inda[0][0] | |||
| except: | |||
| minIndex = 0 | |||
| try: | |||
| maxIndex = indb[0][-1] | |||
| except: | |||
| maxIndex = len(heights) | |||
| if (minIndex < 0) or (minIndex > maxIndex): | |||
| raise ValueError("some value in (%d,%d) is not valid" % ( | |||
| minIndex, maxIndex)) | |||
| if (maxIndex >= self.dataOut.nHeights): | |||
| maxIndex = self.dataOut.nHeights - 1 | |||
| ######################################################################## | |||
| #dataOut.max_nIncohInt * dataOut.nCohInt | |||
| norm = self.dataOut.nIncohInt /self.dataOut.max_nIncohInt | |||
| #print(norm.shape) | |||
| # Subrutina de Remocion de la Interferencia | |||
| for ich in range(num_channel): | |||
| # Se ordena los espectros segun su potencia (menor a mayor) | |||
| #power = jspectra[ich, mask_prof, :] | |||
| interf = jspectra[ich, :, minIndex:maxIndex] | |||
| #print(interf.shape) | |||
| inttef = interf.mean(axis=1) | |||
| for hei in range(num_hei): | |||
| temp = jspectra[ich,:, hei] | |||
| temp -= inttef | |||
| temp += jnoise[ich]*norm[ich,hei] | |||
| jspectra[ich,:, hei] = temp | |||
| # Guardar Resultados | |||
| self.dataOut.data_spc = jspectra | |||
| #self.dataOut.data_cspc = jcspectra | |||
| return 1 | |||
|
ebocanegra
|
r1123 | def removeInterference2(self): | |
| r1279 | |||
|
ebocanegra
|
r1123 | cspc = self.dataOut.data_cspc | |
| spc = self.dataOut.data_spc | |||
| r1279 | Heights = numpy.arange(cspc.shape[2]) | ||
|
ebocanegra
|
r1123 | realCspc = numpy.abs(cspc) | |
| r1279 | |||
|
ebocanegra
|
r1123 | for i in range(cspc.shape[0]): | |
| LinePower= numpy.sum(realCspc[i], axis=0) | |||
| Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)] | |||
| SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ] | |||
| InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 ) | |||
| InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)] | |||
| InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)] | |||
| r1279 | |||
|
ebocanegra
|
r1123 | InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) ) | |
| #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax])) | |||
| if len(InterferenceRange)<int(cspc.shape[1]*0.3): | |||
| cspc[i,InterferenceRange,:] = numpy.NaN | |||
| r1279 | |||
|
ebocanegra
|
r1123 | self.dataOut.data_cspc = cspc | |
| r1279 | |||
|
Juan C. Espinoza
|
r1295 | def removeInterference(self, interf = 2, hei_interf = None, nhei_interf = None, offhei_interf = None): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | jspectra = self.dataOut.data_spc | |
| jcspectra = self.dataOut.data_cspc | |||
| jnoise = self.dataOut.getNoise() | |||
| r1546 | #num_incoh = self.dataOut.nIncohInt | ||
| num_incoh = self.dataOut.max_nIncohInt | |||
| #print("spc: ", jspectra.shape, jcspectra) | |||
|
José Chávez
|
r1120 | num_channel = jspectra.shape[0] | |
| num_prof = jspectra.shape[1] | |||
| num_hei = jspectra.shape[2] | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # hei_interf | |
|
Miguel Valdez
|
r612 | if hei_interf is None: | |
| r1547 | count_hei = int(num_hei / 2) # a half of total ranges | ||
|
George Yong
|
r1167 | hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei | |
|
Daniel Valdez
|
r487 | hei_interf = numpy.asarray(hei_interf)[0] | |
| r1546 | #print(hei_interf) | ||
|
José Chávez
|
r1120 | # nhei_interf | |
|
Daniel Valdez
|
r487 | if (nhei_interf == None): | |
| nhei_interf = 5 | |||
| if (nhei_interf < 1): | |||
|
José Chávez
|
r897 | nhei_interf = 1 | |
|
Daniel Valdez
|
r487 | if (nhei_interf > count_hei): | |
| nhei_interf = count_hei | |||
|
José Chávez
|
r897 | if (offhei_interf == None): | |
|
Daniel Valdez
|
r487 | offhei_interf = 0 | |
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | ind_hei = list(range(num_hei)) | |
|
José Chávez
|
r897 | # mask_prof = numpy.asarray(range(num_prof - 2)) + 1 | |
|
Daniel Valdez
|
r487 | # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1 | |
|
George Yong
|
r1167 | mask_prof = numpy.asarray(list(range(num_prof))) | |
|
Daniel Valdez
|
r487 | num_mask_prof = mask_prof.size | |
|
José Chávez
|
r1120 | comp_mask_prof = [0, num_prof / 2] | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal | |
|
Daniel Valdez
|
r487 | if (jnoise.size < num_channel or numpy.isnan(jnoise).any()): | |
| jnoise = numpy.nan | |||
| noise_exist = jnoise[0] < numpy.Inf | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # Subrutina de Remocion de la Interferencia | |
|
Daniel Valdez
|
r487 | for ich in range(num_channel): | |
|
José Chávez
|
r1120 | # Se ordena los espectros segun su potencia (menor a mayor) | |
| power = jspectra[ich, mask_prof, :] | |||
| power = power[:, hei_interf] | |||
| power = power.sum(axis=0) | |||
|
Daniel Valdez
|
r487 | psort = power.ravel().argsort() | |
| r1583 | #print(hei_interf[psort[list(range(offhei_interf, nhei_interf + offhei_interf))]]) | ||
|
José Chávez
|
r1120 | # Se estima la interferencia promedio en los Espectros de Potencia empleando | |
|
George Yong
|
r1167 | junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range( | |
| offhei_interf, nhei_interf + offhei_interf))]]] | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if noise_exist: | |
|
José Chávez
|
r1120 | # tmp_noise = jnoise[ich] / num_prof | |
|
Daniel Valdez
|
r487 | tmp_noise = jnoise[ich] | |
| junkspc_interf = junkspc_interf - tmp_noise | |||
| #junkspc_interf[:,comp_mask_prof] = 0 | |||
| r1583 | #print(junkspc_interf.shape) | ||
|
José Chávez
|
r1120 | jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf | |
|
Daniel Valdez
|
r487 | jspc_interf = jspc_interf.transpose() | |
|
José Chávez
|
r1120 | # Calculando el espectro de interferencia promedio | |
| r1546 | noiseid = numpy.where(jspc_interf <= tmp_noise / numpy.sqrt(num_incoh)) | ||
|
Daniel Valdez
|
r487 | noiseid = noiseid[0] | |
| cnoiseid = noiseid.size | |||
| r1546 | interfid = numpy.where(jspc_interf > tmp_noise / numpy.sqrt(num_incoh)) | ||
|
Daniel Valdez
|
r487 | interfid = interfid[0] | |
| cinterfid = interfid.size | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | if (cnoiseid > 0): | |
| jspc_interf[noiseid] = 0 | |||
| # Expandiendo los perfiles a limpiar | |||
|
Daniel Valdez
|
r487 | if (cinterfid > 0): | |
|
José Chávez
|
r1120 | new_interfid = ( | |
| numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof | |||
|
José Chávez
|
r897 | new_interfid = numpy.asarray(new_interfid) | |
|
Daniel Valdez
|
r487 | new_interfid = {x for x in new_interfid} | |
| new_interfid = numpy.array(list(new_interfid)) | |||
| new_cinterfid = new_interfid.size | |||
|
José Chávez
|
r1120 | else: | |
| new_cinterfid = 0 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for ip in range(new_cinterfid): | |
|
José Chávez
|
r1120 | ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort() | |
| r1547 | jspc_interf[new_interfid[ip]] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]] | ||
|
José Chávez
|
r897 | ||
| r1547 | jspectra[ich, :, ind_hei] = jspectra[ich, :,ind_hei] - jspc_interf # Corregir indices | ||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # Removiendo la interferencia del punto de mayor interferencia | |
|
Daniel Valdez
|
r487 | ListAux = jspc_interf[mask_prof].tolist() | |
| maxid = ListAux.index(max(ListAux)) | |||
| r1583 | #print(cinterfid) | ||
|
Daniel Valdez
|
r487 | if cinterfid > 0: | |
|
José Chávez
|
r1120 | for ip in range(cinterfid * (interf == 2) - 1): | |
| ind = (jspectra[ich, interfid[ip], :] < tmp_noise * | |||
| (1 + 1 / numpy.sqrt(num_incoh))).nonzero() | |||
|
Daniel Valdez
|
r487 | cind = len(ind) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if (cind > 0): | |
|
José Chávez
|
r1120 | jspectra[ich, interfid[ip], ind] = tmp_noise * \ | |
| (1 + (numpy.random.uniform(cind) - 0.5) / | |||
| numpy.sqrt(num_incoh)) | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | ind = numpy.array([-2, -1, 1, 2]) | |
| xx = numpy.zeros([4, 4]) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for id1 in range(4): | |
|
George Yong
|
r1167 | xx[:, id1] = ind[id1]**numpy.asarray(list(range(4))) | |
|
Daniel Valdez
|
r487 | xx_inv = numpy.linalg.inv(xx) | |
|
José Chávez
|
r1120 | xx = xx_inv[:, 0] | |
| ind = (ind + maxid + num_mask_prof) % num_mask_prof | |||
| yy = jspectra[ich, mask_prof[ind], :] | |||
| r1547 | jspectra[ich, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx) | ||
|
José Chávez
|
r1120 | ||
| indAux = (jspectra[ich, :, :] < tmp_noise * | |||
| (1 - 1 / numpy.sqrt(num_incoh))).nonzero() | |||
| r1583 | #print(indAux) | ||
|
José Chávez
|
r1120 | jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \ | |
| (1 - 1 / numpy.sqrt(num_incoh)) | |||
| # Remocion de Interferencia en el Cross Spectra | |||
| if jcspectra is None: | |||
| return jspectra, jcspectra | |||
|
Juan C. Espinoza
|
r1197 | num_pairs = int(jcspectra.size / (num_prof * num_hei)) | |
|
Daniel Valdez
|
r487 | jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for ip in range(num_pairs): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | #------------------------------------------- | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | cspower = numpy.abs(jcspectra[ip, mask_prof, :]) | |
| cspower = cspower[:, hei_interf] | |||
| cspower = cspower.sum(axis=0) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | cspsort = cspower.ravel().argsort() | |
|
George Yong
|
r1167 | junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range( | |
| offhei_interf, nhei_interf + offhei_interf))]]] | |||
|
Daniel Valdez
|
r487 | junkcspc_interf = junkcspc_interf.transpose() | |
|
José Chávez
|
r1120 | jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort() | |
|
José Chávez
|
r897 | ||
|
George Yong
|
r1194 | median_real = int(numpy.median(numpy.real( | |
| junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :]))) | |||
| median_imag = int(numpy.median(numpy.imag( | |||
| junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :]))) | |||
| comp_mask_prof = [int(e) for e in comp_mask_prof] | |||
|
José Chávez
|
r1120 | junkcspc_interf[comp_mask_prof, :] = numpy.complex( | |
| median_real, median_imag) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for iprof in range(num_prof): | |
|
José Chávez
|
r1120 | ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort() | |
|
George Yong
|
r1194 | jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]] | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # Removiendo la Interferencia | |
| jcspectra[ip, :, ind_hei] = jcspectra[ip, | |||
| :, ind_hei] - jcspc_interf | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist() | |
| maxid = ListAux.index(max(ListAux)) | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | ind = numpy.array([-2, -1, 1, 2]) | |
| xx = numpy.zeros([4, 4]) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for id1 in range(4): | |
|
George Yong
|
r1167 | xx[:, id1] = ind[id1]**numpy.asarray(list(range(4))) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | xx_inv = numpy.linalg.inv(xx) | |
|
José Chávez
|
r1120 | xx = xx_inv[:, 0] | |
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | ind = (ind + maxid + num_mask_prof) % num_mask_prof | |
| yy = jcspectra[ip, mask_prof[ind], :] | |||
| jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx) | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r1120 | # Guardar Resultados | |
|
Daniel Valdez
|
r487 | self.dataOut.data_spc = jspectra | |
| self.dataOut.data_cspc = jcspectra | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return 1 | |
|
José Chávez
|
r897 | ||
| r1547 | def run(self, dataOut, interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None, mode=1, minHei=None, maxHei=None): | ||
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1287 | self.dataOut = dataOut | |
|
Daniel Valdez
|
r487 | ||
|
Juan C. Espinoza
|
r1287 | if mode == 1: | |
| self.removeInterference(interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None) | |||
| elif mode == 2: | |||
| self.removeInterference2() | |||
| r1547 | elif mode == 3: | ||
| self.removeInterference3(min_hei=minHei, max_hei=maxHei) | |||
|
Juan C. Espinoza
|
r1287 | return self.dataOut | |
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1177 | ||
|
José Chávez
|
r1120 | class IncohInt(Operation): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | __profIndex = 0 | |
|
José Chávez
|
r1120 | __withOverapping = False | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | __byTime = False | |
| __initime = None | |||
| __lastdatatime = None | |||
| __integrationtime = None | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | __buffer_spc = None | |
| __buffer_cspc = None | |||
| __buffer_dc = None | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | __dataReady = False | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | __timeInterval = None | |
| r1528 | incohInt = 0 | ||
| nOutliers = 0 | |||
|
Daniel Valdez
|
r487 | n = None | |
| r1579 | |||
| _flagProfilesByRange = False | |||
| _nProfilesByRange = 0 | |||
|
George Yong
|
r1179 | def __init__(self): | |
|
George Yong
|
r1171 | ||
|
George Yong
|
r1179 | Operation.__init__(self) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def setup(self, n=None, timeInterval=None, overlapping=False): | |
| """ | |||
| Set the parameters of the integration class. | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | Inputs: | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | n : Number of coherent integrations | |
| timeInterval : Time of integration. If the parameter "n" is selected this one does not work | |||
|
José Chávez
|
r897 | overlapping : | |
|
Daniel Valdez
|
r487 | """ | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.__initime = None | |
| self.__lastdatatime = 0 | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | self.__buffer_spc = 0 | |
| self.__buffer_cspc = 0 | |||
| self.__buffer_dc = 0 | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | self.__profIndex = 0 | |
| self.__dataReady = False | |||
| self.__byTime = False | |||
| r1528 | self.incohInt = 0 | ||
| self.nOutliers = 0 | |||
|
Miguel Valdez
|
r623 | if n is None and timeInterval is None: | |
|
George Yong
|
r1167 | raise ValueError("n or timeInterval should be specified ...") | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | if n is not None: | |
| self.n = int(n) | |||
|
Daniel Valdez
|
r487 | else: | |
| r1279 | |||
|
José Chávez
|
r1120 | self.__integrationtime = int(timeInterval) | |
|
Miguel Valdez
|
r623 | self.n = None | |
|
Daniel Valdez
|
r487 | self.__byTime = True | |
|
José Chávez
|
r897 | ||
| r1583 | |||
| r1559 | |||
|
Daniel Valdez
|
r487 | def putData(self, data_spc, data_cspc, data_dc): | |
| """ | |||
| Add a profile to the __buffer_spc and increase in one the __profileIndex | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | """ | |
| r1506 | if data_spc.all() == numpy.nan : | ||
| print("nan ") | |||
| return | |||
|
Miguel Valdez
|
r623 | self.__buffer_spc += data_spc | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | if data_cspc is None: | |
| self.__buffer_cspc = None | |||
| else: | |||
| self.__buffer_cspc += data_cspc | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | if data_dc is None: | |
| self.__buffer_dc = None | |||
| else: | |||
| self.__buffer_dc += data_dc | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | self.__profIndex += 1 | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def pushData(self): | |
| """ | |||
| Return the sum of the last profiles and the profiles used in the sum. | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | Affected: | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.__profileIndex | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | """ | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | data_spc = self.__buffer_spc | |
| data_cspc = self.__buffer_cspc | |||
| data_dc = self.__buffer_dc | |||
|
Daniel Valdez
|
r487 | n = self.__profIndex | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | self.__buffer_spc = 0 | |
| self.__buffer_cspc = 0 | |||
| self.__buffer_dc = 0 | |||
| r1528 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return data_spc, data_cspc, data_dc, n | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def byProfiles(self, *args): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.__dataReady = False | |
|
Miguel Valdez
|
r624 | avgdata_spc = None | |
| avgdata_cspc = None | |||
| avgdata_dc = None | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.putData(*args) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if self.__profIndex == self.n: | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData() | |
|
Miguel Valdez
|
r623 | self.n = n | |
|
Daniel Valdez
|
r487 | self.__dataReady = True | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return avgdata_spc, avgdata_cspc, avgdata_dc | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def byTime(self, datatime, *args): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.__dataReady = False | |
|
Miguel Valdez
|
r624 | avgdata_spc = None | |
| avgdata_cspc = None | |||
| avgdata_dc = None | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.putData(*args) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if (datatime - self.__initime) >= self.__integrationtime: | |
| avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData() | |||
| self.n = n | |||
| self.__dataReady = True | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return avgdata_spc, avgdata_cspc, avgdata_dc | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def integrate(self, datatime, *args): | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | if self.__profIndex == 0: | |
|
Daniel Valdez
|
r487 | self.__initime = datatime | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if self.__byTime: | |
|
José Chávez
|
r1120 | avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime( | |
| datatime, *args) | |||
|
Daniel Valdez
|
r487 | else: | |
| avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args) | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | if not self.__dataReady: | |
|
Daniel Valdez
|
r487 | return None, None, None, None | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r623 | return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def run(self, dataOut, n=None, timeInterval=None, overlapping=False): | |
|
José Chávez
|
r1120 | if n == 1: | |
|
Juan C. Espinoza
|
r1287 | return dataOut | |
| r1279 | |||
| r1528 | if dataOut.flagNoData == True: | ||
| return dataOut | |||
| r1540 | |||
| r1579 | if dataOut.flagProfilesByRange == True: | ||
| self._flagProfilesByRange = True | |||
|
Miguel Valdez
|
r623 | dataOut.flagNoData = True | |
| r1559 | dataOut.processingHeaderObj.timeIncohInt = timeInterval | ||
|
Daniel Valdez
|
r487 | if not self.isConfig: | |
| r1583 | self._nProfilesByRange = numpy.zeros(len(dataOut.heightList)) | ||
|
Daniel Valdez
|
r487 | self.setup(n, timeInterval, overlapping) | |
| self.isConfig = True | |||
|
José Chávez
|
r897 | ||
| r1559 | |||
|
Daniel Valdez
|
r487 | avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime, | |
| dataOut.data_spc, | |||
| dataOut.data_cspc, | |||
| dataOut.data_dc) | |||
| r1555 | |||
| r1528 | self.incohInt += dataOut.nIncohInt | ||
| r1579 | |||
| r1555 | |||
| if isinstance(dataOut.data_outlier,numpy.ndarray) or isinstance(dataOut.data_outlier,int) or isinstance(dataOut.data_outlier, float): | |||
| self.nOutliers += dataOut.data_outlier | |||
| r1579 | if self._flagProfilesByRange: | ||
| dataOut.flagProfilesByRange = True | |||
| self._nProfilesByRange += dataOut.nProfilesByRange | |||
|
Daniel Valdez
|
r487 | if self.__dataReady: | |
| r1540 | #print("prof: ",dataOut.max_nIncohInt,self.__profIndex) | ||
|
Daniel Valdez
|
r487 | dataOut.data_spc = avgdata_spc | |
| dataOut.data_cspc = avgdata_cspc | |||
| r1279 | dataOut.data_dc = avgdata_dc | ||
| r1528 | dataOut.nIncohInt = self.incohInt | ||
| dataOut.data_outlier = self.nOutliers | |||
|
Daniel Valdez
|
r487 | dataOut.utctime = avgdatatime | |
|
George Yong
|
r1171 | dataOut.flagNoData = False | |
| r1528 | self.incohInt = 0 | ||
| self.nOutliers = 0 | |||
| self.__profIndex = 0 | |||
| r1579 | dataOut.nProfilesByRange = self._nProfilesByRange | ||
| r1583 | self._nProfilesByRange = numpy.zeros(len(dataOut.heightList)) | ||
| r1579 | self._flagProfilesByRange = False | ||
| r1541 | #print("IncohInt Done") | ||
| r1279 | return dataOut | ||
| r1344 | |||
| class dopplerFlip(Operation): | |||
| r1370 | |||
| r1344 | def run(self, dataOut): | ||
| # arreglo 1: (num_chan, num_profiles, num_heights) | |||
| r1370 | self.dataOut = dataOut | ||
| r1344 | # JULIA-oblicua, indice 2 | ||
| # arreglo 2: (num_profiles, num_heights) | |||
| jspectra = self.dataOut.data_spc[2] | |||
| jspectra_tmp = numpy.zeros(jspectra.shape) | |||
| num_profiles = jspectra.shape[0] | |||
| freq_dc = int(num_profiles / 2) | |||
| # Flip con for | |||
| for j in range(num_profiles): | |||
| jspectra_tmp[num_profiles-j-1]= jspectra[j] | |||
| # Intercambio perfil de DC con perfil inmediato anterior | |||
| jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1] | |||
| jspectra_tmp[freq_dc]= jspectra[freq_dc] | |||
| # canal modificado es re-escrito en el arreglo de canales | |||
| self.dataOut.data_spc[2] = jspectra_tmp | |||
| r1370 | return self.dataOut | ||
| r1570 | |||
| class cleanJULIAInterf(Operation): | |||
| """ | |||
| Operación de prueba | |||
| """ | |||
| __slots__ =('heights_indx', 'repeats','span' ,'step', 'factor', 'idate', 'idxs','isConfig','minHrefN', 'maxHrefN') | |||
| def __init__(self): | |||
| self.repeats = 0 | |||
| self.factor=1 | |||
| self.isConfig = False | |||
| self.idxs = None | |||
| self.heights_indx = None | |||
| def setup(self, dataOutHeightsList, heightsList, span=10, repeats=0, step=0, idate=None, startH=None, endH=None, minHref=None, maxHref=None): | |||
| totalHeihtList = dataOutHeightsList | |||
| heights = [float(hei) for hei in heightsList] | |||
| for r in range(repeats): | |||
| heights += [ (h+(step*(r+1))) for h in heights] | |||
| #print(heights) | |||
| self.heights_indx = [getHei_index(h,h,totalHeihtList)[0] for h in heights] | |||
| self.minHrefN, self.maxHrefN = getHei_index(minHref,maxHref,totalHeihtList) | |||
| self.config = True | |||
| self.span = span | |||
| def run(self, dataOut, heightsList, span=10, repeats=0, step=0, idate=None, startH=None, endH=None, minHref=None, maxHref=None): | |||
| self.dataOut = dataOut | |||
| startTime = datetime.datetime.combine(idate,startH) | |||
| endTime = datetime.datetime.combine(idate,endH) | |||
| currentTime = datetime.datetime.fromtimestamp(self.dataOut.utctime) | |||
| if currentTime < startTime or currentTime > endTime: | |||
| return self.dataOut | |||
| if not self.isConfig: | |||
| self.setup(self.dataOut.heightList,heightsList, span=span, repeats=repeats, step=step, idate=idate, startH=startH, endH=endH, minHref=minHref, maxHref=maxHref ) | |||
| for ch in range(self.dataOut.data_spc.shape[0]): | |||
| i = 0 | |||
| N_ref = self.dataOut.data_spc[ch, :, self.minHrefN: self.maxHrefN].mean() | |||
| mn = self.heights_indx[-1] - self.span/2 | |||
| mx = self.heights_indx[-1] + self.span/2 | |||
| J_lev = self.dataOut.data_spc[ch, :, mn: mx].mean() - N_ref | |||
| for hei in self.heights_indx: | |||
| h = hei - 1 | |||
| mn_i = hei - self.span/2 | |||
| mx_i = hei + self.span/2 | |||
| self.dataOut.data_spc[ch, :,mn_i:mx_i ] -= J_lev | |||
| i += 1 | |||
| return self.dataOut |
