##// END OF EJS Templates
Update from schain_tmp Joab Version - AVP
sebastianVP -
r1753:0a6e6a51ad73
parent child
Show More
@@ -26,6 +26,10 from numpy import NaN
26 26 from scipy.optimize.optimize import OptimizeWarning
27 27 warnings.filterwarnings('ignore')
28 28
29 import os
30 import csv
31 from scipy import signal
32 import matplotlib.pyplot as plt
29 33
30 34 SPEED_OF_LIGHT = 299792458
31 35
@@ -72,27 +76,47 class ParametersProc(ProcessingUnit):
72 76 self.dataOut.useLocalTime = self.dataIn.useLocalTime
73 77
74 78 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
79 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
75 80 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
81
76 82 self.dataOut.channelList = self.dataIn.channelList
77 83 self.dataOut.heightList = self.dataIn.heightList
84 self.dataOut.ipp = self.dataIn.ipp
85 self.dataOut.ippSeconds = self.dataIn.ippSeconds
86 self.dataOut.deltaHeight = self.dataIn.deltaHeight
78 87 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
79 # self.dataOut.nBaud = self.dataIn.nBaud
80 # self.dataOut.nCode = self.dataIn.nCode
81 # self.dataOut.code = self.dataIn.code
88
89 self.dataOut.nBaud = self.dataIn.nBaud
90 self.dataOut.nCode = self.dataIn.nCode
91 self.dataOut.code = self.dataIn.code
92 self.dataOut.nProfiles = self.dataIn.nProfiles
93
82 94 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
83 95 self.dataOut.utctime = self.dataIn.utctime
84 96 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
85 97 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
86 98 self.dataOut.nCohInt = self.dataIn.nCohInt
99 self.dataOut.nIncohInt = self.dataIn.nIncohInt
100 self.dataOut.ippSeconds = self.dataIn.ippSeconds
101 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
102
87 103 self.dataOut.timeInterval1 = self.dataIn.timeInterval
88 104 self.dataOut.heightList = self.dataIn.heightList
89 105 self.dataOut.frequency = self.dataIn.frequency
106 self.dataOut.codeList = self.dataIn.codeList
107 self.dataOut.azimuthList = self.dataIn.azimuthList
108 self.dataOut.elevationList = self.dataIn.elevationList
90 109 self.dataOut.runNextUnit = self.dataIn.runNextUnit
91 110
92 111 def run(self, runNextUnit=0):
93 112
94 113 self.dataIn.runNextUnit = runNextUnit
95 114 #---------------------- Voltage Data ---------------------------
115 try:
116 intype = self.dataIn.type.decode("utf-8")
117 self.dataIn.type = intype
118 except:
119 pass
96 120
97 121 if self.dataIn.type == "Voltage":
98 122
@@ -124,6 +148,21 class ParametersProc(ProcessingUnit):
124 148 self.dataOut.data_pre = [self.dataIn.data_spc, self.dataIn.data_cspc]
125 149 self.dataOut.data_spc = self.dataIn.data_spc
126 150 self.dataOut.data_cspc = self.dataIn.data_cspc
151 if hasattr(self.dataIn, 'data_outlier'):
152 self.dataOut.data_outlier = self.dataIn.data_outlier
153 if hasattr(self.dataIn,'flagPRofilesByRange'):
154 self.dataOut.flagProfilesByRange = self.dataIn.flagProfilesByRange
155 if hasattr(self.dataIn,'nProfilesByRange'):
156 self.dataOut.nProfilesByRange = self.dataIn.nProfilesByRange
157 if hasattr(self.dataIn,'deltaHeight'):
158 self.dataOut.deltaHeight = self.dataIn.deltaHeight
159 if hasattr(self.dataIn,'noise_estimation'):
160 self.dataOut.noise_estimation = self.dataIn.noise_estimation
161 if hasattr(self.dataIn, 'channelList'):
162 self.dataOut.channelList = self.dataIn.channelList
163 if hasattr(self.dataIn, 'pairsList'):
164 self.dataOut.pairsList = self.dataIn.pairsList
165 self.dataOut.groupList = self.dataIn.pairsList
127 166 self.dataOut.nProfiles = self.dataIn.nProfiles
128 167 self.dataOut.nIncohInt = self.dataIn.nIncohInt
129 168 self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
@@ -132,8 +171,6 class ParametersProc(ProcessingUnit):
132 171 self.dataOut.spc_noise = self.dataIn.getNoise()
133 172 self.dataOut.spc_range = (self.dataIn.getFreqRange(1) , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1))
134 173 # self.dataOut.normFactor = self.dataIn.normFactor
135 self.dataOut.pairsList = self.dataIn.pairsList
136 self.dataOut.groupList = self.dataIn.pairsList
137 174 self.dataOut.flagNoData = False
138 175
139 176 if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels
@@ -174,8 +211,14 class ParametersProc(ProcessingUnit):
174 211
175 212 if self.dataIn.type == "Parameters":
176 213 self.dataOut.copy(self.dataIn)
214 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
215 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
177 216 self.dataOut.flagNoData = False
178
217 if isinstance(self.dataIn.nIncohInt,numpy.ndarray):
218 nch, nheis = self.dataIn.nIncohInt.shape
219 if nch != self.dataIn.nChannels:
220 aux = numpy.repeat(self.dataIn.nIncohInt, self.dataIn.nChannels, axis=0)
221 self.dataOut.nIncohInt = aux
179 222 return True
180 223
181 224 self.__updateObjFromInput()
@@ -2454,8 +2497,7 class SpectralMoments(Operation):
2454 2497
2455 2498 return dataOut
2456 2499
2457 def __calculateMoments(self, oldspec, oldfreq, n0,
2458 nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None,id_ch=0):
2500 def __calculateMoments(self, oldspec, oldfreq, n0, normFactor = 1,nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None,id_ch=0):
2459 2501
2460 2502 def __GAUSSWINFIT1(A, flagPDER=0):
2461 2503 nonlocal truex, xvalid
@@ -2631,6 +2673,7 class SpectralMoments(Operation):
2631 2673 vec_n1 = numpy.empty(oldspec.shape[1])
2632 2674 vec_fp = numpy.empty(oldspec.shape[1])
2633 2675 vec_sigma_fd = numpy.empty(oldspec.shape[1])
2676 norm = 1
2634 2677
2635 2678 for ind in range(oldspec.shape[1]):
2636 2679
@@ -2644,6 +2687,10 class SpectralMoments(Operation):
2644 2687 max_spec = aux.max()
2645 2688 m = aux.tolist().index(max_spec)
2646 2689
2690 if hasattr(normFactor, "ndim"):
2691 if normFactor.ndim >= 1:
2692 norm = normFactor[ind]
2693
2647 2694 if m > 2 and m < oldfreq.size - 3:
2648 2695 newindex = m + numpy.array([-2,-1,0,1,2])
2649 2696 newfreq = numpy.arange(20)/20.0*(numpy.max(freq[newindex])-numpy.min(freq[newindex]))+numpy.min(freq[newindex])
@@ -2691,6 +2738,7 class SpectralMoments(Operation):
2691 2738 power = ((spec2[valid] - n0) * fwindow[valid]).sum()
2692 2739 fd = ((spec2[valid]- n0)*freq[valid] * fwindow[valid]).sum() / power
2693 2740 w = numpy.sqrt(((spec2[valid] - n0)*fwindow[valid]*(freq[valid]- fd)**2).sum() / power)
2741 spec2 /=(norm) #compensation for sats remove
2694 2742 snr = (spec2.mean()-n0)/n0
2695 2743 if (snr < 1.e-20): snr = 1.e-20
2696 2744
@@ -3149,6 +3197,7 class SpectralFitting(Operation):
3149 3197 self.bloque0 = numpy.zeros([nChan, nProf, nHei, nBlocks])
3150 3198
3151 3199 def __calculateMoments(self,oldspec, oldfreq, n0, nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None):
3200
3152 3201 if (nicoh is None): nicoh = 1
3153 3202 if (graph is None): graph = 0
3154 3203 if (smooth is None): smooth = 0
@@ -3396,7 +3445,7 class SpectralFitting(Operation):
3396 3445 nPairs = len(crosspairs)
3397 3446 hval=(heights >= min_hei).nonzero()
3398 3447 ih=hval[0]
3399
3448
3400 3449 for ih in range(hval[0][0],nHei):
3401 3450 for ifreq in range(nProf):
3402 3451 for ii in range(n_funct):
@@ -3868,7 +3917,7 class SpectralFitting(Operation):
3868 3917 buffer3 = 0
3869 3918
3870 3919 def run(self, dataOut, getSNR = True, path=None, file=None, groupList=None, filec=None,coh_th=None, hei_th=None,taver=None,proc=None,nhei=None,nprofs=None,ipp=None,channelList=None,Gaussian_Windowed=0):
3871
3920
3872 3921 if not numpy.any(proc):
3873 3922 nChannels = dataOut.nChannels
3874 3923 nHeights= dataOut.heightList.size
@@ -4058,6 +4107,8 class SpectralFitting(Operation):
4058 4107 if getSNR:
4059 4108 listChannels = groupArray.reshape((groupArray.size))
4060 4109 listChannels.sort()
4110 # norm Este factor debe estar implementado para ploteo o grabado como metadata
4111 # norm = dataOut.nProfiles * dataOut.nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
4061 4112 dataOut.data_snr = self.__getSNR(dataOut.data_spc[listChannels,:,:], noise[listChannels])
4062 4113 else:
4063 4114 if numpy.any(taver): taver=int(taver)
@@ -4307,6 +4358,8 class SpectralFitting(Operation):
4307 4358 if 0:
4308 4359 listChannels = groupArray.reshape((groupArray.size))
4309 4360 listChannels.sort()
4361 # norm Este factor debe estar implementado para ploteo o grabado como metadata
4362 # norm = dataOut.nProfiles * dataOut.nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
4310 4363 dataOut.data_snr = self.__getSNR(dataOut.data_spc[listChannels,:,:], my_noises[listChannels])
4311 4364 #print(dataOut.data_snr1_i)
4312 4365 # Adding coherent echoes from possible satellites.
@@ -4348,7 +4401,7 class SpectralFitting(Operation):
4348 4401 fmp=numpy.dot(LT,fm)
4349 4402 return dp-fmp
4350 4403
4351 def __getSNR(self, z, noise):
4404 def __getSNR(self, z, noise, norm=1):
4352 4405
4353 4406 avg = numpy.average(z, axis=1)
4354 4407 SNR = (avg.T-noise)/noise
@@ -4363,7 +4416,7 class SpectralFitting(Operation):
4363 4416 chisq=numpy.dot((dp-fmp).T,(dp-fmp))
4364 4417 return chisq
4365 4418
4366 class WindProfiler_V0(Operation):
4419 class WindProfiler(Operation):
4367 4420
4368 4421 __isConfig = False
4369 4422
@@ -4423,7 +4476,6 class WindProfiler_V0(Operation):
4423 4476 minid = listPhi.index(min(listPhi))
4424 4477
4425 4478 rango = list(range(len(phi)))
4426 # rango = numpy.delete(rango,maxid)
4427 4479
4428 4480 heiRang1 = heiRang*math.cos(phi[maxid])
4429 4481 heiRangAux = heiRang*math.cos(phi[minid])
@@ -4453,18 +4505,12 class WindProfiler_V0(Operation):
4453 4505 def __calculateVelUVW(self, A, velRadial):
4454 4506
4455 4507 #Operacion Matricial
4456 # velUVW = numpy.zeros((velRadial.shape[1],3))
4457 # for ind in range(velRadial.shape[1]):
4458 # velUVW[ind,:] = numpy.dot(A,velRadial[:,ind])
4459 # velUVW = velUVW.transpose()
4460 4508 velUVW = numpy.zeros((A.shape[0],velRadial.shape[1]))
4461 4509 velUVW[:,:] = numpy.dot(A,velRadial)
4462 4510
4463 4511
4464 4512 return velUVW
4465 4513
4466 # def techniqueDBS(self, velRadial0, dirCosx, disrCosy, azimuth, correct, horizontalOnly, heiRang, SNR0):
4467
4468 4514 def techniqueDBS(self, kwargs):
4469 4515 """
4470 4516 Function that implements Doppler Beam Swinging (DBS) technique.
@@ -4541,17 +4587,6 class WindProfiler_V0(Operation):
4541 4587
4542 4588 return distx, disty, dist, ang
4543 4589 #Calculo de Matrices
4544 # nPairs = len(pairs)
4545 # ang1 = numpy.zeros((nPairs, 2, 1))
4546 # dist1 = numpy.zeros((nPairs, 2, 1))
4547 #
4548 # for j in range(nPairs):
4549 # dist1[j,0,0] = dist[pairs[j][0]]
4550 # dist1[j,1,0] = dist[pairs[j][1]]
4551 # ang1[j,0,0] = ang[pairs[j][0]]
4552 # ang1[j,1,0] = ang[pairs[j][1]]
4553 #
4554 # return distx,disty, dist1,ang1
4555 4590
4556 4591
4557 4592 def __calculateVelVer(self, phase, lagTRange, _lambda):
@@ -4579,26 +4614,6 class WindProfiler_V0(Operation):
4579 4614
4580 4615 return vel
4581 4616
4582 # def __getPairsAutoCorr(self, pairsList, nChannels):
4583 #
4584 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
4585 #
4586 # for l in range(len(pairsList)):
4587 # firstChannel = pairsList[l][0]
4588 # secondChannel = pairsList[l][1]
4589 #
4590 # #Obteniendo pares de Autocorrelacion
4591 # if firstChannel == secondChannel:
4592 # pairsAutoCorr[firstChannel] = int(l)
4593 #
4594 # pairsAutoCorr = pairsAutoCorr.astype(int)
4595 #
4596 # pairsCrossCorr = range(len(pairsList))
4597 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
4598 #
4599 # return pairsAutoCorr, pairsCrossCorr
4600
4601 # def techniqueSA(self, pairsSelected, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, lagTRange, correctFactor):
4602 4617 def techniqueSA(self, kwargs):
4603 4618
4604 4619 """
@@ -4626,22 +4641,10 class WindProfiler_V0(Operation):
4626 4641 _lambda = kwargs['_lambda']
4627 4642
4628 4643 #Cross Correlation pairs obtained
4629 # pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairssList, nChannels)
4630 # pairsArray = numpy.array(pairsList)[pairsCrossCorr]
4631 # pairsSelArray = numpy.array(pairsSelected)
4632 # pairs = []
4633 #
4634 # #Wind estimation pairs obtained
4635 # for i in range(pairsSelArray.shape[0]/2):
4636 # ind1 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i], axis = 1))[0][0]
4637 # ind2 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i + 1], axis = 1))[0][0]
4638 # pairs.append((ind1,ind2))
4639 4644
4640 4645 indtau = tau.shape[0]/2
4641 4646 tau1 = tau[:indtau,:]
4642 4647 tau2 = tau[indtau:-1,:]
4643 # tau1 = tau1[pairs,:]
4644 # tau2 = tau2[pairs,:]
4645 4648 phase1 = tau[-1,:]
4646 4649
4647 4650 #---------------------------------------------------------------------
@@ -4651,11 +4654,6 class WindProfiler_V0(Operation):
4651 4654 winds = stats.nanmean(winds, axis=0)
4652 4655 #---------------------------------------------------------------------
4653 4656 #Metodo General
4654 # distx, disty, dist = self.calculateDistance(position_x,position_y,pairsCrossCorr, pairsList, azimuth)
4655 # #Calculo Coeficientes de Funcion de Correlacion
4656 # F,G,A,B,H = self.calculateCoef(tau1,tau2,distx,disty,n)
4657 # #Calculo de Velocidades
4658 # winds = self.calculateVelUV(F,G,A,B,H)
4659 4657
4660 4658 #---------------------------------------------------------------------
4661 4659 winds[2,:] = self.__calculateVelVer(phase1, lagTRange, _lambda)
@@ -5781,7 +5779,6 class EWDriftsEstimation(Operation):
5781 5779
5782 5780 rbufc=dataOut.data_paramC[:,:,0]
5783 5781 ebufc=dataOut.data_paramC[:,:,1]
5784 #SNR = dataOut.data_snr
5785 5782 SNR = dataOut.data_snr1_i
5786 5783 rbufi = dataOut.data_snr1_i
5787 5784 velRerr = dataOut.data_error[:,4,:]
@@ -7189,7 +7186,6 class SMOperations():
7189 7186
7190 7187 ph0_aux = ph0 + ph1
7191 7188 ph0_aux = numpy.angle(numpy.exp(1j*ph0_aux))
7192
7193 7189 #First Estimation
7194 7190 cosdir0[:,i] = (ph0_aux)/(2*numpy.pi*(d0 - d1))
7195 7191
@@ -7466,7 +7462,6 class MergeProc(ProcessingUnit):
7466 7462 #exit(1)
7467 7463 self.dataOut.NLAG = 16
7468 7464 self.dataOut.NLAG = self.dataOut.data_acf.shape[1]
7469
7470 7465 self.dataOut.NRANGE = self.dataOut.data_acf.shape[-1]
7471 7466
7472 7467 #print(numpy.shape(self.dataOut.data_spc))
@@ -7571,3 +7566,48 class MST_Den_Conv(Operation):
7571 7566 print("pow den shape", numpy.shape(dataOut.PowDen))
7572 7567 print("far den shape", numpy.shape(dataOut.FarDen))
7573 7568 return dataOut
7569
7570 class addTxPower(Operation):
7571 '''
7572 Transmited power level integrated in the dataOut ->AMISR
7573 resolution 1 min
7574 The power files have the pattern power_YYYYMMDD.csv
7575 '''
7576 __slots__ =('isConfig','dataDatetimes','txPowers')
7577 def __init__(self):
7578
7579 Operation.__init__(self)
7580 self.isConfig = False
7581 self.dataDatetimes = []
7582 self.txPowers = []
7583
7584 def setup(self, powerFile, dutyCycle):
7585 if not os.path.isfile(powerFile):
7586 raise schainpy.admin.SchainError('There is no file named :{}'.format(powerFile))
7587 return
7588
7589 with open(powerFile, newline='') as pfile:
7590 reader = csv.reader(pfile, delimiter=',', quotechar='|')
7591 next(reader)
7592 for row in reader:
7593 #'2022-10-25 00:00:00'
7594 self.dataDatetimes.append(datetime.datetime.strptime(row[0], "%Y-%m-%d %H:%M:%S"))
7595 self.txPowers.append(float(row[1])/dutyCycle)
7596 self.isConfig = True
7597
7598 def run(self, dataOut, path, DS=0.05):
7599
7600 #dataOut.flagNoData = True
7601
7602 if not(self.isConfig):
7603 self.setup(path, DS)
7604
7605 dataDate = datetime.datetime.utcfromtimestamp(dataOut.utctime).replace(second=0, microsecond=0)#no seconds
7606 try:
7607 indx = self.dataDatetimes.index(dataDate)
7608 dataOut.txPower = self.txPowers[indx]
7609 except:
7610 log.warning("No power available for the datetime {}, setting power to 0 w", self.name)
7611 dataOut.txPower = 0
7612
7613 return dataOut No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now