##// END OF EJS Templates
schain
RSS
remSatsProfiles
joabAM -
r1548:334daf8eec15
parent child
Show More
Side by Side Unified
Context file:
r1548:334daf8eec15 Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,1104 +1,1109
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Definition of diferent Data objects for different types of data
6 6
7 7 Here you will find the diferent data objects for the different types
8 8 of data, this data objects must be used as dataIn or dataOut objects in
9 9 processing units and operations. Currently the supported data objects are:
10 10 Voltage, Spectra, SpectraHeis, Fits, Correlation and Parameters
11 11 """
12 12
13 13 import copy
14 14 import numpy
15 15 import datetime
16 16 import json
17 17
18 18 import schainpy.admin
19 19 from schainpy.utils import log
20 20 from .jroheaderIO import SystemHeader, RadarControllerHeader
21 21 from schainpy.model.data import _noise
22 22
23 23
24 24 def getNumpyDtype(dataTypeCode):
25 25
26 26 if dataTypeCode == 0:
27 27 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
28 28 elif dataTypeCode == 1:
29 29 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
30 30 elif dataTypeCode == 2:
31 31 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
32 32 elif dataTypeCode == 3:
33 33 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
34 34 elif dataTypeCode == 4:
35 35 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
36 36 elif dataTypeCode == 5:
37 37 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
38 38 else:
39 39 raise ValueError('dataTypeCode was not defined')
40 40
41 41 return numpyDtype
42 42
43 43
44 44 def getDataTypeCode(numpyDtype):
45 45
46 46 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
47 47 datatype = 0
48 48 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
49 49 datatype = 1
50 50 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
51 51 datatype = 2
52 52 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
53 53 datatype = 3
54 54 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
55 55 datatype = 4
56 56 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
57 57 datatype = 5
58 58 else:
59 59 datatype = None
60 60
61 61 return datatype
62 62
63 63
64 64 def hildebrand_sekhon(data, navg):
65 65 """
66 66 This method is for the objective determination of the noise level in Doppler spectra. This
67 67 implementation technique is based on the fact that the standard deviation of the spectral
68 68 densities is equal to the mean spectral density for white Gaussian noise
69 69
70 70 Inputs:
71 71 Data : heights
72 72 navg : numbers of averages
73 73
74 74 Return:
75 75 mean : noise's level
76 76 """
77 77
78 78 sortdata = numpy.sort(data, axis=None)
79 79 '''
80 80 lenOfData = len(sortdata)
81 81 nums_min = lenOfData*0.5
82 82
83 83 if nums_min <= 5:
84 84
85 85 nums_min = 5
86 86
87 87 sump = 0.
88 88 sumq = 0.
89 89
90 90 j = 0
91 91 cont = 1
92 92
93 93 while((cont == 1)and(j < lenOfData)):
94 94
95 95 sump += sortdata[j]
96 96 sumq += sortdata[j]**2
97 97
98 98 if j > nums_min:
99 99 rtest = float(j)/(j-1) + 1.0/navg
100 100 if ((sumq*j) > (rtest*sump**2)):
101 101 j = j - 1
102 102 sump = sump - sortdata[j]
103 103 sumq = sumq - sortdata[j]**2
104 104 cont = 0
105 105
106 106 j += 1
107 107
108 108 lnoise = sump / j
109 109 return lnoise
110 110 '''
111 111 return _noise.hildebrand_sekhon(sortdata, navg)
112 112
113 113
114 114 class Beam:
115 115
116 116 def __init__(self):
117 117 self.codeList = []
118 118 self.azimuthList = []
119 119 self.zenithList = []
120 120
121 121
122 122
123 123 class GenericData(object):
124 124
125 125 flagNoData = True
126 126
127 127 def copy(self, inputObj=None):
128 128
129 129 if inputObj == None:
130 130 return copy.deepcopy(self)
131 131
132 132 for key in list(inputObj.__dict__.keys()):
133 133
134 134 attribute = inputObj.__dict__[key]
135 135
136 136 # If this attribute is a tuple or list
137 137 if type(inputObj.__dict__[key]) in (tuple, list):
138 138 self.__dict__[key] = attribute[:]
139 139 continue
140 140
141 141 # If this attribute is another object or instance
142 142 if hasattr(attribute, '__dict__'):
143 143 self.__dict__[key] = attribute.copy()
144 144 continue
145 145
146 146 self.__dict__[key] = inputObj.__dict__[key]
147 147
148 148 def deepcopy(self):
149 149
150 150 return copy.deepcopy(self)
151 151
152 152 def isEmpty(self):
153 153
154 154 return self.flagNoData
155 155
156 156 def isReady(self):
157 157
158 158 return not self.flagNoData
159 159
160 160
161 161 class JROData(GenericData):
162 162
163 163 useInputBuffer = False
164 164 buffer_empty = True
165 165
166 166 systemHeaderObj = SystemHeader()
167 167 radarControllerHeaderObj = RadarControllerHeader()
168 168 type = None
169 169 datatype = None # dtype but in string
170 170 nProfiles = None
171 171 heightList = None
172 172 channelList = None
173 173 flagDiscontinuousBlock = False
174 174 useLocalTime = False
175 175 utctime = None
176 176 timeZone = None
177 177 dstFlag = None
178 178 errorCount = None
179 179 blocksize = None
180 180 flagDecodeData = False # asumo q la data no esta decodificada
181 181 flagDeflipData = False # asumo q la data no esta sin flip
182 182 flagShiftFFT = False
183 183 nCohInt = None
184 184 windowOfFilter = 1
185 185 C = 3e8
186 186 frequency = 49.92e6
187 187 realtime = False
188 188 beacon_heiIndexList = None
189 189 last_block = None
190 190 blocknow = None
191 191 azimuth = None
192 192 zenith = None
193 193 beam = Beam()
194 194 profileIndex = None
195 195 error = None
196 196 data = None
197 197 nmodes = None
198 198 metadata_list = ['heightList', 'timeZone', 'type']
199 199 codeList = []
200 200 azimuthList = []
201 201 elevationList = []
202 202 last_noise = None
203 203
204 204 def __str__(self):
205 205
206 206 return '{} - {}'.format(self.type, self.datatime())
207 207
208 208 def getNoise(self):
209 209
210 210 raise NotImplementedError
211 211
212 212 @property
213 213 def nChannels(self):
214 214
215 215 return len(self.channelList)
216 216
217 217 @property
218 218 def channelIndexList(self):
219 219
220 220 return list(range(self.nChannels))
221 221
222 222 @property
223 223 def nHeights(self):
224 224
225 225 return len(self.heightList)
226 226
227 227 def getDeltaH(self):
228 228
229 229 return self.heightList[1] - self.heightList[0]
230 230
231 231 @property
232 232 def ltctime(self):
233 233
234 234 if self.useLocalTime:
235 235 return self.utctime - self.timeZone * 60
236 236
237 237 return self.utctime
238 238
239 239 @property
240 240 def datatime(self):
241 241
242 242 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
243 243 return datatimeValue
244 244
245 245 def getTimeRange(self):
246 246
247 247 datatime = []
248 248
249 249 datatime.append(self.ltctime)
250 250 datatime.append(self.ltctime + self.timeInterval + 1)
251 251
252 252 datatime = numpy.array(datatime)
253 253
254 254 return datatime
255 255
256 256 def getFmaxTimeResponse(self):
257 257
258 258 period = (10**-6) * self.getDeltaH() / (0.15)
259 259
260 260 PRF = 1. / (period * self.nCohInt)
261 261
262 262 fmax = PRF
263 263
264 264 return fmax
265 265
266 266 def getFmax(self):
267 267 PRF = 1. / (self.ippSeconds * self.nCohInt)
268 268
269 269 fmax = PRF
270 270 return fmax
271 271
272 272 def getVmax(self):
273 273
274 274 _lambda = self.C / self.frequency
275 275
276 276 vmax = self.getFmax() * _lambda / 2
277 277
278 278 return vmax
279 279
280 280 @property
281 281 def ippSeconds(self):
282 282 '''
283 283 '''
284 284 return self.radarControllerHeaderObj.ippSeconds
285 285
286 286 @ippSeconds.setter
287 287 def ippSeconds(self, ippSeconds):
288 288 '''
289 289 '''
290 290 self.radarControllerHeaderObj.ippSeconds = ippSeconds
291 291
292 292 @property
293 293 def code(self):
294 294 '''
295 295 '''
296 296 return self.radarControllerHeaderObj.code
297 297
298 298 @code.setter
299 299 def code(self, code):
300 300 '''
301 301 '''
302 302 self.radarControllerHeaderObj.code = code
303 303
304 304 @property
305 305 def nCode(self):
306 306 '''
307 307 '''
308 308 return self.radarControllerHeaderObj.nCode
309 309
310 310 @nCode.setter
311 311 def nCode(self, ncode):
312 312 '''
313 313 '''
314 314 self.radarControllerHeaderObj.nCode = ncode
315 315
316 316 @property
317 317 def nBaud(self):
318 318 '''
319 319 '''
320 320 return self.radarControllerHeaderObj.nBaud
321 321
322 322 @nBaud.setter
323 323 def nBaud(self, nbaud):
324 324 '''
325 325 '''
326 326 self.radarControllerHeaderObj.nBaud = nbaud
327 327
328 328 @property
329 329 def ipp(self):
330 330 '''
331 331 '''
332 332 return self.radarControllerHeaderObj.ipp
333 333
334 334 @ipp.setter
335 335 def ipp(self, ipp):
336 336 '''
337 337 '''
338 338 self.radarControllerHeaderObj.ipp = ipp
339 339
340 340 @property
341 341 def metadata(self):
342 342 '''
343 343 '''
344 344
345 345 return {attr: getattr(self, attr) for attr in self.metadata_list}
346 346
347 347
348 348 class Voltage(JROData):
349 349
350 350 dataPP_POW = None
351 351 dataPP_DOP = None
352 352 dataPP_WIDTH = None
353 353 dataPP_SNR = None
354 354
355 355 def __init__(self):
356 356 '''
357 357 Constructor
358 358 '''
359 359
360 360 self.useLocalTime = True
361 361 self.radarControllerHeaderObj = RadarControllerHeader()
362 362 self.systemHeaderObj = SystemHeader()
363 363 self.type = "Voltage"
364 364 self.data = None
365 365 self.nProfiles = None
366 366 self.heightList = None
367 367 self.channelList = None
368 368 self.flagNoData = True
369 369 self.flagDiscontinuousBlock = False
370 370 self.utctime = None
371 371 self.timeZone = 0
372 372 self.dstFlag = None
373 373 self.errorCount = None
374 374 self.nCohInt = None
375 375 self.blocksize = None
376 376 self.flagCohInt = False
377 377 self.flagDecodeData = False # asumo q la data no esta decodificada
378 378 self.flagDeflipData = False # asumo q la data no esta sin flip
379 379 self.flagShiftFFT = False
380 380 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
381 381 self.profileIndex = 0
382 382 self.metadata_list = ['type', 'heightList', 'timeZone', 'nProfiles', 'channelList', 'nCohInt',
383 383 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp']
384 384
385 385 def getNoisebyHildebrand(self, channel=None, ymin_index=None, ymax_index=None):
386 386 """
387 387 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
388 388
389 389 Return:
390 390 noiselevel
391 391 """
392 392
393 393 if channel != None:
394 394 data = self.data[channel,ymin_index:ymax_index]
395 395 nChannels = 1
396 396 else:
397 397 data = self.data[:,ymin_index:ymax_index]
398 398 nChannels = self.nChannels
399 399
400 400 noise = numpy.zeros(nChannels)
401 401 power = data * numpy.conjugate(data)
402 402
403 403 for thisChannel in range(nChannels):
404 404 if nChannels == 1:
405 405 daux = power[:].real
406 406 else:
407 407 daux = power[thisChannel, :].real
408 408 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
409 409
410 410 return noise
411 411
412 412 def getNoise(self, type=1, channel=None,ymin_index=None, ymax_index=None):
413 413
414 414 if type == 1:
415 415 noise = self.getNoisebyHildebrand(channel,ymin_index, ymax_index)
416 416
417 417 return noise
418 418
419 419 def getPower(self, channel=None):
420 420
421 421 if channel != None:
422 422 data = self.data[channel]
423 423 else:
424 424 data = self.data
425 425
426 426 power = data * numpy.conjugate(data)
427 427 powerdB = 10 * numpy.log10(power.real)
428 428 powerdB = numpy.squeeze(powerdB)
429 429
430 430 return powerdB
431 431
432 432 @property
433 433 def timeInterval(self):
434 434
435 435 return self.ippSeconds * self.nCohInt
436 436
437 437 noise = property(getNoise, "I'm the 'nHeights' property.")
438 438
439 439
440 440 class Spectra(JROData):
441 441
442 442 data_outlier = 0
443 443
444 444 def __init__(self):
445 445 '''
446 446 Constructor
447 447 '''
448 448
449 449 self.data_dc = None
450 450 self.data_spc = None
451 451 self.data_cspc = None
452 452 self.useLocalTime = True
453 453 self.radarControllerHeaderObj = RadarControllerHeader()
454 454 self.systemHeaderObj = SystemHeader()
455 455 self.type = "Spectra"
456 456 self.timeZone = 0
457 457 self.nProfiles = None
458 458 self.heightList = None
459 459 self.channelList = None
460 460 self.pairsList = None
461 461 self.flagNoData = True
462 462 self.flagDiscontinuousBlock = False
463 463 self.utctime = None
464 464 self.nCohInt = None
465 465 self.nIncohInt = None
466 466 self.blocksize = None
467 467 self.nFFTPoints = None
468 468 self.wavelength = None
469 469 self.flagDecodeData = False # asumo q la data no esta decodificada
470 470 self.flagDeflipData = False # asumo q la data no esta sin flip
471 471 self.flagShiftFFT = False
472 472 self.ippFactor = 1
473 473 self.beacon_heiIndexList = []
474 474 self.noise_estimation = None
475 475 self.codeList = []
476 476 self.azimuthList = []
477 477 self.elevationList = []
478 478 self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt',
479 479 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp','nIncohInt', 'nFFTPoints', 'nProfiles']
480 480
481 481
482 482 self.max_nIncohInt = 1
483 483
484 484
485 485 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
486 486 """
487 487 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
488 488
489 489 Return:
490 490 noiselevel
491 491 """
492 492 # if hasattr(self.nIncohInt, "__len__"): #nIncohInt is a matrix
493 493 #
494 494 # heis = self.data_spc.shape[2]
495 495 #
496 496 # noise = numpy.zeros((self.nChannels, heis))
497 497 # for hei in range(heis):
498 498 # for channel in range(self.nChannels):
499 499 # daux = self.data_spc[channel, xmin_index:xmax_index, hei]
500 500 #
501 501 # noise[channel,hei] = hildebrand_sekhon(daux, self.nIncohInt[channel,hei])
502 502 #
503 503 # else:
504 504 # noise = numpy.zeros(self.nChannels)
505 505 # for channel in range(self.nChannels):
506 506 # daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
507 507 #
508 508 # noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
509 509 noise = numpy.zeros(self.nChannels)
510 510 for channel in range(self.nChannels):
511 511 daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
512 512
513 513 noise[channel] = hildebrand_sekhon(daux, self.max_nIncohInt)
514 514
515 515 return noise
516 516
517 517 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
518 518
519 519 if self.noise_estimation is not None:
520 520 # this was estimated by getNoise Operation defined in jroproc_spectra.py
521 521 return self.noise_estimation
522 522 else:
523 523 noise = self.getNoisebyHildebrand(
524 524 xmin_index, xmax_index, ymin_index, ymax_index)
525 525 return noise
526 526
527 527 def getFreqRangeTimeResponse(self, extrapoints=0):
528 528
529 529 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
530 530 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
531 531
532 532 return freqrange
533 533
534 534 def getAcfRange(self, extrapoints=0):
535 535
536 536 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
537 537 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
538 538
539 539 return freqrange
540 540
541 541 def getFreqRange(self, extrapoints=0):
542 542
543 543 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
544 544 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
545 545
546 546 return freqrange
547 547
548 548 def getVelRange(self, extrapoints=0):
549 549
550 550 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
551 551 velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
552 552
553 553 if self.nmodes:
554 554 return velrange/self.nmodes
555 555 else:
556 556 return velrange
557 557
558 558 @property
559 559 def nPairs(self):
560 560
561 561 return len(self.pairsList)
562 562
563 563 @property
564 564 def pairsIndexList(self):
565 565
566 566 return list(range(self.nPairs))
567 567
568 568 @property
569 569 def normFactor(self):
570 570
571 571 pwcode = 1
572 572
573 573 if self.flagDecodeData:
574 574 pwcode = numpy.sum(self.code[0]**2)
575 575 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
576 576 normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
577 577
578 578
579 579 return normFactor
580 580
581 581 @property
582 582 def flag_cspc(self):
583 583
584 584 if self.data_cspc is None:
585 585 return True
586 586
587 587 return False
588 588
589 589 @property
590 590 def flag_dc(self):
591 591
592 592 if self.data_dc is None:
593 593 return True
594 594
595 595 return False
596 596
597 597 @property
598 598 def timeInterval(self):
599 599
600 600 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
601 601 if self.nmodes:
602 602 return self.nmodes*timeInterval
603 603 else:
604 604 return timeInterval
605 605
606 606 def getPower(self):
607 607
608 608 factor = self.normFactor
609 z = numpy.divide(self.data_spc,factor)
609 power = numpy.zeros( (self.nChannels,self.nHeights) )
610 for ch in range(self.nChannels):
611 if hasattr(factor,'shape'):
612 z = numpy.divide(self.data_spc[ch],factor[ch])
613 else:
614 z = numpy.divide(self.data_spc[ch],factor)
610 615 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
611 avg = numpy.average(z, axis=1)
612
613 return 10 * numpy.log10(avg)
616 avg = numpy.average(z, axis=0)
617 power[ch,:] = 10 * numpy.log10(avg)
618 return power
614 619
615 620 def getCoherence(self, pairsList=None, phase=False):
616 621
617 622 z = []
618 623 if pairsList is None:
619 624 pairsIndexList = self.pairsIndexList
620 625 else:
621 626 pairsIndexList = []
622 627 for pair in pairsList:
623 628 if pair not in self.pairsList:
624 629 raise ValueError("Pair %s is not in dataOut.pairsList" % (
625 630 pair))
626 631 pairsIndexList.append(self.pairsList.index(pair))
627 632 for i in range(len(pairsIndexList)):
628 633 pair = self.pairsList[pairsIndexList[i]]
629 634 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
630 635 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
631 636 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
632 637 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
633 638 if phase:
634 639 data = numpy.arctan2(avgcoherenceComplex.imag,
635 640 avgcoherenceComplex.real) * 180 / numpy.pi
636 641 else:
637 642 data = numpy.abs(avgcoherenceComplex)
638 643
639 644 z.append(data)
640 645
641 646 return numpy.array(z)
642 647
643 648 def setValue(self, value):
644 649
645 650 print("This property should not be initialized", value)
646 651
647 652 return
648 653
649 654 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
650 655
651 656
652 657 class SpectraHeis(Spectra):
653 658
654 659 def __init__(self):
655 660
656 661 self.radarControllerHeaderObj = RadarControllerHeader()
657 662 self.systemHeaderObj = SystemHeader()
658 663 self.type = "SpectraHeis"
659 664 self.nProfiles = None
660 665 self.heightList = None
661 666 self.channelList = None
662 667 self.flagNoData = True
663 668 self.flagDiscontinuousBlock = False
664 669 self.utctime = None
665 670 self.blocksize = None
666 671 self.profileIndex = 0
667 672 self.nCohInt = 1
668 673 self.nIncohInt = 1
669 674
670 675 @property
671 676 def normFactor(self):
672 677 pwcode = 1
673 678 if self.flagDecodeData:
674 679 pwcode = numpy.sum(self.code[0]**2)
675 680
676 681 normFactor = self.nIncohInt * self.nCohInt * pwcode
677 682
678 683 return normFactor
679 684
680 685 @property
681 686 def timeInterval(self):
682 687
683 688 return self.ippSeconds * self.nCohInt * self.nIncohInt
684 689
685 690
686 691 class Fits(JROData):
687 692
688 693 def __init__(self):
689 694
690 695 self.type = "Fits"
691 696 self.nProfiles = None
692 697 self.heightList = None
693 698 self.channelList = None
694 699 self.flagNoData = True
695 700 self.utctime = None
696 701 self.nCohInt = 1
697 702 self.nIncohInt = 1
698 703 self.useLocalTime = True
699 704 self.profileIndex = 0
700 705 self.timeZone = 0
701 706
702 707 def getTimeRange(self):
703 708
704 709 datatime = []
705 710
706 711 datatime.append(self.ltctime)
707 712 datatime.append(self.ltctime + self.timeInterval)
708 713
709 714 datatime = numpy.array(datatime)
710 715
711 716 return datatime
712 717
713 718 def getChannelIndexList(self):
714 719
715 720 return list(range(self.nChannels))
716 721
717 722 def getNoise(self, type=1):
718 723
719 724
720 725 if type == 1:
721 726 noise = self.getNoisebyHildebrand()
722 727
723 728 if type == 2:
724 729 noise = self.getNoisebySort()
725 730
726 731 if type == 3:
727 732 noise = self.getNoisebyWindow()
728 733
729 734 return noise
730 735
731 736 @property
732 737 def timeInterval(self):
733 738
734 739 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
735 740
736 741 return timeInterval
737 742
738 743 @property
739 744 def ippSeconds(self):
740 745 '''
741 746 '''
742 747 return self.ipp_sec
743 748
744 749 noise = property(getNoise, "I'm the 'nHeights' property.")
745 750
746 751
747 752 class Correlation(JROData):
748 753
749 754 def __init__(self):
750 755 '''
751 756 Constructor
752 757 '''
753 758 self.radarControllerHeaderObj = RadarControllerHeader()
754 759 self.systemHeaderObj = SystemHeader()
755 760 self.type = "Correlation"
756 761 self.data = None
757 762 self.dtype = None
758 763 self.nProfiles = None
759 764 self.heightList = None
760 765 self.channelList = None
761 766 self.flagNoData = True
762 767 self.flagDiscontinuousBlock = False
763 768 self.utctime = None
764 769 self.timeZone = 0
765 770 self.dstFlag = None
766 771 self.errorCount = None
767 772 self.blocksize = None
768 773 self.flagDecodeData = False # asumo q la data no esta decodificada
769 774 self.flagDeflipData = False # asumo q la data no esta sin flip
770 775 self.pairsList = None
771 776 self.nPoints = None
772 777
773 778 def getPairsList(self):
774 779
775 780 return self.pairsList
776 781
777 782 def getNoise(self, mode=2):
778 783
779 784 indR = numpy.where(self.lagR == 0)[0][0]
780 785 indT = numpy.where(self.lagT == 0)[0][0]
781 786
782 787 jspectra0 = self.data_corr[:, :, indR, :]
783 788 jspectra = copy.copy(jspectra0)
784 789
785 790 num_chan = jspectra.shape[0]
786 791 num_hei = jspectra.shape[2]
787 792
788 793 freq_dc = jspectra.shape[1] / 2
789 794 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
790 795
791 796 if ind_vel[0] < 0:
792 797 ind_vel[list(range(0, 1))] = ind_vel[list(
793 798 range(0, 1))] + self.num_prof
794 799
795 800 if mode == 1:
796 801 jspectra[:, freq_dc, :] = (
797 802 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
798 803
799 804 if mode == 2:
800 805
801 806 vel = numpy.array([-2, -1, 1, 2])
802 807 xx = numpy.zeros([4, 4])
803 808
804 809 for fil in range(4):
805 810 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
806 811
807 812 xx_inv = numpy.linalg.inv(xx)
808 813 xx_aux = xx_inv[0, :]
809 814
810 815 for ich in range(num_chan):
811 816 yy = jspectra[ich, ind_vel, :]
812 817 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
813 818
814 819 junkid = jspectra[ich, freq_dc, :] <= 0
815 820 cjunkid = sum(junkid)
816 821
817 822 if cjunkid.any():
818 823 jspectra[ich, freq_dc, junkid.nonzero()] = (
819 824 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
820 825
821 826 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
822 827
823 828 return noise
824 829
825 830 @property
826 831 def timeInterval(self):
827 832
828 833 return self.ippSeconds * self.nCohInt * self.nProfiles
829 834
830 835 def splitFunctions(self):
831 836
832 837 pairsList = self.pairsList
833 838 ccf_pairs = []
834 839 acf_pairs = []
835 840 ccf_ind = []
836 841 acf_ind = []
837 842 for l in range(len(pairsList)):
838 843 chan0 = pairsList[l][0]
839 844 chan1 = pairsList[l][1]
840 845
841 846 # Obteniendo pares de Autocorrelacion
842 847 if chan0 == chan1:
843 848 acf_pairs.append(chan0)
844 849 acf_ind.append(l)
845 850 else:
846 851 ccf_pairs.append(pairsList[l])
847 852 ccf_ind.append(l)
848 853
849 854 data_acf = self.data_cf[acf_ind]
850 855 data_ccf = self.data_cf[ccf_ind]
851 856
852 857 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
853 858
854 859 @property
855 860 def normFactor(self):
856 861 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
857 862 acf_pairs = numpy.array(acf_pairs)
858 863 normFactor = numpy.zeros((self.nPairs, self.nHeights))
859 864
860 865 for p in range(self.nPairs):
861 866 pair = self.pairsList[p]
862 867
863 868 ch0 = pair[0]
864 869 ch1 = pair[1]
865 870
866 871 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
867 872 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
868 873 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
869 874
870 875 return normFactor
871 876
872 877
873 878 class Parameters(Spectra):
874 879
875 880 groupList = None # List of Pairs, Groups, etc
876 881 data_param = None # Parameters obtained
877 882 data_pre = None # Data Pre Parametrization
878 883 data_SNR = None # Signal to Noise Ratio
879 884 data_outlier = None
880 885 abscissaList = None # Abscissa, can be velocities, lags or time
881 886 utctimeInit = None # Initial UTC time
882 887 paramInterval = None # Time interval to calculate Parameters in seconds
883 888 useLocalTime = True
884 889 # Fitting
885 890 data_error = None # Error of the estimation
886 891 constants = None
887 892 library = None
888 893 # Output signal
889 894 outputInterval = None # Time interval to calculate output signal in seconds
890 895 data_output = None # Out signal
891 896 nAvg = None
892 897 noise_estimation = None
893 898 GauSPC = None # Fit gaussian SPC
894 899 max_nIncohInt = 1
895 900 def __init__(self):
896 901 '''
897 902 Constructor
898 903 '''
899 904 self.radarControllerHeaderObj = RadarControllerHeader()
900 905 self.systemHeaderObj = SystemHeader()
901 906 self.type = "Parameters"
902 907 self.timeZone = 0
903 908
904 909 def getTimeRange1(self, interval):
905 910
906 911 datatime = []
907 912
908 913 if self.useLocalTime:
909 914 time1 = self.utctimeInit - self.timeZone * 60
910 915 else:
911 916 time1 = self.utctimeInit
912 917
913 918 datatime.append(time1)
914 919 datatime.append(time1 + interval)
915 920 datatime = numpy.array(datatime)
916 921
917 922 return datatime
918 923
919 924 @property
920 925 def timeInterval(self):
921 926
922 927 if hasattr(self, 'timeInterval1'):
923 928 return self.timeInterval1
924 929 else:
925 930 return self.paramInterval
926 931
927 932 def setValue(self, value):
928 933
929 934 print("This property should not be initialized")
930 935
931 936 return
932 937
933 938 def getNoise(self):
934 939
935 940 return self.spc_noise
936 941
937 942 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
938 943
939 944
940 945 class PlotterData(object):
941 946 '''
942 947 Object to hold data to be plotted
943 948 '''
944 949
945 950 MAXNUMX = 200
946 951 MAXNUMY = 200
947 952
948 953 def __init__(self, code, exp_code, localtime=True):
949 954
950 955 self.key = code
951 956 self.exp_code = exp_code
952 957 self.ready = False
953 958 self.flagNoData = False
954 959 self.localtime = localtime
955 960 self.data = {}
956 961 self.meta = {}
957 962 self.__heights = []
958 963
959 964 def __str__(self):
960 965 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
961 966 return 'Data[{}][{}]'.format(';'.join(dum), len(self.times))
962 967
963 968 def __len__(self):
964 969 return len(self.data)
965 970
966 971 def __getitem__(self, key):
967 972 if isinstance(key, int):
968 973 return self.data[self.times[key]]
969 974 elif isinstance(key, str):
970 975 ret = numpy.array([self.data[x][key] for x in self.times])
971 976 if ret.ndim > 1:
972 977 ret = numpy.swapaxes(ret, 0, 1)
973 978 return ret
974 979
975 980 def __contains__(self, key):
976 981 return key in self.data[self.min_time]
977 982
978 983 def setup(self):
979 984 '''
980 985 Configure object
981 986 '''
982 987 self.type = ''
983 988 self.ready = False
984 989 del self.data
985 990 self.data = {}
986 991 self.__heights = []
987 992 self.__all_heights = set()
988 993
989 994 def shape(self, key):
990 995 '''
991 996 Get the shape of the one-element data for the given key
992 997 '''
993 998
994 999 if len(self.data[self.min_time][key]):
995 1000 return self.data[self.min_time][key].shape
996 1001 return (0,)
997 1002
998 1003 def update(self, data, tm, meta={}):
999 1004 '''
1000 1005 Update data object with new dataOut
1001 1006 '''
1002 1007
1003 1008 self.data[tm] = data
1004 1009
1005 1010 for key, value in meta.items():
1006 1011 setattr(self, key, value)
1007 1012
1008 1013 def normalize_heights(self):
1009 1014 '''
1010 1015 Ensure same-dimension of the data for different heighList
1011 1016 '''
1012 1017
1013 1018 H = numpy.array(list(self.__all_heights))
1014 1019 H.sort()
1015 1020 for key in self.data:
1016 1021 shape = self.shape(key)[:-1] + H.shape
1017 1022 for tm, obj in list(self.data[key].items()):
1018 1023 h = self.__heights[self.times.tolist().index(tm)]
1019 1024 if H.size == h.size:
1020 1025 continue
1021 1026 index = numpy.where(numpy.in1d(H, h))[0]
1022 1027 dummy = numpy.zeros(shape) + numpy.nan
1023 1028 if len(shape) == 2:
1024 1029 dummy[:, index] = obj
1025 1030 else:
1026 1031 dummy[index] = obj
1027 1032 self.data[key][tm] = dummy
1028 1033
1029 1034 self.__heights = [H for tm in self.times]
1030 1035
1031 1036 def jsonify(self, tm, plot_name, plot_type, decimate=False):
1032 1037 '''
1033 1038 Convert data to json
1034 1039 '''
1035 1040
1036 1041 meta = {}
1037 1042 meta['xrange'] = []
1038 1043 dy = int(len(self.yrange)/self.MAXNUMY) + 1
1039 1044 tmp = self.data[tm][self.key]
1040 1045 shape = tmp.shape
1041 1046 if len(shape) == 2:
1042 1047 data = self.roundFloats(self.data[tm][self.key][::, ::dy].tolist())
1043 1048 elif len(shape) == 3:
1044 1049 dx = int(self.data[tm][self.key].shape[1]/self.MAXNUMX) + 1
1045 1050 data = self.roundFloats(
1046 1051 self.data[tm][self.key][::, ::dx, ::dy].tolist())
1047 1052 meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1048 1053 else:
1049 1054 data = self.roundFloats(self.data[tm][self.key].tolist())
1050 1055
1051 1056 ret = {
1052 1057 'plot': plot_name,
1053 1058 'code': self.exp_code,
1054 1059 'time': float(tm),
1055 1060 'data': data,
1056 1061 }
1057 1062 meta['type'] = plot_type
1058 1063 meta['interval'] = float(self.interval)
1059 1064 meta['localtime'] = self.localtime
1060 1065 meta['yrange'] = self.roundFloats(self.yrange[::dy].tolist())
1061 1066 meta.update(self.meta)
1062 1067 ret['metadata'] = meta
1063 1068 return json.dumps(ret)
1064 1069
1065 1070 @property
1066 1071 def times(self):
1067 1072 '''
1068 1073 Return the list of times of the current data
1069 1074 '''
1070 1075
1071 1076 ret = [t for t in self.data]
1072 1077 ret.sort()
1073 1078 return numpy.array(ret)
1074 1079
1075 1080 @property
1076 1081 def min_time(self):
1077 1082 '''
1078 1083 Return the minimun time value
1079 1084 '''
1080 1085
1081 1086 return self.times[0]
1082 1087
1083 1088 @property
1084 1089 def max_time(self):
1085 1090 '''
1086 1091 Return the maximun time value
1087 1092 '''
1088 1093
1089 1094 return self.times[-1]
1090 1095
1091 1096 # @property
1092 1097 # def heights(self):
1093 1098 # '''
1094 1099 # Return the list of heights of the current data
1095 1100 # '''
1096 1101
1097 1102 # return numpy.array(self.__heights[-1])
1098 1103
1099 1104 @staticmethod
1100 1105 def roundFloats(obj):
1101 1106 if isinstance(obj, list):
1102 1107 return list(map(PlotterData.roundFloats, obj))
1103 1108 elif isinstance(obj, float):
1104 1109 return round(obj, 2)
Show file before | Show file after | Hide comments
@@ -1,1194 +1,1194
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Classes to plot Spectra data
6 6
7 7 """
8 8
9 9 import os
10 10 import numpy
11 11
12 12 from schainpy.model.graphics.jroplot_base import Plot, plt, log
13 13 from itertools import combinations
14 14 from matplotlib.ticker import LinearLocator
15 15
16 16 class SpectraPlot(Plot):
17 17 '''
18 18 Plot for Spectra data
19 19 '''
20 20
21 21 CODE = 'spc'
22 22 colormap = 'jet'
23 23 plot_type = 'pcolor'
24 24 buffering = False
25 25 channelList = []
26 26 elevationList = []
27 27 azimuthList = []
28 28
29 29 def setup(self):
30 30
31 31 self.nplots = len(self.data.channels)
32 32 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
33 33 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
34 34 self.height = 3.4 * self.nrows
35 35
36 36 self.cb_label = 'dB'
37 37 if self.showprofile:
38 38 self.width = 5.2 * self.ncols
39 39 else:
40 40 self.width = 4.2* self.ncols
41 41 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.12})
42 42 self.ylabel = 'Range [km]'
43 43
44 44
45 45 def update_list(self,dataOut):
46 46 if len(self.channelList) == 0:
47 47 self.channelList = dataOut.channelList
48 48 if len(self.elevationList) == 0:
49 49 self.elevationList = dataOut.elevationList
50 50 if len(self.azimuthList) == 0:
51 51 self.azimuthList = dataOut.azimuthList
52 52
53 53 def update(self, dataOut):
54 54
55 55 self.update_list(dataOut)
56 56 data = {}
57 57 meta = {}
58 58 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
59 59 data['spc'] = spc
60 60 data['rti'] = dataOut.getPower()
61 61 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
62 62 noise = 10*numpy.log10(dataOut.getNoise()/float(norm))
63 63 data['noise'] = noise
64 64
65 65 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
66 66 if self.CODE == 'spc_moments':
67 67 data['moments'] = dataOut.moments
68 68
69 69 return data, meta
70 70
71 71 def plot(self):
72 72 if self.xaxis == "frequency":
73 73 x = self.data.xrange[0]
74 74 self.xlabel = "Frequency (kHz)"
75 75 elif self.xaxis == "time":
76 76 x = self.data.xrange[1]
77 77 self.xlabel = "Time (ms)"
78 78 else:
79 79 x = self.data.xrange[2]
80 80 self.xlabel = "Velocity (m/s)"
81 81
82 82 if self.CODE == 'spc_moments':
83 83 x = self.data.xrange[2]
84 84 self.xlabel = "Velocity (m/s)"
85 85
86 86 self.titles = []
87 87 y = self.data.yrange
88 88 self.y = y
89 89
90 90 data = self.data[-1]
91 91 z = data['spc']
92 92 #print(z.shape, x.shape, y.shape)
93 93 for n, ax in enumerate(self.axes):
94 94 noise = self.data['noise'][n][0]
95 95 #print(noise)
96 96 if self.CODE == 'spc_moments':
97 97 mean = data['moments'][n, 1]
98 98 if ax.firsttime:
99 99 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
100 100 self.xmin = self.xmin if self.xmin else -self.xmax
101 101 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
102 102 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
103 103 ax.plt = ax.pcolormesh(x, y, z[n].T,
104 104 vmin=self.zmin,
105 105 vmax=self.zmax,
106 106 cmap=plt.get_cmap(self.colormap)
107 107 )
108 108
109 109 if self.showprofile:
110 110 ax.plt_profile = self.pf_axes[n].plot(
111 111 data['rti'][n], y)[0]
112 112 # ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
113 113 # color="k", linestyle="dashed", lw=1)[0]
114 114 if self.CODE == 'spc_moments':
115 115 ax.plt_mean = ax.plot(mean, y, color='k')[0]
116 116 else:
117 117 ax.plt.set_array(z[n].T.ravel())
118 118 if self.showprofile:
119 119 ax.plt_profile.set_data(data['rti'][n], y)
120 120 #ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
121 121 if self.CODE == 'spc_moments':
122 122 ax.plt_mean.set_data(mean, y)
123 123 if len(self.azimuthList) > 0 and len(self.elevationList) > 0:
124 124 self.titles.append('CH {}: {:2.1f}elv {:2.1f}az {:3.2f}dB'.format(self.channelList[n], noise, self.elevationList[n], self.azimuthList[n]))
125 125 else:
126 126 self.titles.append('CH {}: {:3.2f}dB'.format(self.channelList[n], noise))
127 127
128 128
129 129 class CrossSpectraPlot(Plot):
130 130
131 131 CODE = 'cspc'
132 132 colormap = 'jet'
133 133 plot_type = 'pcolor'
134 134 zmin_coh = None
135 135 zmax_coh = None
136 136 zmin_phase = None
137 137 zmax_phase = None
138 138 realChannels = None
139 139 crossPairs = None
140 140
141 141 def setup(self):
142 142
143 143 self.ncols = 4
144 144 self.nplots = len(self.data.pairs) * 2
145 145 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
146 146 self.width = 3.1 * self.ncols
147 147 self.height = 2.6 * self.nrows
148 148 self.ylabel = 'Range [km]'
149 149 self.showprofile = False
150 150 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
151 151
152 152 def update(self, dataOut):
153 153
154 154 data = {}
155 155 meta = {}
156 156
157 157 spc = dataOut.data_spc
158 158 cspc = dataOut.data_cspc
159 159 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
160 160 rawPairs = list(combinations(list(range(dataOut.nChannels)), 2))
161 161 meta['pairs'] = rawPairs
162 162
163 163 if self.crossPairs == None:
164 164 self.crossPairs = dataOut.pairsList
165 165
166 166 tmp = []
167 167
168 168 for n, pair in enumerate(meta['pairs']):
169 169
170 170 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
171 171 coh = numpy.abs(out)
172 172 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
173 173 tmp.append(coh)
174 174 tmp.append(phase)
175 175
176 176 data['cspc'] = numpy.array(tmp)
177 177
178 178 return data, meta
179 179
180 180 def plot(self):
181 181
182 182 if self.xaxis == "frequency":
183 183 x = self.data.xrange[0]
184 184 self.xlabel = "Frequency (kHz)"
185 185 elif self.xaxis == "time":
186 186 x = self.data.xrange[1]
187 187 self.xlabel = "Time (ms)"
188 188 else:
189 189 x = self.data.xrange[2]
190 190 self.xlabel = "Velocity (m/s)"
191 191
192 192 self.titles = []
193 193
194 194 y = self.data.yrange
195 195 self.y = y
196 196
197 197 data = self.data[-1]
198 198 cspc = data['cspc']
199 199
200 200 for n in range(len(self.data.pairs)):
201 201
202 202 pair = self.crossPairs[n]
203 203
204 204 coh = cspc[n*2]
205 205 phase = cspc[n*2+1]
206 206 ax = self.axes[2 * n]
207 207
208 208 if ax.firsttime:
209 209 ax.plt = ax.pcolormesh(x, y, coh.T,
210 210 vmin=self.zmin_coh,
211 211 vmax=self.zmax_coh,
212 212 cmap=plt.get_cmap(self.colormap_coh)
213 213 )
214 214 else:
215 215 ax.plt.set_array(coh.T.ravel())
216 216 self.titles.append(
217 217 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
218 218
219 219 ax = self.axes[2 * n + 1]
220 220 if ax.firsttime:
221 221 ax.plt = ax.pcolormesh(x, y, phase.T,
222 222 vmin=-180,
223 223 vmax=180,
224 224 cmap=plt.get_cmap(self.colormap_phase)
225 225 )
226 226 else:
227 227 ax.plt.set_array(phase.T.ravel())
228 228
229 229 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
230 230
231 231
232 232 class RTIPlot(Plot):
233 233 '''
234 234 Plot for RTI data
235 235 '''
236 236
237 237 CODE = 'rti'
238 238 colormap = 'jet'
239 239 plot_type = 'pcolorbuffer'
240 240 titles = None
241 241 channelList = []
242 242 elevationList = []
243 243 azimuthList = []
244 244
245 245 def setup(self):
246 246 self.xaxis = 'time'
247 247 self.ncols = 1
248 248 #print("dataChannels ",self.data.channels)
249 249 self.nrows = len(self.data.channels)
250 250 self.nplots = len(self.data.channels)
251 251 self.ylabel = 'Range [km]'
252 252 self.xlabel = 'Time'
253 253 self.cb_label = 'dB'
254 254 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
255 255 self.titles = ['{} Channel {}'.format(
256 256 self.CODE.upper(), x) for x in range(self.nplots)]
257 257
258 258 def update_list(self,dataOut):
259 259
260 260 if len(self.channelList) == 0:
261 261 self.channelList = dataOut.channelList
262 262 if len(self.elevationList) == 0:
263 263 self.elevationList = dataOut.elevationList
264 264 if len(self.azimuthList) == 0:
265 265 self.azimuthList = dataOut.azimuthList
266 266
267 267
268 268 def update(self, dataOut):
269 269 if len(self.channelList) == 0:
270 270 self.update_list(dataOut)
271 271 data = {}
272 272 meta = {}
273 273 data['rti'] = dataOut.getPower()
274 274
275 275 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
276 276 noise = 10*numpy.log10(dataOut.getNoise()/float(norm))
277 277 data['noise'] = noise
278 278
279 279 return data, meta
280 280
281 281 def plot(self):
282 282
283 283 self.x = self.data.times
284 284 self.y = self.data.yrange
285 285 #print(" x, y: ",self.x, self.y)
286 286 self.z = self.data[self.CODE]
287 287 self.z = numpy.array(self.z, dtype=float)
288 288 self.z = numpy.ma.masked_invalid(self.z)
289 289
290 290 try:
291 291 if self.channelList != None:
292 292 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
293 293 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
294 294 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
295 295 else:
296 296 self.titles = ['{} Channel {}'.format(
297 297 self.CODE.upper(), x) for x in self.channelList]
298 298 except:
299 299 if self.channelList.any() != None:
300 300
301 301 self.titles = ['{} Channel {}'.format(
302 302 self.CODE.upper(), x) for x in self.channelList]
303 303
304 304 if self.decimation is None:
305 305 x, y, z = self.fill_gaps(self.x, self.y, self.z)
306 306 else:
307 307 x, y, z = self.fill_gaps(*self.decimate())
308 308
309 309 #dummy_var = self.axes #Extrañamente esto actualiza el valor axes
310 310 for n, ax in enumerate(self.axes):
311 311 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
312 312 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
313 313 data = self.data[-1]
314 314
315 315 if ax.firsttime:
316 316 ax.plt = ax.pcolormesh(x, y, z[n].T,
317 317 vmin=self.zmin,
318 318 vmax=self.zmax,
319 319 cmap=plt.get_cmap(self.colormap)
320 320 )
321 321 if self.showprofile:
322 322 ax.plot_profile = self.pf_axes[n].plot(data[self.CODE][n], self.y)[0]
323 323 if "noise" in self.data:
324 324
325 325 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
326 326 color="k", linestyle="dashed", lw=1)[0]
327 327 else:
328 328 ax.collections.remove(ax.collections[0])
329 329 ax.plt = ax.pcolormesh(x, y, z[n].T,
330 330 vmin=self.zmin,
331 331 vmax=self.zmax,
332 332 cmap=plt.get_cmap(self.colormap)
333 333 )
334 334 if self.showprofile:
335 335 ax.plot_profile.set_data(data[self.CODE][n], self.y)
336 336 if "noise" in self.data:
337 337 ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
338 338
339 339
340 340 class CoherencePlot(RTIPlot):
341 341 '''
342 342 Plot for Coherence data
343 343 '''
344 344
345 345 CODE = 'coh'
346 346
347 347 def setup(self):
348 348 self.xaxis = 'time'
349 349 self.ncols = 1
350 350 self.nrows = len(self.data.pairs)
351 351 self.nplots = len(self.data.pairs)
352 352 self.ylabel = 'Range [km]'
353 353 self.xlabel = 'Time'
354 354 self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1,'right':0.95})
355 355 if self.CODE == 'coh':
356 356 self.cb_label = ''
357 357 self.titles = [
358 358 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
359 359 else:
360 360 self.cb_label = 'Degrees'
361 361 self.titles = [
362 362 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
363 363
364 364 def update(self, dataOut):
365 365 self.update_list(dataOut)
366 366 data = {}
367 367 meta = {}
368 368 data['coh'] = dataOut.getCoherence()
369 369 meta['pairs'] = dataOut.pairsList
370 370
371 371
372 372 return data, meta
373 373
374 374 class PhasePlot(CoherencePlot):
375 375 '''
376 376 Plot for Phase map data
377 377 '''
378 378
379 379 CODE = 'phase'
380 380 colormap = 'seismic'
381 381
382 382 def update(self, dataOut):
383 383
384 384 data = {}
385 385 meta = {}
386 386 data['phase'] = dataOut.getCoherence(phase=True)
387 387 meta['pairs'] = dataOut.pairsList
388 388
389 389 return data, meta
390 390
391 391 class NoisePlot(Plot):
392 392 '''
393 393 Plot for noise
394 394 '''
395 395
396 396 CODE = 'noise'
397 397 plot_type = 'scatterbuffer'
398 398
399 399 def setup(self):
400 400 self.xaxis = 'time'
401 401 self.ncols = 1
402 402 self.nrows = 1
403 403 self.nplots = 1
404 404 self.ylabel = 'Intensity [dB]'
405 405 self.xlabel = 'Time'
406 406 self.titles = ['Noise']
407 407 self.colorbar = False
408 408 self.plots_adjust.update({'right': 0.85 })
409 409 #if not self.titles:
410 410 self.titles = ['Noise Plot']
411 411
412 412 def update(self, dataOut):
413 413
414 414 data = {}
415 415 meta = {}
416 416 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
417 #noise = 10*numpy.log10(dataOut.getNoise()/norm)
418 417 noise = 10*numpy.log10(dataOut.getNoise())
419 418 noise = noise.reshape(dataOut.nChannels, 1)
420 419 data['noise'] = noise
421 420 meta['yrange'] = numpy.array([])
422 421
423 422 return data, meta
424 423
425 424 def plot(self):
426 425
427 426 x = self.data.times
428 427 xmin = self.data.min_time
429 428 xmax = xmin + self.xrange * 60 * 60
430 429 Y = self.data['noise']
431 430
432 431 if self.axes[0].firsttime:
433 432 if self.ymin is None: self.ymin = numpy.nanmin(Y) - 5
434 433 if self.ymax is None: self.ymax = numpy.nanmax(Y) + 5
435 434 for ch in self.data.channels:
436 435 y = Y[ch]
437 436 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
438 437 plt.legend(bbox_to_anchor=(1.18, 1.0))
439 438 else:
440 439 for ch in self.data.channels:
441 440 y = Y[ch]
442 441 self.axes[0].lines[ch].set_data(x, y)
443 442
444 443
445 444 class PowerProfilePlot(Plot):
446 445
447 446 CODE = 'pow_profile'
448 447 plot_type = 'scatter'
449 448
450 449 def setup(self):
451 450
452 451 self.ncols = 1
453 452 self.nrows = 1
454 453 self.nplots = 1
455 454 self.height = 4
456 455 self.width = 3
457 456 self.ylabel = 'Range [km]'
458 457 self.xlabel = 'Intensity [dB]'
459 458 self.titles = ['Power Profile']
460 459 self.colorbar = False
461 460
462 461 def update(self, dataOut):
463 462
464 463 data = {}
465 464 meta = {}
466 465 data[self.CODE] = dataOut.getPower()
467 466
468 467 return data, meta
469 468
470 469 def plot(self):
471 470
472 471 y = self.data.yrange
473 472 self.y = y
474 473
475 474 x = self.data[-1][self.CODE]
476 475
477 476 if self.xmin is None: self.xmin = numpy.nanmin(x)*0.9
478 477 if self.xmax is None: self.xmax = numpy.nanmax(x)*1.1
479 478
480 479 if self.axes[0].firsttime:
481 480 for ch in self.data.channels:
482 481 self.axes[0].plot(x[ch], y, lw=1, label='Ch{}'.format(ch))
483 482 plt.legend()
484 483 else:
485 484 for ch in self.data.channels:
486 485 self.axes[0].lines[ch].set_data(x[ch], y)
487 486
488 487
489 488 class SpectraCutPlot(Plot):
490 489
491 490 CODE = 'spc_cut'
492 491 plot_type = 'scatter'
493 492 buffering = False
494 493 heights = []
495 494 channelList = []
496 495 maintitle = "Spectra Cuts"
497 496 flag_setIndex = False
498 497
499 498 def setup(self):
500 499
501 500 self.nplots = len(self.data.channels)
502 501 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
503 502 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
504 503 self.width = 4.5 * self.ncols + 2.5
505 504 self.height = 4.8 * self.nrows
506 505 self.ylabel = 'Power [dB]'
507 506 self.colorbar = False
508 507 self.plots_adjust.update({'left':0.1, 'hspace':0.3, 'right': 0.9, 'bottom':0.08})
509 508
510 509 if len(self.selectedHeightsList) > 0:
511 510 self.maintitle = "Spectra Cut"# for %d km " %(int(self.selectedHeight))
512 511
513 512
514 513
515 514 def update(self, dataOut):
516 515 if len(self.channelList) == 0:
517 516 self.channelList = dataOut.channelList
518 517
519 518 self.heights = dataOut.heightList
520 519 #print("sels: ",self.selectedHeightsList)
521 520 if len(self.selectedHeightsList)>0 and not self.flag_setIndex:
522 521
523 522 for sel_height in self.selectedHeightsList:
524 523 index_list = numpy.where(self.heights >= sel_height)
525 524 index_list = index_list[0]
526 525 self.height_index.append(index_list[0])
527 526 #print("sels i:"", self.height_index)
528 527 self.flag_setIndex = True
529 528 #print(self.height_index)
530 529 data = {}
531 530 meta = {}
532 531
533 532 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
534 533 n0 = 10*numpy.log10(dataOut.getNoise()/float(norm))
535 534
536 535 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
537 536 noise = numpy.repeat(n0,(dataOut.nFFTPoints*dataOut.nHeights)).reshape(dataOut.nChannels,dataOut.nFFTPoints,dataOut.nHeights)
538 537
539 538 data['spc'] = spc - noise
540 539 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
541 540
542 541 return data, meta
543 542
544 543 def plot(self):
545 544 if self.xaxis == "frequency":
546 545 x = self.data.xrange[0][1:]
547 546 self.xlabel = "Frequency (kHz)"
548 547 elif self.xaxis == "time":
549 548 x = self.data.xrange[1]
550 549 self.xlabel = "Time (ms)"
551 550 else:
552 551 x = self.data.xrange[2]
553 552 self.xlabel = "Velocity (m/s)"
554 553
555 554 self.titles = []
556 555
557 556 y = self.data.yrange
558 557 z = self.data[-1]['spc']
559 558 #print(z.shape)
560 559 if len(self.height_index) > 0:
561 560 index = self.height_index
562 561 else:
563 562 index = numpy.arange(0, len(y), int((len(y))/9))
564 563 #print("inde x ", index, self.axes)
565 564
566 565 for n, ax in enumerate(self.axes):
567 566
568 567 if ax.firsttime:
569 568
570 569
571 570 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
572 571 self.xmin = self.xmin if self.xmin else -self.xmax
573 572 self.ymin = self.ymin if self.ymin else numpy.nanmin(z)
574 573 self.ymax = self.ymax if self.ymax else numpy.nanmax(z)
575 574
576 575
577 576 ax.plt = ax.plot(x, z[n, :, index].T)
578 577 labels = ['Range = {:2.1f}km'.format(y[i]) for i in index]
579 578 self.figures[0].legend(ax.plt, labels, loc='center right', prop={'size': 8})
580 579 ax.minorticks_on()
581 580 ax.grid(which='major', axis='both')
582 581 ax.grid(which='minor', axis='x')
583 582 else:
584 583 for i, line in enumerate(ax.plt):
585 584 line.set_data(x, z[n, :, index[i]])
586 585
587 586
588 587 self.titles.append('CH {}'.format(self.channelList[n]))
589 588 plt.suptitle(self.maintitle, fontsize=10)
590 589
591 590
592 591 class BeaconPhase(Plot):
593 592
594 593 __isConfig = None
595 594 __nsubplots = None
596 595
597 596 PREFIX = 'beacon_phase'
598 597
599 598 def __init__(self):
600 599 Plot.__init__(self)
601 600 self.timerange = 24*60*60
602 601 self.isConfig = False
603 602 self.__nsubplots = 1
604 603 self.counter_imagwr = 0
605 604 self.WIDTH = 800
606 605 self.HEIGHT = 400
607 606 self.WIDTHPROF = 120
608 607 self.HEIGHTPROF = 0
609 608 self.xdata = None
610 609 self.ydata = None
611 610
612 611 self.PLOT_CODE = BEACON_CODE
613 612
614 613 self.FTP_WEI = None
615 614 self.EXP_CODE = None
616 615 self.SUB_EXP_CODE = None
617 616 self.PLOT_POS = None
618 617
619 618 self.filename_phase = None
620 619
621 620 self.figfile = None
622 621
623 622 self.xmin = None
624 623 self.xmax = None
625 624
626 625 def getSubplots(self):
627 626
628 627 ncol = 1
629 628 nrow = 1
630 629
631 630 return nrow, ncol
632 631
633 632 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
634 633
635 634 self.__showprofile = showprofile
636 635 self.nplots = nplots
637 636
638 637 ncolspan = 7
639 638 colspan = 6
640 639 self.__nsubplots = 2
641 640
642 641 self.createFigure(id = id,
643 642 wintitle = wintitle,
644 643 widthplot = self.WIDTH+self.WIDTHPROF,
645 644 heightplot = self.HEIGHT+self.HEIGHTPROF,
646 645 show=show)
647 646
648 647 nrow, ncol = self.getSubplots()
649 648
650 649 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
651 650
652 651 def save_phase(self, filename_phase):
653 652 f = open(filename_phase,'w+')
654 653 f.write('\n\n')
655 654 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
656 655 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
657 656 f.close()
658 657
659 658 def save_data(self, filename_phase, data, data_datetime):
660 659 f=open(filename_phase,'a')
661 660 timetuple_data = data_datetime.timetuple()
662 661 day = str(timetuple_data.tm_mday)
663 662 month = str(timetuple_data.tm_mon)
664 663 year = str(timetuple_data.tm_year)
665 664 hour = str(timetuple_data.tm_hour)
666 665 minute = str(timetuple_data.tm_min)
667 666 second = str(timetuple_data.tm_sec)
668 667 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
669 668 f.close()
670 669
671 670 def plot(self):
672 671 log.warning('TODO: Not yet implemented...')
673 672
674 673 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
675 674 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
676 675 timerange=None,
677 676 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
678 677 server=None, folder=None, username=None, password=None,
679 678 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
680 679
681 680 if dataOut.flagNoData:
682 681 return dataOut
683 682
684 683 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
685 684 return
686 685
687 686 if pairsList == None:
688 687 pairsIndexList = dataOut.pairsIndexList[:10]
689 688 else:
690 689 pairsIndexList = []
691 690 for pair in pairsList:
692 691 if pair not in dataOut.pairsList:
693 692 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
694 693 pairsIndexList.append(dataOut.pairsList.index(pair))
695 694
696 695 if pairsIndexList == []:
697 696 return
698 697
699 698 # if len(pairsIndexList) > 4:
700 699 # pairsIndexList = pairsIndexList[0:4]
701 700
702 701 hmin_index = None
703 702 hmax_index = None
704 703
705 704 if hmin != None and hmax != None:
706 705 indexes = numpy.arange(dataOut.nHeights)
707 706 hmin_list = indexes[dataOut.heightList >= hmin]
708 707 hmax_list = indexes[dataOut.heightList <= hmax]
709 708
710 709 if hmin_list.any():
711 710 hmin_index = hmin_list[0]
712 711
713 712 if hmax_list.any():
714 713 hmax_index = hmax_list[-1]+1
715 714
716 715 x = dataOut.getTimeRange()
717 716
718 717 thisDatetime = dataOut.datatime
719 718
720 719 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
721 720 xlabel = "Local Time"
722 721 ylabel = "Phase (degrees)"
723 722
724 723 update_figfile = False
725 724
726 725 nplots = len(pairsIndexList)
727 726 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
728 727 phase_beacon = numpy.zeros(len(pairsIndexList))
729 728 for i in range(nplots):
730 729 pair = dataOut.pairsList[pairsIndexList[i]]
731 730 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
732 731 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
733 732 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
734 733 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
735 734 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
736 735
737 736 if dataOut.beacon_heiIndexList:
738 737 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
739 738 else:
740 739 phase_beacon[i] = numpy.average(phase)
741 740
742 741 if not self.isConfig:
743 742
744 743 nplots = len(pairsIndexList)
745 744
746 745 self.setup(id=id,
747 746 nplots=nplots,
748 747 wintitle=wintitle,
749 748 showprofile=showprofile,
750 749 show=show)
751 750
752 751 if timerange != None:
753 752 self.timerange = timerange
754 753
755 754 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
756 755
757 756 if ymin == None: ymin = 0
758 757 if ymax == None: ymax = 360
759 758
760 759 self.FTP_WEI = ftp_wei
761 760 self.EXP_CODE = exp_code
762 761 self.SUB_EXP_CODE = sub_exp_code
763 762 self.PLOT_POS = plot_pos
764 763
765 764 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
766 765 self.isConfig = True
767 766 self.figfile = figfile
768 767 self.xdata = numpy.array([])
769 768 self.ydata = numpy.array([])
770 769
771 770 update_figfile = True
772 771
773 772 #open file beacon phase
774 773 path = '%s%03d' %(self.PREFIX, self.id)
775 774 beacon_file = os.path.join(path,'%s.txt'%self.name)
776 775 self.filename_phase = os.path.join(figpath,beacon_file)
777 776 #self.save_phase(self.filename_phase)
778 777
779 778
780 779 #store data beacon phase
781 780 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
782 781
783 782 self.setWinTitle(title)
784 783
785 784
786 785 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
787 786
788 787 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
789 788
790 789 axes = self.axesList[0]
791 790
792 791 self.xdata = numpy.hstack((self.xdata, x[0:1]))
793 792
794 793 if len(self.ydata)==0:
795 794 self.ydata = phase_beacon.reshape(-1,1)
796 795 else:
797 796 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
798 797
799 798
800 799 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
801 800 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
802 801 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
803 802 XAxisAsTime=True, grid='both'
804 803 )
805 804
806 805 self.draw()
807 806
808 807 if dataOut.ltctime >= self.xmax:
809 808 self.counter_imagwr = wr_period
810 809 self.isConfig = False
811 810 update_figfile = True
812 811
813 812 self.save(figpath=figpath,
814 813 figfile=figfile,
815 814 save=save,
816 815 ftp=ftp,
817 816 wr_period=wr_period,
818 817 thisDatetime=thisDatetime,
819 818 update_figfile=update_figfile)
820 819
821 820 return dataOut
822 821
823 822 class NoiselessSpectraPlot(Plot):
824 823 '''
825 824 Plot for Spectra data, subtracting
826 825 the noise in all channels, using for
827 826 amisr-14 data
828 827 '''
829 828
830 829 CODE = 'noiseless_spc'
831 830 colormap = 'jet'
832 831 plot_type = 'pcolor'
833 832 buffering = False
834 833 channelList = []
835 834 last_noise = None
836 835
837 836 def setup(self):
838 837
839 838 self.nplots = len(self.data.channels)
840 839 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
841 840 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
842 841 self.height = 3.5 * self.nrows
843 842
844 843 self.cb_label = 'dB'
845 844 if self.showprofile:
846 845 self.width = 5.8 * self.ncols
847 846 else:
848 847 self.width = 4.8* self.ncols
849 848 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.92, 'bottom': 0.12})
850 849
851 850 self.ylabel = 'Range [km]'
852 851
853 852
854 853 def update_list(self,dataOut):
855 854 if len(self.channelList) == 0:
856 855 self.channelList = dataOut.channelList
857 856
858 857 def update(self, dataOut):
859 858
860 859 self.update_list(dataOut)
861 860 data = {}
862 861 meta = {}
863 862
864 863 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
865 864 n0 = 10*numpy.log10(dataOut.getNoise()/float(norm))
866 865
867 866
868 867 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
869 868
870 869 noise = numpy.repeat(n0,dataOut.nHeights).reshape(dataOut.nChannels,dataOut.nHeights)
871 870 data['rti'] = dataOut.getPower() - noise
872 871
873 872 noise = numpy.repeat(n0,(dataOut.nFFTPoints*dataOut.nHeights)).reshape(dataOut.nChannels,dataOut.nFFTPoints,dataOut.nHeights)
874 873 data['spc'] = spc - noise
875 874
876 875
877 876 # data['noise'] = noise
878 877 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
879 878
880 879 return data, meta
881 880
882 881 def plot(self):
883 882 if self.xaxis == "frequency":
884 883 x = self.data.xrange[0]
885 884 self.xlabel = "Frequency (kHz)"
886 885 elif self.xaxis == "time":
887 886 x = self.data.xrange[1]
888 887 self.xlabel = "Time (ms)"
889 888 else:
890 889 x = self.data.xrange[2]
891 890 self.xlabel = "Velocity (m/s)"
892 891
893 892 self.titles = []
894 893 y = self.data.yrange
895 894 self.y = y
896 895
897 896 data = self.data[-1]
898 897 z = data['spc']
899 898
900 899 for n, ax in enumerate(self.axes):
901 900 #noise = data['noise'][n]
902 901
903 902 if ax.firsttime:
904 903 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
905 904 self.xmin = self.xmin if self.xmin else -self.xmax
906 905 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
907 906 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
908 907 ax.plt = ax.pcolormesh(x, y, z[n].T,
909 908 vmin=self.zmin,
910 909 vmax=self.zmax,
911 910 cmap=plt.get_cmap(self.colormap)
912 911 )
913 912
914 913 if self.showprofile:
915 914 ax.plt_profile = self.pf_axes[n].plot(
916 915 data['rti'][n], y)[0]
917 916
918 917
919 918 else:
920 919 ax.plt.set_array(z[n].T.ravel())
921 920 if self.showprofile:
922 921 ax.plt_profile.set_data(data['rti'][n], y)
923 922
924 923
925 924 self.titles.append('CH {}'.format(self.channelList[n]))
926 925
927 926
928 927 class NoiselessRTIPlot(Plot):
929 928 '''
930 929 Plot for RTI data
931 930 '''
932 931
933 932 CODE = 'noiseless_rti'
934 933 colormap = 'jet'
935 934 plot_type = 'pcolorbuffer'
936 935 titles = None
937 936 channelList = []
938 937 elevationList = []
939 938 azimuthList = []
940 939 last_noise = None
941 940
942 941 def setup(self):
943 942 self.xaxis = 'time'
944 943 self.ncols = 1
945 944 #print("dataChannels ",self.data.channels)
946 945 self.nrows = len(self.data.channels)
947 946 self.nplots = len(self.data.channels)
948 947 self.ylabel = 'Range [km]'
949 948 self.xlabel = 'Time'
950 949 self.cb_label = 'dB'
951 950 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
952 951 self.titles = ['{} Channel {}'.format(
953 952 self.CODE.upper(), x) for x in range(self.nplots)]
954 953
955 954 def update_list(self,dataOut):
956 955 if len(self.channelList) == 0:
957 956 self.channelList = dataOut.channelList
958 957 if len(self.elevationList) == 0:
959 958 self.elevationList = dataOut.elevationList
960 959 if len(self.azimuthList) == 0:
961 960 self.azimuthList = dataOut.azimuthList
962 961
963 962 def update(self, dataOut):
964 963 if len(self.channelList) == 0:
965 964 self.update_list(dataOut)
966 965 data = {}
967 966 meta = {}
968 967
969 968
970 969 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
971 970 #print("Norm: ", norm)
972 971 #print(dataOut.nProfiles , dataOut.max_nIncohInt ,dataOut.nCohInt, dataOut.windowOfFilter)
973 972 n0 = 10*numpy.log10(dataOut.getNoise()/float(norm))
974 973
975 974 data['noise'] = n0
976 975 noise = numpy.repeat(n0,dataOut.nHeights).reshape(dataOut.nChannels,dataOut.nHeights)
976
977 977 data['noiseless_rti'] = dataOut.getPower() - noise
978 978
979 979 return data, meta
980 980
981 981 def plot(self):
982 982
983 983 self.x = self.data.times
984 984 self.y = self.data.yrange
985 985 self.z = self.data['noiseless_rti']
986 986 self.z = numpy.array(self.z, dtype=float)
987 987 self.z = numpy.ma.masked_invalid(self.z)
988 988
989 989
990 990 try:
991 991 if self.channelList != None:
992 992 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
993 993 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
994 994 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
995 995 else:
996 996 self.titles = ['{} Channel {}'.format(
997 997 self.CODE.upper(), x) for x in self.channelList]
998 998 except:
999 999 if self.channelList.any() != None:
1000 1000
1001 1001 self.titles = ['{} Channel {}'.format(
1002 1002 self.CODE.upper(), x) for x in self.channelList]
1003 1003
1004 1004
1005 1005 if self.decimation is None:
1006 1006 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1007 1007 else:
1008 1008 x, y, z = self.fill_gaps(*self.decimate())
1009 1009 dummy_var = self.axes #Extrañamente esto actualiza el valor axes
1010 1010 #print("plot shapes ", z.shape, x.shape, y.shape)
1011 1011 for n, ax in enumerate(self.axes):
1012 1012
1013 1013
1014 1014 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
1015 1015 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
1016 1016 data = self.data[-1]
1017 1017 if ax.firsttime:
1018 1018 ax.plt = ax.pcolormesh(x, y, z[n].T,
1019 1019 vmin=self.zmin,
1020 1020 vmax=self.zmax,
1021 1021 cmap=plt.get_cmap(self.colormap)
1022 1022 )
1023 1023 if self.showprofile:
1024 1024 ax.plot_profile = self.pf_axes[n].plot(data['noiseless_rti'][n], self.y)[0]
1025 1025
1026 1026 else:
1027 1027 ax.collections.remove(ax.collections[0])
1028 1028 ax.plt = ax.pcolormesh(x, y, z[n].T,
1029 1029 vmin=self.zmin,
1030 1030 vmax=self.zmax,
1031 1031 cmap=plt.get_cmap(self.colormap)
1032 1032 )
1033 1033 if self.showprofile:
1034 1034 ax.plot_profile.set_data(data['noiseless_rti'][n], self.y)
1035 1035 # if "noise" in self.data:
1036 1036 # #ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
1037 1037 # ax.plot_noise.set_data(data['noise'][n], self.y)
1038 1038
1039 1039
1040 1040 class OutliersRTIPlot(Plot):
1041 1041 '''
1042 1042 Plot for data_xxxx object
1043 1043 '''
1044 1044
1045 1045 CODE = 'outlier_rtc' # Range Time Counts
1046 1046 colormap = 'cool'
1047 1047 plot_type = 'pcolorbuffer'
1048 1048
1049 1049 def setup(self):
1050 1050 self.xaxis = 'time'
1051 1051 self.ncols = 1
1052 1052 self.nrows = self.data.shape('outlier_rtc')[0]
1053 1053 self.nplots = self.nrows
1054 1054 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1055 1055
1056 1056
1057 1057 if not self.xlabel:
1058 1058 self.xlabel = 'Time'
1059 1059
1060 1060 self.ylabel = 'Height [km]'
1061 1061 if not self.titles:
1062 1062 self.titles = ['Outliers Ch:{}'.format(x) for x in range(self.nrows)]
1063 1063
1064 1064 def update(self, dataOut):
1065 1065
1066 1066 data = {}
1067 1067 data['outlier_rtc'] = dataOut.data_outlier
1068 1068
1069 1069 meta = {}
1070 1070
1071 1071 return data, meta
1072 1072
1073 1073 def plot(self):
1074 1074 # self.data.normalize_heights()
1075 1075 self.x = self.data.times
1076 1076 self.y = self.data.yrange
1077 1077 self.z = self.data['outlier_rtc']
1078 1078
1079 1079 #self.z = numpy.ma.masked_invalid(self.z)
1080 1080
1081 1081 if self.decimation is None:
1082 1082 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1083 1083 else:
1084 1084 x, y, z = self.fill_gaps(*self.decimate())
1085 1085
1086 1086 for n, ax in enumerate(self.axes):
1087 1087
1088 1088 self.zmax = self.zmax if self.zmax is not None else numpy.max(
1089 1089 self.z[n])
1090 1090 self.zmin = self.zmin if self.zmin is not None else numpy.min(
1091 1091 self.z[n])
1092 1092 data = self.data[-1]
1093 1093 if ax.firsttime:
1094 1094 if self.zlimits is not None:
1095 1095 self.zmin, self.zmax = self.zlimits[n]
1096 1096
1097 1097 ax.plt = ax.pcolormesh(x, y, z[n].T,
1098 1098 vmin=self.zmin,
1099 1099 vmax=self.zmax,
1100 1100 cmap=self.cmaps[n]
1101 1101 )
1102 1102 if self.showprofile:
1103 1103 ax.plot_profile = self.pf_axes[n].plot(data['outlier_rtc'][n], self.y)[0]
1104 1104 self.pf_axes[n].set_xlabel('')
1105 1105 else:
1106 1106 if self.zlimits is not None:
1107 1107 self.zmin, self.zmax = self.zlimits[n]
1108 1108 ax.collections.remove(ax.collections[0])
1109 1109 ax.plt = ax.pcolormesh(x, y, z[n].T ,
1110 1110 vmin=self.zmin,
1111 1111 vmax=self.zmax,
1112 1112 cmap=self.cmaps[n]
1113 1113 )
1114 1114 if self.showprofile:
1115 1115 ax.plot_profile.set_data(data['outlier_rtc'][n], self.y)
1116 1116 self.pf_axes[n].set_xlabel('')
1117 1117
1118 1118 class NIncohIntRTIPlot(Plot):
1119 1119 '''
1120 1120 Plot for data_xxxx object
1121 1121 '''
1122 1122
1123 1123 CODE = 'integrations_rtc' # Range Time Counts
1124 1124 colormap = 'BuGn'
1125 1125 plot_type = 'pcolorbuffer'
1126 1126
1127 1127 def setup(self):
1128 1128 self.xaxis = 'time'
1129 1129 self.ncols = 1
1130 1130 self.nrows = self.data.shape('integrations_rtc')[0]
1131 1131 self.nplots = self.nrows
1132 1132 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1133 1133
1134 1134
1135 1135 if not self.xlabel:
1136 1136 self.xlabel = 'Time'
1137 1137
1138 1138 self.ylabel = 'Height [km]'
1139 1139 if not self.titles:
1140 1140 self.titles = ['Integration Ch:{}'.format(x) for x in range(self.nrows)]
1141 1141
1142 1142 def update(self, dataOut):
1143 1143
1144 1144 data = {}
1145 1145 data['integrations_rtc'] = dataOut.nIncohInt
1146 1146
1147 1147 meta = {}
1148 1148
1149 1149 return data, meta
1150 1150
1151 1151 def plot(self):
1152 1152 # self.data.normalize_heights()
1153 1153 self.x = self.data.times
1154 1154 self.y = self.data.yrange
1155 1155 self.z = self.data['integrations_rtc']
1156 1156
1157 1157 #self.z = numpy.ma.masked_invalid(self.z)
1158 1158
1159 1159 if self.decimation is None:
1160 1160 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1161 1161 else:
1162 1162 x, y, z = self.fill_gaps(*self.decimate())
1163 1163
1164 1164 for n, ax in enumerate(self.axes):
1165 1165
1166 1166 self.zmax = self.zmax if self.zmax is not None else numpy.max(
1167 1167 self.z[n])
1168 1168 self.zmin = self.zmin if self.zmin is not None else numpy.min(
1169 1169 self.z[n])
1170 1170 data = self.data[-1]
1171 1171 if ax.firsttime:
1172 1172 if self.zlimits is not None:
1173 1173 self.zmin, self.zmax = self.zlimits[n]
1174 1174
1175 1175 ax.plt = ax.pcolormesh(x, y, z[n].T,
1176 1176 vmin=self.zmin,
1177 1177 vmax=self.zmax,
1178 1178 cmap=self.cmaps[n]
1179 1179 )
1180 1180 if self.showprofile:
1181 1181 ax.plot_profile = self.pf_axes[n].plot(data['integrations_rtc'][n], self.y)[0]
1182 1182 self.pf_axes[n].set_xlabel('')
1183 1183 else:
1184 1184 if self.zlimits is not None:
1185 1185 self.zmin, self.zmax = self.zlimits[n]
1186 1186 ax.collections.remove(ax.collections[0])
1187 1187 ax.plt = ax.pcolormesh(x, y, z[n].T ,
1188 1188 vmin=self.zmin,
1189 1189 vmax=self.zmax,
1190 1190 cmap=self.cmaps[n]
1191 1191 )
1192 1192 if self.showprofile:
1193 1193 ax.plot_profile.set_data(data['integrations_rtc'][n], self.y)
1194 1194 self.pf_axes[n].set_xlabel('')
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,2154 +1,2155
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Spectra processing Unit and operations
6 6
7 7 Here you will find the processing unit `SpectraProc` and several operations
8 8 to work with Spectra data type
9 9 """
10 10
11 11 import time
12 12 import itertools
13 13
14 14 import numpy
15 15 import math
16 16
17 17 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
18 18 from schainpy.model.data.jrodata import Spectra
19 19 from schainpy.model.data.jrodata import hildebrand_sekhon
20 20 from schainpy.model.data import _noise
21 21
22 22 from schainpy.utils import log
23 23 import matplotlib.pyplot as plt
24 24 #from scipy.optimize import curve_fit
25 25 from schainpy.model.io.utils import getHei_index
26 26
27 27 class SpectraProc(ProcessingUnit):
28 28
29 29 def __init__(self):
30 30
31 31 ProcessingUnit.__init__(self)
32 32
33 33 self.buffer = None
34 34 self.firstdatatime = None
35 35 self.profIndex = 0
36 36 self.dataOut = Spectra()
37 37 self.id_min = None
38 38 self.id_max = None
39 39 self.setupReq = False #Agregar a todas las unidades de proc
40 40
41 41 def __updateSpecFromVoltage(self):
42 42
43 43
44 44
45 45 self.dataOut.timeZone = self.dataIn.timeZone
46 46 self.dataOut.dstFlag = self.dataIn.dstFlag
47 47 self.dataOut.errorCount = self.dataIn.errorCount
48 48 self.dataOut.useLocalTime = self.dataIn.useLocalTime
49 49 try:
50 50 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
51 51 except:
52 52 pass
53 53 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
54 54 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
55 55 self.dataOut.channelList = self.dataIn.channelList
56 56 self.dataOut.heightList = self.dataIn.heightList
57 57 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
58 58 self.dataOut.nProfiles = self.dataOut.nFFTPoints
59 59 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
60 60 self.dataOut.utctime = self.firstdatatime
61 61 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
62 62 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
63 63 self.dataOut.flagShiftFFT = False
64 64 self.dataOut.nCohInt = self.dataIn.nCohInt
65 65 self.dataOut.nIncohInt = 1
66 66 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
67 67 self.dataOut.frequency = self.dataIn.frequency
68 68 self.dataOut.realtime = self.dataIn.realtime
69 69 self.dataOut.azimuth = self.dataIn.azimuth
70 70 self.dataOut.zenith = self.dataIn.zenith
71 71 self.dataOut.codeList = self.dataIn.codeList
72 72 self.dataOut.azimuthList = self.dataIn.azimuthList
73 73 self.dataOut.elevationList = self.dataIn.elevationList
74 74
75 75
76 76 def __getFft(self):
77 77 # print("fft donw")
78 78 """
79 79 Convierte valores de Voltaje a Spectra
80 80
81 81 Affected:
82 82 self.dataOut.data_spc
83 83 self.dataOut.data_cspc
84 84 self.dataOut.data_dc
85 85 self.dataOut.heightList
86 86 self.profIndex
87 87 self.buffer
88 88 self.dataOut.flagNoData
89 89 """
90 90 fft_volt = numpy.fft.fft(
91 91 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
92 92 fft_volt = fft_volt.astype(numpy.dtype('complex'))
93 93 dc = fft_volt[:, 0, :]
94 94
95 95 # calculo de self-spectra
96 96 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
97 97 spc = fft_volt * numpy.conjugate(fft_volt)
98 98 spc = spc.real
99 99
100 100 blocksize = 0
101 101 blocksize += dc.size
102 102 blocksize += spc.size
103 103
104 104 cspc = None
105 105 pairIndex = 0
106 106 if self.dataOut.pairsList != None:
107 107 # calculo de cross-spectra
108 108 cspc = numpy.zeros(
109 109 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
110 110 for pair in self.dataOut.pairsList:
111 111 if pair[0] not in self.dataOut.channelList:
112 112 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
113 113 str(pair), str(self.dataOut.channelList)))
114 114 if pair[1] not in self.dataOut.channelList:
115 115 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
116 116 str(pair), str(self.dataOut.channelList)))
117 117
118 118 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
119 119 numpy.conjugate(fft_volt[pair[1], :, :])
120 120 pairIndex += 1
121 121 blocksize += cspc.size
122 122
123 123 self.dataOut.data_spc = spc
124 124 self.dataOut.data_cspc = cspc
125 125 self.dataOut.data_dc = dc
126 126 self.dataOut.blockSize = blocksize
127 127 self.dataOut.flagShiftFFT = False
128 128
129 129 def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False, zeroPad=False):
130 130 #print("run spc proc")
131 131 try:
132 132 type = self.dataIn.type.decode("utf-8")
133 133 self.dataIn.type = type
134 134 except:
135 135 pass
136 136 if self.dataIn.type == "Spectra":
137 137
138 138 try:
139 139 self.dataOut.copy(self.dataIn)
140 140
141 141 except Exception as e:
142 142 print("Error dataIn ",e)
143 143
144 144 if shift_fft:
145 145 #desplaza a la derecha en el eje 2 determinadas posiciones
146 146 shift = int(self.dataOut.nFFTPoints/2)
147 147 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
148 148
149 149 if self.dataOut.data_cspc is not None:
150 150 #desplaza a la derecha en el eje 2 determinadas posiciones
151 151 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
152 152 if pairsList:
153 153 self.__selectPairs(pairsList)
154 154
155 155
156 156 elif self.dataIn.type == "Voltage":
157 157
158 158 self.dataOut.flagNoData = True
159 159
160 160 if nFFTPoints == None:
161 161 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
162 162
163 163 if nProfiles == None:
164 164 nProfiles = nFFTPoints
165 165
166 166 if ippFactor == None:
167 167 self.dataOut.ippFactor = 1
168 168
169 169 self.dataOut.nFFTPoints = nFFTPoints
170 170 #print(" volts ch,prof, h: ", self.dataIn.data.shape)
171 171 if self.buffer is None:
172 172 if not zeroPad:
173 173 self.buffer = numpy.zeros((self.dataIn.nChannels,
174 174 nProfiles,
175 175 self.dataIn.nHeights),
176 176 dtype='complex')
177 177 else:
178 178 self.buffer = numpy.zeros((self.dataIn.nChannels,
179 179 nFFTPoints,
180 180 self.dataIn.nHeights),
181 181 dtype='complex')
182 182
183 183 if self.dataIn.flagDataAsBlock:
184 184 nVoltProfiles = self.dataIn.data.shape[1]
185 185
186 186 if nVoltProfiles == nProfiles or zeroPad:
187 187 self.buffer = self.dataIn.data.copy()
188 188 self.profIndex = nVoltProfiles
189 189
190 190 elif nVoltProfiles < nProfiles:
191 191
192 192 if self.profIndex == 0:
193 193 self.id_min = 0
194 194 self.id_max = nVoltProfiles
195 195
196 196 self.buffer[:, self.id_min:self.id_max,
197 197 :] = self.dataIn.data
198 198 self.profIndex += nVoltProfiles
199 199 self.id_min += nVoltProfiles
200 200 self.id_max += nVoltProfiles
201 201 else:
202 202 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
203 203 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
204 204 self.dataOut.flagNoData = True
205 205 else:
206 206 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
207 207 self.profIndex += 1
208 208
209 209 if self.firstdatatime == None:
210 210 self.firstdatatime = self.dataIn.utctime
211 211
212 212 if self.profIndex == nProfiles or zeroPad:
213 213
214 214 self.__updateSpecFromVoltage()
215 215
216 216 if pairsList == None:
217 217 self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)]
218 218 else:
219 219 self.dataOut.pairsList = pairsList
220 220 self.__getFft()
221 221 self.dataOut.flagNoData = False
222 222 self.firstdatatime = None
223 223 self.profIndex = 0
224 224
225 225 elif self.dataIn.type == "Parameters":
226 226
227 227 self.dataOut.data_spc = self.dataIn.data_spc
228 228 self.dataOut.data_cspc = self.dataIn.data_cspc
229 229 self.dataOut.data_outlier = self.dataIn.data_outlier
230 230 self.dataOut.nProfiles = self.dataIn.nProfiles
231 231 self.dataOut.nIncohInt = self.dataIn.nIncohInt
232 232 self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
233 233 self.dataOut.ippFactor = self.dataIn.ippFactor
234 234 self.dataOut.max_nIncohInt = self.dataIn.max_nIncohInt
235 235 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
236 236 self.dataOut.ipp = self.dataIn.ipp
237 237 #self.dataOut.abscissaList = self.dataIn.getVelRange(1)
238 238 #self.dataOut.spc_noise = self.dataIn.getNoise()
239 239 #self.dataOut.spc_range = (self.dataIn.getFreqRange(1) , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1))
240 240 # self.dataOut.normFactor = self.dataIn.normFactor
241 241 if hasattr(self.dataIn, 'channelList'):
242 242 self.dataOut.channelList = self.dataIn.channelList
243 243 if hasattr(self.dataIn, 'pairsList'):
244 244 self.dataOut.pairsList = self.dataIn.pairsList
245 245 self.dataOut.groupList = self.dataIn.pairsList
246 246
247 247 self.dataOut.flagNoData = False
248 248
249 249 if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels
250 250 self.dataOut.ChanDist = self.dataIn.ChanDist
251 251 else: self.dataOut.ChanDist = None
252 252
253 253 #if hasattr(self.dataIn, 'VelRange'): #Velocities range
254 254 # self.dataOut.VelRange = self.dataIn.VelRange
255 255 #else: self.dataOut.VelRange = None
256 256
257 257
258 258
259 259 else:
260 260 raise ValueError("The type of input object {} is not valid".format(
261 261 self.dataIn.type))
262
262 #print("spc proc Done")
263 263
264 264 def __selectPairs(self, pairsList):
265 265
266 266 if not pairsList:
267 267 return
268 268
269 269 pairs = []
270 270 pairsIndex = []
271 271
272 272 for pair in pairsList:
273 273 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
274 274 continue
275 275 pairs.append(pair)
276 276 pairsIndex.append(pairs.index(pair))
277 277
278 278 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
279 279 self.dataOut.pairsList = pairs
280 280
281 281 return
282 282
283 283 def selectFFTs(self, minFFT, maxFFT ):
284 284 """
285 285 Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
286 286 minFFT<= FFT <= maxFFT
287 287 """
288 288
289 289 if (minFFT > maxFFT):
290 290 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
291 291
292 292 if (minFFT < self.dataOut.getFreqRange()[0]):
293 293 minFFT = self.dataOut.getFreqRange()[0]
294 294
295 295 if (maxFFT > self.dataOut.getFreqRange()[-1]):
296 296 maxFFT = self.dataOut.getFreqRange()[-1]
297 297
298 298 minIndex = 0
299 299 maxIndex = 0
300 300 FFTs = self.dataOut.getFreqRange()
301 301
302 302 inda = numpy.where(FFTs >= minFFT)
303 303 indb = numpy.where(FFTs <= maxFFT)
304 304
305 305 try:
306 306 minIndex = inda[0][0]
307 307 except:
308 308 minIndex = 0
309 309
310 310 try:
311 311 maxIndex = indb[0][-1]
312 312 except:
313 313 maxIndex = len(FFTs)
314 314
315 315 self.selectFFTsByIndex(minIndex, maxIndex)
316 316
317 317 return 1
318 318
319 319 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
320 320 newheis = numpy.where(
321 321 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
322 322
323 323 if hei_ref != None:
324 324 newheis = numpy.where(self.dataOut.heightList > hei_ref)
325 325
326 326 minIndex = min(newheis[0])
327 327 maxIndex = max(newheis[0])
328 328 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
329 329 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
330 330
331 331 # determina indices
332 332 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
333 333 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
334 334 avg_dB = 10 * \
335 335 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
336 336 beacon_dB = numpy.sort(avg_dB)[-nheis:]
337 337 beacon_heiIndexList = []
338 338 for val in avg_dB.tolist():
339 339 if val >= beacon_dB[0]:
340 340 beacon_heiIndexList.append(avg_dB.tolist().index(val))
341 341
342 342 #data_spc = data_spc[:,:,beacon_heiIndexList]
343 343 data_cspc = None
344 344 if self.dataOut.data_cspc is not None:
345 345 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
346 346 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
347 347
348 348 data_dc = None
349 349 if self.dataOut.data_dc is not None:
350 350 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
351 351 #data_dc = data_dc[:,beacon_heiIndexList]
352 352
353 353 self.dataOut.data_spc = data_spc
354 354 self.dataOut.data_cspc = data_cspc
355 355 self.dataOut.data_dc = data_dc
356 356 self.dataOut.heightList = heightList
357 357 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
358 358
359 359 return 1
360 360
361 361 def selectFFTsByIndex(self, minIndex, maxIndex):
362 362 """
363 363
364 364 """
365 365
366 366 if (minIndex < 0) or (minIndex > maxIndex):
367 367 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
368 368
369 369 if (maxIndex >= self.dataOut.nProfiles):
370 370 maxIndex = self.dataOut.nProfiles-1
371 371
372 372 #Spectra
373 373 data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
374 374
375 375 data_cspc = None
376 376 if self.dataOut.data_cspc is not None:
377 377 data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
378 378
379 379 data_dc = None
380 380 if self.dataOut.data_dc is not None:
381 381 data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
382 382
383 383 self.dataOut.data_spc = data_spc
384 384 self.dataOut.data_cspc = data_cspc
385 385 self.dataOut.data_dc = data_dc
386 386
387 387 self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
388 388 self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
389 389 self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
390 390
391 391 return 1
392 392
393 393 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
394 394 # validacion de rango
395 395 if minHei == None:
396 396 minHei = self.dataOut.heightList[0]
397 397
398 398 if maxHei == None:
399 399 maxHei = self.dataOut.heightList[-1]
400 400
401 401 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
402 402 print('minHei: %.2f is out of the heights range' % (minHei))
403 403 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
404 404 minHei = self.dataOut.heightList[0]
405 405
406 406 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
407 407 print('maxHei: %.2f is out of the heights range' % (maxHei))
408 408 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
409 409 maxHei = self.dataOut.heightList[-1]
410 410
411 411 # validacion de velocidades
412 412 velrange = self.dataOut.getVelRange(1)
413 413
414 414 if minVel == None:
415 415 minVel = velrange[0]
416 416
417 417 if maxVel == None:
418 418 maxVel = velrange[-1]
419 419
420 420 if (minVel < velrange[0]) or (minVel > maxVel):
421 421 print('minVel: %.2f is out of the velocity range' % (minVel))
422 422 print('minVel is setting to %.2f' % (velrange[0]))
423 423 minVel = velrange[0]
424 424
425 425 if (maxVel > velrange[-1]) or (maxVel < minVel):
426 426 print('maxVel: %.2f is out of the velocity range' % (maxVel))
427 427 print('maxVel is setting to %.2f' % (velrange[-1]))
428 428 maxVel = velrange[-1]
429 429
430 430 # seleccion de indices para rango
431 431 minIndex = 0
432 432 maxIndex = 0
433 433 heights = self.dataOut.heightList
434 434
435 435 inda = numpy.where(heights >= minHei)
436 436 indb = numpy.where(heights <= maxHei)
437 437
438 438 try:
439 439 minIndex = inda[0][0]
440 440 except:
441 441 minIndex = 0
442 442
443 443 try:
444 444 maxIndex = indb[0][-1]
445 445 except:
446 446 maxIndex = len(heights)
447 447
448 448 if (minIndex < 0) or (minIndex > maxIndex):
449 449 raise ValueError("some value in (%d,%d) is not valid" % (
450 450 minIndex, maxIndex))
451 451
452 452 if (maxIndex >= self.dataOut.nHeights):
453 453 maxIndex = self.dataOut.nHeights - 1
454 454
455 455 # seleccion de indices para velocidades
456 456 indminvel = numpy.where(velrange >= minVel)
457 457 indmaxvel = numpy.where(velrange <= maxVel)
458 458 try:
459 459 minIndexVel = indminvel[0][0]
460 460 except:
461 461 minIndexVel = 0
462 462
463 463 try:
464 464 maxIndexVel = indmaxvel[0][-1]
465 465 except:
466 466 maxIndexVel = len(velrange)
467 467
468 468 # seleccion del espectro
469 469 data_spc = self.dataOut.data_spc[:,
470 470 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
471 471 # estimacion de ruido
472 472 noise = numpy.zeros(self.dataOut.nChannels)
473 473
474 474 for channel in range(self.dataOut.nChannels):
475 475 daux = data_spc[channel, :, :]
476 476 sortdata = numpy.sort(daux, axis=None)
477 477 noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt)
478 478
479 479 self.dataOut.noise_estimation = noise.copy()
480 480
481 481 return 1
482 482
483 483 class removeDC(Operation):
484 484
485 485 def run(self, dataOut, mode=2):
486 486 self.dataOut = dataOut
487 487 jspectra = self.dataOut.data_spc
488 488 jcspectra = self.dataOut.data_cspc
489 489
490 490 num_chan = jspectra.shape[0]
491 491 num_hei = jspectra.shape[2]
492 492
493 493 if jcspectra is not None:
494 494 jcspectraExist = True
495 495 num_pairs = jcspectra.shape[0]
496 496 else:
497 497 jcspectraExist = False
498 498
499 499 freq_dc = int(jspectra.shape[1] / 2)
500 500 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
501 501 ind_vel = ind_vel.astype(int)
502 502
503 503 if ind_vel[0] < 0:
504 504 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
505 505
506 506 if mode == 1:
507 507 jspectra[:, freq_dc, :] = (
508 508 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
509 509
510 510 if jcspectraExist:
511 511 jcspectra[:, freq_dc, :] = (
512 512 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
513 513
514 514 if mode == 2:
515 515
516 516 vel = numpy.array([-2, -1, 1, 2])
517 517 xx = numpy.zeros([4, 4])
518 518
519 519 for fil in range(4):
520 520 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
521 521
522 522 xx_inv = numpy.linalg.inv(xx)
523 523 xx_aux = xx_inv[0, :]
524 524
525 525 for ich in range(num_chan):
526 526 yy = jspectra[ich, ind_vel, :]
527 527 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
528 528
529 529 junkid = jspectra[ich, freq_dc, :] <= 0
530 530 cjunkid = sum(junkid)
531 531
532 532 if cjunkid.any():
533 533 jspectra[ich, freq_dc, junkid.nonzero()] = (
534 534 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
535 535
536 536 if jcspectraExist:
537 537 for ip in range(num_pairs):
538 538 yy = jcspectra[ip, ind_vel, :]
539 539 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
540 540
541 541 self.dataOut.data_spc = jspectra
542 542 self.dataOut.data_cspc = jcspectra
543 543
544 544 return self.dataOut
545 545
546 546 class getNoiseB(Operation):
547 547
548 548 __slots__ =('offset','warnings', 'isConfig', 'minIndex','maxIndex','minIndexFFT','maxIndexFFT')
549 549 def __init__(self):
550 550
551 551 Operation.__init__(self)
552 552 self.isConfig = False
553 553
554 554 def setup(self, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
555 555
556 556 self.warnings = warnings
557 557 if minHei == None:
558 558 minHei = self.dataOut.heightList[0]
559 559
560 560 if maxHei == None:
561 561 maxHei = self.dataOut.heightList[-1]
562 562
563 563 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
564 564 if self.warnings:
565 565 print('minHei: %.2f is out of the heights range' % (minHei))
566 566 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
567 567 minHei = self.dataOut.heightList[0]
568 568
569 569 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
570 570 if self.warnings:
571 571 print('maxHei: %.2f is out of the heights range' % (maxHei))
572 572 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
573 573 maxHei = self.dataOut.heightList[-1]
574 574
575 575
576 576 #indices relativos a los puntos de fft, puede ser de acuerdo a velocidad o frecuencia
577 577 minIndexFFT = 0
578 578 maxIndexFFT = 0
579 579 # validacion de velocidades
580 580 indminPoint = None
581 581 indmaxPoint = None
582 582 if self.dataOut.type == 'Spectra':
583 583 if minVel == None and maxVel == None :
584 584
585 585 freqrange = self.dataOut.getFreqRange(1)
586 586
587 587 if minFreq == None:
588 588 minFreq = freqrange[0]
589 589
590 590 if maxFreq == None:
591 591 maxFreq = freqrange[-1]
592 592
593 593 if (minFreq < freqrange[0]) or (minFreq > maxFreq):
594 594 if self.warnings:
595 595 print('minFreq: %.2f is out of the frequency range' % (minFreq))
596 596 print('minFreq is setting to %.2f' % (freqrange[0]))
597 597 minFreq = freqrange[0]
598 598
599 599 if (maxFreq > freqrange[-1]) or (maxFreq < minFreq):
600 600 if self.warnings:
601 601 print('maxFreq: %.2f is out of the frequency range' % (maxFreq))
602 602 print('maxFreq is setting to %.2f' % (freqrange[-1]))
603 603 maxFreq = freqrange[-1]
604 604
605 605 indminPoint = numpy.where(freqrange >= minFreq)
606 606 indmaxPoint = numpy.where(freqrange <= maxFreq)
607 607
608 608 else:
609 609
610 610 velrange = self.dataOut.getVelRange(1)
611 611
612 612 if minVel == None:
613 613 minVel = velrange[0]
614 614
615 615 if maxVel == None:
616 616 maxVel = velrange[-1]
617 617
618 618 if (minVel < velrange[0]) or (minVel > maxVel):
619 619 if self.warnings:
620 620 print('minVel: %.2f is out of the velocity range' % (minVel))
621 621 print('minVel is setting to %.2f' % (velrange[0]))
622 622 minVel = velrange[0]
623 623
624 624 if (maxVel > velrange[-1]) or (maxVel < minVel):
625 625 if self.warnings:
626 626 print('maxVel: %.2f is out of the velocity range' % (maxVel))
627 627 print('maxVel is setting to %.2f' % (velrange[-1]))
628 628 maxVel = velrange[-1]
629 629
630 630 indminPoint = numpy.where(velrange >= minVel)
631 631 indmaxPoint = numpy.where(velrange <= maxVel)
632 632
633 633
634 634 # seleccion de indices para rango
635 635 minIndex = 0
636 636 maxIndex = 0
637 637 heights = self.dataOut.heightList
638 638
639 639 inda = numpy.where(heights >= minHei)
640 640 indb = numpy.where(heights <= maxHei)
641 641
642 642 try:
643 643 minIndex = inda[0][0]
644 644 except:
645 645 minIndex = 0
646 646
647 647 try:
648 648 maxIndex = indb[0][-1]
649 649 except:
650 650 maxIndex = len(heights)
651 651
652 652 if (minIndex < 0) or (minIndex > maxIndex):
653 653 raise ValueError("some value in (%d,%d) is not valid" % (
654 654 minIndex, maxIndex))
655 655
656 656 if (maxIndex >= self.dataOut.nHeights):
657 657 maxIndex = self.dataOut.nHeights - 1
658 658 #############################################################3
659 659 # seleccion de indices para velocidades
660 660 if self.dataOut.type == 'Spectra':
661 661 try:
662 662 minIndexFFT = indminPoint[0][0]
663 663 except:
664 664 minIndexFFT = 0
665 665
666 666 try:
667 667 maxIndexFFT = indmaxPoint[0][-1]
668 668 except:
669 669 maxIndexFFT = len( self.dataOut.getFreqRange(1))
670 670
671 671 self.minIndex, self.maxIndex, self.minIndexFFT, self.maxIndexFFT = minIndex, maxIndex, minIndexFFT, maxIndexFFT
672 672 self.isConfig = True
673 673 self.offset = 1
674 674 if offset!=None:
675 675 self.offset = 10**(offset/10)
676 676 #print("config getNoiseB Done")
677 677
678 678 def run(self, dataOut, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
679 679 self.dataOut = dataOut
680 680
681 681 if not self.isConfig:
682 682 self.setup(offset, minHei, maxHei,minVel, maxVel, minFreq, maxFreq, warnings)
683 683
684 684 self.dataOut.noise_estimation = None
685 685 noise = None
686 686 if self.dataOut.type == 'Voltage':
687 687 noise = self.dataOut.getNoise(ymin_index=self.minIndex, ymax_index=self.maxIndex)
688 688 #print(minIndex, maxIndex,minIndexVel, maxIndexVel)
689 689 elif self.dataOut.type == 'Spectra':
690 690 #print(self.minIndex, self.maxIndex,self.minIndexFFT, self.maxIndexFFT, self.dataOut.nIncohInt)
691 691 noise = numpy.zeros( self.dataOut.nChannels)
692 692 norm = 1
693 693
694 694 for channel in range( self.dataOut.nChannels):
695 695 if not hasattr(self.dataOut.nIncohInt,'__len__'):
696 696 norm = 1
697 697 else:
698 698 norm = self.dataOut.max_nIncohInt/self.dataOut.nIncohInt[channel, self.minIndex:self.maxIndex]
699 699 #print("norm nIncoh: ", norm ,self.dataOut.data_spc.shape)
700 700 daux = self.dataOut.data_spc[channel,self.minIndexFFT:self.maxIndexFFT, self.minIndex:self.maxIndex]
701 701 daux = numpy.multiply(daux, norm)
702 702 #print("offset: ", self.offset, 10*numpy.log10(self.offset))
703 703 # noise[channel] = self.getNoiseByMean(daux)/self.offset
704 704 #print(daux.shape, daux)
705 705 #noise[channel] = self.getNoiseByHS(daux, self.dataOut.max_nIncohInt)/self.offset
706 706 sortdata = numpy.sort(daux, axis=None)
707 707 noise[channel] = _noise.hildebrand_sekhon(sortdata, self.dataOut.max_nIncohInt)/self.offset
708 708 # data = numpy.mean(daux,axis=1)
709 709 # sortdata = numpy.sort(data, axis=None)
710 710 # noise[channel] = _noise.hildebrand_sekhon(sortdata, self.dataOut.max_nIncohInt)/self.offset
711 711
712 712 #noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex)
713 713 else:
714 714 noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex)
715 715 self.dataOut.noise_estimation = noise.copy() # dataOut.noise
716 716 #print("2: ",10*numpy.log10(self.dataOut.noise_estimation/64))
717
717 #print("2: ",self.dataOut.noise_estimation)
718 718 #print(self.dataOut.flagNoData)
719 719 #print("getNoise Done", noise)
720 720 return self.dataOut
721 721
722 722 def getNoiseByMean(self,data):
723 723 #data debe estar ordenado
724 724 data = numpy.mean(data,axis=1)
725 725 sortdata = numpy.sort(data, axis=None)
726 726 #sortID=data.argsort()
727 727 #print(data.shape)
728 728
729 729 pnoise = None
730 730 j = 0
731 731
732 732 mean = numpy.mean(sortdata)
733 733 min = numpy.min(sortdata)
734 734 delta = mean - min
735 735 indexes = numpy.where(sortdata > (mean+delta))[0] #only array of indexes
736 736 #print(len(indexes))
737 737 if len(indexes)==0:
738 738 pnoise = numpy.mean(sortdata)
739 739 else:
740 740 j = indexes[0]
741 741 pnoise = numpy.mean(sortdata[0:j])
742 742
743 743 # from matplotlib import pyplot as plt
744 744 # plt.plot(sortdata)
745 745 # plt.vlines(j,(pnoise-delta),(pnoise+delta), color='r')
746 746 # plt.show()
747 747 #print("noise: ", 10*numpy.log10(pnoise))
748 748 return pnoise
749 749
750 750 def getNoiseByHS(self,data, navg):
751 751 #data debe estar ordenado
752 752 #data = numpy.mean(data,axis=1)
753 753 sortdata = numpy.sort(data, axis=None)
754 754
755 755 lenOfData = len(sortdata)
756 756 nums_min = lenOfData*0.2
757 757
758 758 if nums_min <= 5:
759 759
760 760 nums_min = 5
761 761
762 762 sump = 0.
763 763 sumq = 0.
764 764
765 765 j = 0
766 766 cont = 1
767 767
768 768 while((cont == 1)and(j < lenOfData)):
769 769
770 770 sump += sortdata[j]
771 771 sumq += sortdata[j]**2
772 772 #sumq -= sump**2
773 773 if j > nums_min:
774 774 rtest = float(j)/(j-1) + 1.0/navg
775 775 #if ((sumq*j) > (sump**2)):
776 776 if ((sumq*j) > (rtest*sump**2)):
777 777 j = j - 1
778 778 sump = sump - sortdata[j]
779 779 sumq = sumq - sortdata[j]**2
780 780 cont = 0
781 781
782 782 j += 1
783 783
784 784 lnoise = sump / j
785 785
786 786 return lnoise
787 787
788 788
789 789
790 790 def fit_func( x, a0, a1, a2): #, a3, a4, a5):
791 791 z = (x - a1) / a2
792 792 y = a0 * numpy.exp(-z**2 / a2) #+ a3 + a4 * x + a5 * x**2
793 793 return y
794 794
795 795
796 796 class CleanRayleigh(Operation):
797 797
798 798 def __init__(self):
799 799
800 800 Operation.__init__(self)
801 801 self.i=0
802 802 self.isConfig = False
803 803 self.__dataReady = False
804 804 self.__profIndex = 0
805 805 self.byTime = False
806 806 self.byProfiles = False
807 807
808 808 self.bloques = None
809 809 self.bloque0 = None
810 810
811 811 self.index = 0
812 812
813 813 self.buffer = 0
814 814 self.buffer2 = 0
815 815 self.buffer3 = 0
816 816
817 817
818 818 def setup(self,dataOut,min_hei,max_hei,n, timeInterval,factor_stdv):
819 819
820 820 self.nChannels = dataOut.nChannels
821 821 self.nProf = dataOut.nProfiles
822 822 self.nPairs = dataOut.data_cspc.shape[0]
823 823 self.pairsArray = numpy.array(dataOut.pairsList)
824 824 self.spectra = dataOut.data_spc
825 825 self.cspectra = dataOut.data_cspc
826 826 self.heights = dataOut.heightList #alturas totales
827 827 self.nHeights = len(self.heights)
828 828 self.min_hei = min_hei
829 829 self.max_hei = max_hei
830 830 if (self.min_hei == None):
831 831 self.min_hei = 0
832 832 if (self.max_hei == None):
833 833 self.max_hei = dataOut.heightList[-1]
834 834 self.hval = ((self.max_hei>=self.heights) & (self.heights >= self.min_hei)).nonzero()
835 835 self.heightsClean = self.heights[self.hval] #alturas filtradas
836 836 self.hval = self.hval[0] # forma (N,), an solo N elementos -> Indices de alturas
837 837 self.nHeightsClean = len(self.heightsClean)
838 838 self.channels = dataOut.channelList
839 839 self.nChan = len(self.channels)
840 840 self.nIncohInt = dataOut.nIncohInt
841 841 self.__initime = dataOut.utctime
842 842 self.maxAltInd = self.hval[-1]+1
843 843 self.minAltInd = self.hval[0]
844 844
845 845 self.crosspairs = dataOut.pairsList
846 846 self.nPairs = len(self.crosspairs)
847 847 self.normFactor = dataOut.normFactor
848 848 self.nFFTPoints = dataOut.nFFTPoints
849 849 self.ippSeconds = dataOut.ippSeconds
850 850 self.currentTime = self.__initime
851 851 self.pairsArray = numpy.array(dataOut.pairsList)
852 852 self.factor_stdv = factor_stdv
853 853
854 854 if n != None :
855 855 self.byProfiles = True
856 856 self.nIntProfiles = n
857 857 else:
858 858 self.__integrationtime = timeInterval
859 859
860 860 self.__dataReady = False
861 861 self.isConfig = True
862 862
863 863
864 864
865 865 def run(self, dataOut,min_hei=None,max_hei=None, n=None, timeInterval=10,factor_stdv=2.5):
866 866 #print("runing cleanRayleigh")
867 867 if not self.isConfig :
868 868
869 869 self.setup(dataOut, min_hei,max_hei,n,timeInterval,factor_stdv)
870 870
871 871 tini=dataOut.utctime
872 872
873 873 if self.byProfiles:
874 874 if self.__profIndex == self.nIntProfiles:
875 875 self.__dataReady = True
876 876 else:
877 877 if (tini - self.__initime) >= self.__integrationtime:
878 878
879 879 self.__dataReady = True
880 880 self.__initime = tini
881 881
882 882 #if (tini.tm_min % 2) == 0 and (tini.tm_sec < 5 and self.fint==0):
883 883
884 884 if self.__dataReady:
885 885
886 886 self.__profIndex = 0
887 887 jspc = self.buffer
888 888 jcspc = self.buffer2
889 889 #jnoise = self.buffer3
890 890 self.buffer = dataOut.data_spc
891 891 self.buffer2 = dataOut.data_cspc
892 892 #self.buffer3 = dataOut.noise
893 893 self.currentTime = dataOut.utctime
894 894 if numpy.any(jspc) :
895 895 #print( jspc.shape, jcspc.shape)
896 896 jspc = numpy.reshape(jspc,(int(len(jspc)/self.nChannels),self.nChannels,self.nFFTPoints,self.nHeights))
897 897 try:
898 898 jcspc= numpy.reshape(jcspc,(int(len(jcspc)/self.nPairs),self.nPairs,self.nFFTPoints,self.nHeights))
899 899 except:
900 900 print("no cspc")
901 901 self.__dataReady = False
902 902 #print( jspc.shape, jcspc.shape)
903 903 dataOut.flagNoData = False
904 904 else:
905 905 dataOut.flagNoData = True
906 906 self.__dataReady = False
907 907 return dataOut
908 908 else:
909 909 #print( len(self.buffer))
910 910 if numpy.any(self.buffer):
911 911 self.buffer = numpy.concatenate((self.buffer,dataOut.data_spc), axis=0)
912 912 try:
913 913 self.buffer2 = numpy.concatenate((self.buffer2,dataOut.data_cspc), axis=0)
914 914 self.buffer3 += dataOut.data_dc
915 915 except:
916 916 pass
917 917 else:
918 918 self.buffer = dataOut.data_spc
919 919 self.buffer2 = dataOut.data_cspc
920 920 self.buffer3 = dataOut.data_dc
921 921 #print self.index, self.fint
922 922 #print self.buffer2.shape
923 923 dataOut.flagNoData = True ## NOTE: ?? revisar LUEGO
924 924 self.__profIndex += 1
925 925 return dataOut ## NOTE: REV
926 926
927 927
928 928 #index = tini.tm_hour*12+tini.tm_min/5
929 929 '''
930 930 #REVISAR
931 931 '''
932 932 # jspc = jspc/self.nFFTPoints/self.normFactor
933 933 # jcspc = jcspc/self.nFFTPoints/self.normFactor
934 934
935 935
936 936
937 937 tmp_spectra,tmp_cspectra = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
938 938 dataOut.data_spc = tmp_spectra
939 939 dataOut.data_cspc = tmp_cspectra
940 940
941 941 #dataOut.data_spc,dataOut.data_cspc = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
942 942
943 943 dataOut.data_dc = self.buffer3
944 944 dataOut.nIncohInt *= self.nIntProfiles
945 945 dataOut.max_nIncohInt = self.nIntProfiles
946 946 dataOut.utctime = self.currentTime #tiempo promediado
947 947 #print("Time: ",time.localtime(dataOut.utctime))
948 948 # dataOut.data_spc = sat_spectra
949 949 # dataOut.data_cspc = sat_cspectra
950 950 self.buffer = 0
951 951 self.buffer2 = 0
952 952 self.buffer3 = 0
953 953
954 954 return dataOut
955 955
956 956 def cleanRayleigh(self,dataOut,spectra,cspectra,factor_stdv):
957 957 print("OP cleanRayleigh")
958 958 #import matplotlib.pyplot as plt
959 959 #for k in range(149):
960 960 #channelsProcssd = []
961 961 #channelA_ok = False
962 962 #rfunc = cspectra.copy() #self.bloques
963 963 rfunc = spectra.copy()
964 964 #rfunc = cspectra
965 965 #val_spc = spectra*0.0 #self.bloque0*0.0
966 966 #val_cspc = cspectra*0.0 #self.bloques*0.0
967 967 #in_sat_spectra = spectra.copy() #self.bloque0
968 968 #in_sat_cspectra = cspectra.copy() #self.bloques
969 969
970 970
971 971 ###ONLY FOR TEST:
972 972 raxs = math.ceil(math.sqrt(self.nPairs))
973 973 if raxs == 0:
974 974 raxs = 1
975 975 caxs = math.ceil(self.nPairs/raxs)
976 976 if self.nPairs <4:
977 977 raxs = 2
978 978 caxs = 2
979 979 #print(raxs, caxs)
980 980 fft_rev = 14 #nFFT to plot
981 981 hei_rev = ((self.heights >= 550) & (self.heights <= 551)).nonzero() #hei to plot
982 982 hei_rev = hei_rev[0]
983 983 #print(hei_rev)
984 984
985 985 #print numpy.absolute(rfunc[:,0,0,14])
986 986
987 987 gauss_fit, covariance = None, None
988 988 for ih in range(self.minAltInd,self.maxAltInd):
989 989 for ifreq in range(self.nFFTPoints):
990 990 '''
991 991 ###ONLY FOR TEST:
992 992 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
993 993 fig, axs = plt.subplots(raxs, caxs)
994 994 fig2, axs2 = plt.subplots(raxs, caxs)
995 995 col_ax = 0
996 996 row_ax = 0
997 997 '''
998 998 #print(self.nPairs)
999 999 for ii in range(self.nChan): #PARES DE CANALES SELF y CROSS
1000 1000 # if self.crosspairs[ii][1]-self.crosspairs[ii][0] > 1: # APLICAR SOLO EN PARES CONTIGUOS
1001 1001 # continue
1002 1002 # if not self.crosspairs[ii][0] in channelsProcssd:
1003 1003 # channelA_ok = True
1004 1004 #print("pair: ",self.crosspairs[ii])
1005 1005 '''
1006 1006 ###ONLY FOR TEST:
1007 1007 if (col_ax%caxs==0 and col_ax!=0 and self.nPairs !=1):
1008 1008 col_ax = 0
1009 1009 row_ax += 1
1010 1010 '''
1011 1011 func2clean = 10*numpy.log10(numpy.absolute(rfunc[:,ii,ifreq,ih])) #Potencia?
1012 1012 #print(func2clean.shape)
1013 1013 val = (numpy.isfinite(func2clean)==True).nonzero()
1014 1014
1015 1015 if len(val)>0: #limitador
1016 1016 min_val = numpy.around(numpy.amin(func2clean)-2) #> (-40)
1017 1017 if min_val <= -40 :
1018 1018 min_val = -40
1019 1019 max_val = numpy.around(numpy.amax(func2clean)+2) #< 200
1020 1020 if max_val >= 200 :
1021 1021 max_val = 200
1022 1022 #print min_val, max_val
1023 1023 step = 1
1024 1024 #print("Getting bins and the histogram")
1025 1025 x_dist = min_val + numpy.arange(1 + ((max_val-(min_val))/step))*step
1026 1026 y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
1027 1027 #print(len(y_dist),len(binstep[:-1]))
1028 1028 #print(row_ax,col_ax, " ..")
1029 1029 #print(self.pairsArray[ii][0],self.pairsArray[ii][1])
1030 1030 mean = numpy.sum(x_dist * y_dist) / numpy.sum(y_dist)
1031 1031 sigma = numpy.sqrt(numpy.sum(y_dist * (x_dist - mean)**2) / numpy.sum(y_dist))
1032 1032 parg = [numpy.amax(y_dist),mean,sigma]
1033 1033
1034 1034 newY = None
1035 1035
1036 1036 try :
1037 1037 gauss_fit, covariance = curve_fit(fit_func, x_dist, y_dist,p0=parg)
1038 1038 mode = gauss_fit[1]
1039 1039 stdv = gauss_fit[2]
1040 1040 #print(" FIT OK",gauss_fit)
1041 1041 '''
1042 1042 ###ONLY FOR TEST:
1043 1043 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1044 1044 newY = fit_func(x_dist,gauss_fit[0],gauss_fit[1],gauss_fit[2])
1045 1045 axs[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
1046 1046 axs[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
1047 1047 axs[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
1048 1048 '''
1049 1049 except:
1050 1050 mode = mean
1051 1051 stdv = sigma
1052 1052 #print("FIT FAIL")
1053 1053 #continue
1054 1054
1055 1055
1056 1056 #print(mode,stdv)
1057 1057 #Removing echoes greater than mode + std_factor*stdv
1058 1058 noval = (abs(func2clean - mode)>=(factor_stdv*stdv)).nonzero()
1059 1059 #noval tiene los indices que se van a remover
1060 1060 #print("Chan ",ii," novals: ",len(noval[0]))
1061 1061 if len(noval[0]) > 0: #forma de array (N,) es igual a longitud (N)
1062 1062 novall = ((func2clean - mode) >= (factor_stdv*stdv)).nonzero()
1063 1063 #print(novall)
1064 1064 #print(" ",self.pairsArray[ii])
1065 1065 #cross_pairs = self.pairsArray[ii]
1066 1066 #Getting coherent echoes which are removed.
1067 1067 # if len(novall[0]) > 0:
1068 1068 #
1069 1069 # val_spc[novall[0],cross_pairs[0],ifreq,ih] = 1
1070 1070 # val_spc[novall[0],cross_pairs[1],ifreq,ih] = 1
1071 1071 # val_cspc[novall[0],ii,ifreq,ih] = 1
1072 1072 #print("OUT NOVALL 1")
1073 1073 try:
1074 1074 pair = (self.channels[ii],self.channels[ii + 1])
1075 1075 except:
1076 1076 pair = (99,99)
1077 1077 #print("par ", pair)
1078 1078 if ( pair in self.crosspairs):
1079 1079 q = self.crosspairs.index(pair)
1080 1080 #print("está aqui: ", q, (ii,ii + 1))
1081 1081 new_a = numpy.delete(cspectra[:,q,ifreq,ih], noval[0])
1082 1082 cspectra[noval,q,ifreq,ih] = numpy.mean(new_a) #mean CrossSpectra
1083 1083
1084 1084 #if channelA_ok:
1085 1085 #chA = self.channels.index(cross_pairs[0])
1086 1086 new_b = numpy.delete(spectra[:,ii,ifreq,ih], noval[0])
1087 1087 spectra[noval,ii,ifreq,ih] = numpy.mean(new_b) #mean Spectra Pair A
1088 1088 #channelA_ok = False
1089 1089
1090 1090 # chB = self.channels.index(cross_pairs[1])
1091 1091 # new_c = numpy.delete(spectra[:,chB,ifreq,ih], noval[0])
1092 1092 # spectra[noval,chB,ifreq,ih] = numpy.mean(new_c) #mean Spectra Pair B
1093 1093 #
1094 1094 # channelsProcssd.append(self.crosspairs[ii][0]) # save channel A
1095 1095 # channelsProcssd.append(self.crosspairs[ii][1]) # save channel B
1096 1096 '''
1097 1097 ###ONLY FOR TEST:
1098 1098 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1099 1099 func2clean = 10*numpy.log10(numpy.absolute(spectra[:,ii,ifreq,ih]))
1100 1100 y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
1101 1101 axs2[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
1102 1102 axs2[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
1103 1103 axs2[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
1104 1104 '''
1105 1105 '''
1106 1106 ###ONLY FOR TEST:
1107 1107 col_ax += 1 #contador de ploteo columnas
1108 1108 ##print(col_ax)
1109 1109 ###ONLY FOR TEST:
1110 1110 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1111 1111 title = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km"
1112 1112 title2 = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km CLEANED"
1113 1113 fig.suptitle(title)
1114 1114 fig2.suptitle(title2)
1115 1115 plt.show()
1116 1116 '''
1117 1117 ##################################################################################################
1118 1118
1119 1119 #print("Getting average of the spectra and cross-spectra from incoherent echoes.")
1120 1120 out_spectra = numpy.zeros([self.nChan,self.nFFTPoints,self.nHeights], dtype=float) #+numpy.nan
1121 1121 out_cspectra = numpy.zeros([self.nPairs,self.nFFTPoints,self.nHeights], dtype=complex) #+numpy.nan
1122 1122 for ih in range(self.nHeights):
1123 1123 for ifreq in range(self.nFFTPoints):
1124 1124 for ich in range(self.nChan):
1125 1125 tmp = spectra[:,ich,ifreq,ih]
1126 1126 valid = (numpy.isfinite(tmp[:])==True).nonzero()
1127 1127
1128 1128 if len(valid[0]) >0 :
1129 1129 out_spectra[ich,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
1130 1130
1131 1131 for icr in range(self.nPairs):
1132 1132 tmp = numpy.squeeze(cspectra[:,icr,ifreq,ih])
1133 1133 valid = (numpy.isfinite(tmp)==True).nonzero()
1134 1134 if len(valid[0]) > 0:
1135 1135 out_cspectra[icr,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
1136 1136
1137 1137 return out_spectra, out_cspectra
1138 1138
1139 1139 def REM_ISOLATED_POINTS(self,array,rth):
1140 1140 # import matplotlib.pyplot as plt
1141 1141 if rth == None :
1142 1142 rth = 4
1143 1143 #print("REM ISO")
1144 1144 num_prof = len(array[0,:,0])
1145 1145 num_hei = len(array[0,0,:])
1146 1146 n2d = len(array[:,0,0])
1147 1147
1148 1148 for ii in range(n2d) :
1149 1149 #print ii,n2d
1150 1150 tmp = array[ii,:,:]
1151 1151 #print tmp.shape, array[ii,101,:],array[ii,102,:]
1152 1152
1153 1153 # fig = plt.figure(figsize=(6,5))
1154 1154 # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1155 1155 # ax = fig.add_axes([left, bottom, width, height])
1156 1156 # x = range(num_prof)
1157 1157 # y = range(num_hei)
1158 1158 # cp = ax.contour(y,x,tmp)
1159 1159 # ax.clabel(cp, inline=True,fontsize=10)
1160 1160 # plt.show()
1161 1161
1162 1162 #indxs = WHERE(FINITE(tmp) AND tmp GT 0,cindxs)
1163 1163 tmp = numpy.reshape(tmp,num_prof*num_hei)
1164 1164 indxs1 = (numpy.isfinite(tmp)==True).nonzero()
1165 1165 indxs2 = (tmp > 0).nonzero()
1166 1166
1167 1167 indxs1 = (indxs1[0])
1168 1168 indxs2 = indxs2[0]
1169 1169 #indxs1 = numpy.array(indxs1[0])
1170 1170 #indxs2 = numpy.array(indxs2[0])
1171 1171 indxs = None
1172 1172 #print indxs1 , indxs2
1173 1173 for iv in range(len(indxs2)):
1174 1174 indv = numpy.array((indxs1 == indxs2[iv]).nonzero())
1175 1175 #print len(indxs2), indv
1176 1176 if len(indv[0]) > 0 :
1177 1177 indxs = numpy.concatenate((indxs,indxs2[iv]), axis=None)
1178 1178 # print indxs
1179 1179 indxs = indxs[1:]
1180 1180 #print(indxs, len(indxs))
1181 1181 if len(indxs) < 4 :
1182 1182 array[ii,:,:] = 0.
1183 1183 return
1184 1184
1185 1185 xpos = numpy.mod(indxs ,num_hei)
1186 1186 ypos = (indxs / num_hei)
1187 1187 sx = numpy.argsort(xpos) # Ordering respect to "x" (time)
1188 1188 #print sx
1189 1189 xpos = xpos[sx]
1190 1190 ypos = ypos[sx]
1191 1191
1192 1192 # *********************************** Cleaning isolated points **********************************
1193 1193 ic = 0
1194 1194 while True :
1195 1195 r = numpy.sqrt(list(numpy.power((xpos[ic]-xpos),2)+ numpy.power((ypos[ic]-ypos),2)))
1196 1196 #no_coh = WHERE(FINITE(r) AND (r LE rth),cno_coh)
1197 1197 #plt.plot(r)
1198 1198 #plt.show()
1199 1199 no_coh1 = (numpy.isfinite(r)==True).nonzero()
1200 1200 no_coh2 = (r <= rth).nonzero()
1201 1201 #print r, no_coh1, no_coh2
1202 1202 no_coh1 = numpy.array(no_coh1[0])
1203 1203 no_coh2 = numpy.array(no_coh2[0])
1204 1204 no_coh = None
1205 1205 #print valid1 , valid2
1206 1206 for iv in range(len(no_coh2)):
1207 1207 indv = numpy.array((no_coh1 == no_coh2[iv]).nonzero())
1208 1208 if len(indv[0]) > 0 :
1209 1209 no_coh = numpy.concatenate((no_coh,no_coh2[iv]), axis=None)
1210 1210 no_coh = no_coh[1:]
1211 1211 #print len(no_coh), no_coh
1212 1212 if len(no_coh) < 4 :
1213 1213 #print xpos[ic], ypos[ic], ic
1214 1214 # plt.plot(r)
1215 1215 # plt.show()
1216 1216 xpos[ic] = numpy.nan
1217 1217 ypos[ic] = numpy.nan
1218 1218
1219 1219 ic = ic + 1
1220 1220 if (ic == len(indxs)) :
1221 1221 break
1222 1222 #print( xpos, ypos)
1223 1223
1224 1224 indxs = (numpy.isfinite(list(xpos))==True).nonzero()
1225 1225 #print indxs[0]
1226 1226 if len(indxs[0]) < 4 :
1227 1227 array[ii,:,:] = 0.
1228 1228 return
1229 1229
1230 1230 xpos = xpos[indxs[0]]
1231 1231 ypos = ypos[indxs[0]]
1232 1232 for i in range(0,len(ypos)):
1233 1233 ypos[i]=int(ypos[i])
1234 1234 junk = tmp
1235 1235 tmp = junk*0.0
1236 1236
1237 1237 tmp[list(xpos + (ypos*num_hei))] = junk[list(xpos + (ypos*num_hei))]
1238 1238 array[ii,:,:] = numpy.reshape(tmp,(num_prof,num_hei))
1239 1239
1240 1240 #print array.shape
1241 1241 #tmp = numpy.reshape(tmp,(num_prof,num_hei))
1242 1242 #print tmp.shape
1243 1243
1244 1244 # fig = plt.figure(figsize=(6,5))
1245 1245 # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1246 1246 # ax = fig.add_axes([left, bottom, width, height])
1247 1247 # x = range(num_prof)
1248 1248 # y = range(num_hei)
1249 1249 # cp = ax.contour(y,x,array[ii,:,:])
1250 1250 # ax.clabel(cp, inline=True,fontsize=10)
1251 1251 # plt.show()
1252 1252 return array
1253 1253
1254 1254
1255 1255 class IntegrationFaradaySpectra(Operation):
1256 1256
1257 1257 __profIndex = 0
1258 1258 __withOverapping = False
1259 1259
1260 1260 __byTime = False
1261 1261 __initime = None
1262 1262 __lastdatatime = None
1263 1263 __integrationtime = None
1264 1264
1265 1265 __buffer_spc = None
1266 1266 __buffer_cspc = None
1267 1267 __buffer_dc = None
1268 1268
1269 1269 __dataReady = False
1270 1270
1271 1271 __timeInterval = None
1272 1272 n_ints = None #matriz de numero de integracions (CH,HEI)
1273 1273 n = None
1274 1274 minHei_ind = None
1275 1275 maxHei_ind = None
1276 1276 navg = 1.0
1277 1277 factor = 0.0
1278 1278 dataoutliers = None # (CHANNELS, HEIGHTS)
1279 1279
1280 1280 def __init__(self):
1281 1281
1282 1282 Operation.__init__(self)
1283 1283
1284 1284 def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None, minHei=None, maxHei=None, avg=1,factor=0.75):
1285 1285 """
1286 1286 Set the parameters of the integration class.
1287 1287
1288 1288 Inputs:
1289 1289
1290 1290 n : Number of coherent integrations
1291 1291 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1292 1292 overlapping :
1293 1293
1294 1294 """
1295 1295
1296 1296 self.__initime = None
1297 1297 self.__lastdatatime = 0
1298 1298
1299 1299 self.__buffer_spc = []
1300 1300 self.__buffer_cspc = []
1301 1301 self.__buffer_dc = 0
1302 1302
1303 1303 self.__profIndex = 0
1304 1304 self.__dataReady = False
1305 1305 self.__byTime = False
1306 1306
1307 1307 self.factor = factor
1308 1308 self.navg = avg
1309 1309 #self.ByLags = dataOut.ByLags ###REDEFINIR
1310 1310 self.ByLags = False
1311 1311 self.maxProfilesInt = 1
1312
1312 self.__nChannels = dataOut.nChannels
1313 1313 if DPL != None:
1314 1314 self.DPL=DPL
1315 1315 else:
1316 1316 #self.DPL=dataOut.DPL ###REDEFINIR
1317 1317 self.DPL=0
1318 1318
1319 1319 if n is None and timeInterval is None:
1320 1320 raise ValueError("n or timeInterval should be specified ...")
1321 1321
1322 1322 if n is not None:
1323 1323 self.n = int(n)
1324 1324 else:
1325 1325 self.__integrationtime = int(timeInterval)
1326 1326 self.n = None
1327 1327 self.__byTime = True
1328 1328
1329 1329 if minHei == None:
1330 1330 minHei = self.dataOut.heightList[0]
1331 1331
1332 1332 if maxHei == None:
1333 1333 maxHei = self.dataOut.heightList[-1]
1334 1334
1335 1335 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
1336 1336 print('minHei: %.2f is out of the heights range' % (minHei))
1337 1337 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
1338 1338 minHei = self.dataOut.heightList[0]
1339 1339
1340 1340 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
1341 1341 print('maxHei: %.2f is out of the heights range' % (maxHei))
1342 1342 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
1343 1343 maxHei = self.dataOut.heightList[-1]
1344 1344
1345 1345 ind_list1 = numpy.where(self.dataOut.heightList >= minHei)
1346 1346 ind_list2 = numpy.where(self.dataOut.heightList <= maxHei)
1347 1347 self.minHei_ind = ind_list1[0][0]
1348 1348 self.maxHei_ind = ind_list2[0][-1]
1349 #print("setup rem sats done")
1349 1350
1350 1351 def putData(self, data_spc, data_cspc, data_dc):
1351 1352 """
1352 1353 Add a profile to the __buffer_spc and increase in one the __profileIndex
1353 1354
1354 1355 """
1355 1356
1356 1357 self.__buffer_spc.append(data_spc)
1357 1358
1358 if data_cspc is None:
1359 if self.__nChannels < 2:
1359 1360 self.__buffer_cspc = None
1360 1361 else:
1361 1362 self.__buffer_cspc.append(data_cspc)
1362 1363
1363 1364 if data_dc is None:
1364 1365 self.__buffer_dc = None
1365 1366 else:
1366 1367 self.__buffer_dc += data_dc
1367 1368
1368 1369 self.__profIndex += 1
1369 1370
1370 1371 return
1371 1372
1372 1373 def hildebrand_sekhon_Integration(self,sortdata,navg, factor):
1373 1374 #data debe estar ordenado
1374 1375 #sortdata = numpy.sort(data, axis=None)
1375 1376 #sortID=data.argsort()
1376 1377 lenOfData = len(sortdata)
1377 1378 nums_min = lenOfData*factor
1378 1379 if nums_min <= 5:
1379 1380 nums_min = 5
1380 1381 sump = 0.
1381 1382 sumq = 0.
1382 1383 j = 0
1383 1384 cont = 1
1384 1385 while((cont == 1)and(j < lenOfData)):
1385 1386 sump += sortdata[j]
1386 1387 sumq += sortdata[j]**2
1387 1388 if j > nums_min:
1388 1389 rtest = float(j)/(j-1) + 1.0/navg
1389 1390 if ((sumq*j) > (rtest*sump**2)):
1390 1391 j = j - 1
1391 1392 sump = sump - sortdata[j]
1392 1393 sumq = sumq - sortdata[j]**2
1393 1394 cont = 0
1394 1395 j += 1
1395 1396 #lnoise = sump / j
1396 1397 #print("H S done")
1397 1398 #return j,sortID
1398 1399 return j
1399 1400
1400 1401
1401 1402 def pushData(self):
1402 1403 """
1403 1404 Return the sum of the last profiles and the profiles used in the sum.
1404 1405
1405 1406 Affected:
1406 1407
1407 1408 self.__profileIndex
1408 1409
1409 1410 """
1410 1411 bufferH=None
1411 1412 buffer=None
1412 1413 buffer1=None
1413 1414 buffer_cspc=None
1414 1415 #print("aes: ", self.__buffer_cspc)
1415 1416 self.__buffer_spc=numpy.array(self.__buffer_spc)
1416 if self.__buffer_cspc != None:
1417 if self.__nChannels > 1 :
1417 1418 self.__buffer_cspc=numpy.array(self.__buffer_cspc)
1418 1419
1419 1420 #print("FREQ_DC",self.__buffer_spc.shape,self.__buffer_cspc.shape)
1420 1421
1421 1422 freq_dc = int(self.__buffer_spc.shape[2] / 2)
1422 1423 #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights)
1423 1424
1424 1425 self.dataOutliers = numpy.zeros((self.nChannels,self.nHeights)) # --> almacen de outliers
1425 1426
1426 1427 for k in range(self.minHei_ind,self.maxHei_ind):
1427 if self.__buffer_cspc != None:
1428 if self.__nChannels > 1:
1428 1429 buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k])
1429 1430
1430 1431 outliers_IDs_cspc=[]
1431 1432 cspc_outliers_exist=False
1432 1433 for i in range(self.nChannels):#dataOut.nChannels):
1433 1434
1434 1435 buffer1=numpy.copy(self.__buffer_spc[:,i,:,k])
1435 1436 indexes=[]
1436 1437 #sortIDs=[]
1437 1438 outliers_IDs=[]
1438 1439
1439 1440 for j in range(self.nProfiles): #frecuencias en el tiempo
1440 1441 # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0
1441 1442 # continue
1442 1443 # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1
1443 1444 # continue
1444 1445 buffer=buffer1[:,j]
1445 1446 sortdata = numpy.sort(buffer, axis=None)
1446 1447
1447 1448 sortID=buffer.argsort()
1448 1449 index = _noise.hildebrand_sekhon2(sortdata,self.navg)
1449 1450
1450 1451 #index,sortID=self.hildebrand_sekhon_Integration(buffer,1,self.factor)
1451 1452
1452 1453 # fig,ax = plt.subplots()
1453 1454 # ax.set_title(str(k)+" "+str(j))
1454 1455 # x=range(len(sortdata))
1455 1456 # ax.scatter(x,sortdata)
1456 1457 # ax.axvline(index)
1457 1458 # plt.show()
1458 1459
1459 1460 indexes.append(index)
1460 1461 #sortIDs.append(sortID)
1461 1462 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
1462 1463
1463 1464 #print("Outliers: ",outliers_IDs)
1464 1465 outliers_IDs=numpy.array(outliers_IDs)
1465 1466 outliers_IDs=outliers_IDs.ravel()
1466 1467 outliers_IDs=numpy.unique(outliers_IDs)
1467 1468 outliers_IDs=outliers_IDs.astype(numpy.dtype('int64'))
1468 1469 indexes=numpy.array(indexes)
1469 1470 indexmin=numpy.min(indexes)
1470 1471
1471 1472
1472 1473 #print(indexmin,buffer1.shape[0], k)
1473 1474
1474 1475 # fig,ax = plt.subplots()
1475 1476 # ax.plot(sortdata)
1476 1477 # ax2 = ax.twinx()
1477 1478 # x=range(len(indexes))
1478 1479 # #plt.scatter(x,indexes)
1479 1480 # ax2.scatter(x,indexes)
1480 1481 # plt.show()
1481 1482
1482 1483 if indexmin != buffer1.shape[0]:
1483 if self.nChannels > 1:
1484 if self.__nChannels > 1:
1484 1485 cspc_outliers_exist= True
1485 1486
1486 1487 lt=outliers_IDs
1487 1488 #avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0)
1488 1489
1489 1490 for p in list(outliers_IDs):
1490 1491 #buffer1[p,:]=avg
1491 1492 buffer1[p,:] = numpy.NaN
1492 1493
1493 1494 self.dataOutliers[i,k] = len(outliers_IDs)
1494 1495
1495 1496
1496 1497 self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1)
1497 1498
1498 1499
1499 if self.__buffer_cspc != None:
1500 if self.__nChannels > 1:
1500 1501 outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs)
1501 1502
1502 1503
1503 if self.__buffer_cspc != None:
1504 if self.__nChannels > 1:
1504 1505 outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64'))
1505 1506 if cspc_outliers_exist:
1506 1507
1507 1508 lt=outliers_IDs_cspc
1508 1509
1509 1510 #avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0)
1510 1511 for p in list(outliers_IDs_cspc):
1511 1512 #buffer_cspc[p,:]=avg
1512 1513 buffer_cspc[p,:] = numpy.NaN
1513 1514
1514 if self.__buffer_cspc != None:
1515 if self.__nChannels > 1:
1515 1516 self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc)
1516 1517
1517 1518
1518 1519
1519 1520
1520 1521 nOutliers = len(outliers_IDs)
1521 1522 #print("Outliers n: ",self.dataOutliers,nOutliers)
1522 1523 buffer=None
1523 1524 bufferH=None
1524 1525 buffer1=None
1525 1526 buffer_cspc=None
1526 1527
1527 1528
1528 1529 buffer=None
1529 1530
1530 1531 #data_spc = numpy.sum(self.__buffer_spc,axis=0)
1531 1532 data_spc = numpy.nansum(self.__buffer_spc,axis=0)
1532 if self.__buffer_cspc != None:
1533 if self.__nChannels > 1:
1533 1534 #data_cspc = numpy.sum(self.__buffer_cspc,axis=0)
1534 1535 data_cspc = numpy.nansum(self.__buffer_cspc,axis=0)
1535 1536 else:
1536 1537 data_cspc = None
1537 1538 data_dc = self.__buffer_dc
1538 1539 #(CH, HEIGH)
1539 1540 self.maxProfilesInt = self.__profIndex
1540 1541 n = self.__profIndex - self.dataOutliers # n becomes a matrix
1541 1542
1542 1543 self.__buffer_spc = []
1543 1544 self.__buffer_cspc = []
1544 1545 self.__buffer_dc = 0
1545 1546 self.__profIndex = 0
1546 1547 #print("cleaned ",data_cspc)
1547 1548 return data_spc, data_cspc, data_dc, n
1548 1549
1549 1550 def byProfiles(self, *args):
1550 1551
1551 1552 self.__dataReady = False
1552 1553 avgdata_spc = None
1553 1554 avgdata_cspc = None
1554 1555 avgdata_dc = None
1555 1556
1556 1557 self.putData(*args)
1557 1558
1558 1559 if self.__profIndex == self.n:
1559 1560
1560 1561 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1561 1562 self.n_ints = n
1562 1563 self.__dataReady = True
1563 1564
1564 1565 return avgdata_spc, avgdata_cspc, avgdata_dc
1565 1566
1566 1567 def byTime(self, datatime, *args):
1567 1568
1568 1569 self.__dataReady = False
1569 1570 avgdata_spc = None
1570 1571 avgdata_cspc = None
1571 1572 avgdata_dc = None
1572 1573
1573 1574 self.putData(*args)
1574 1575
1575 1576 if (datatime - self.__initime) >= self.__integrationtime:
1576 1577 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1577 1578 self.n_ints = n
1578 1579 self.__dataReady = True
1579 1580
1580 1581 return avgdata_spc, avgdata_cspc, avgdata_dc
1581 1582
1582 1583 def integrate(self, datatime, *args):
1583 1584
1584 1585 if self.__profIndex == 0:
1585 1586 self.__initime = datatime
1586 1587
1587 1588 if self.__byTime:
1588 1589 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1589 1590 datatime, *args)
1590 1591 else:
1591 1592 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1592 1593
1593 1594 if not self.__dataReady:
1594 1595 return None, None, None, None
1595 1596
1596 1597 #print("integrate", avgdata_cspc)
1597 1598 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1598 1599
1599 1600 def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False, minHei=None, maxHei=None, avg=1, factor=0.75):
1600 1601 self.dataOut = dataOut
1601 1602 if n == 1:
1602 1603 return self.dataOut
1603 1604
1604 1605 #print("nchannels", self.dataOut.nChannels)
1605 1606 if self.dataOut.nChannels == 1:
1606 1607 self.dataOut.data_cspc = None #si es un solo canal no vale la pena acumular DATOS
1607 1608 #print("IN spc:", self.dataOut.data_spc.shape, self.dataOut.data_cspc)
1608 1609 if not self.isConfig:
1609 1610 self.setup(self.dataOut, n, timeInterval, overlapping,DPL ,minHei, maxHei, avg, factor)
1610 1611 self.isConfig = True
1611 1612
1612 1613 if not self.ByLags:
1613 1614 self.nProfiles=self.dataOut.nProfiles
1614 1615 self.nChannels=self.dataOut.nChannels
1615 1616 self.nHeights=self.dataOut.nHeights
1616 1617 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1617 1618 self.dataOut.data_spc,
1618 1619 self.dataOut.data_cspc,
1619 1620 self.dataOut.data_dc)
1620 1621 else:
1621 1622 self.nProfiles=self.dataOut.nProfiles
1622 1623 self.nChannels=self.dataOut.nChannels
1623 1624 self.nHeights=self.dataOut.nHeights
1624 1625 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1625 1626 self.dataOut.dataLag_spc,
1626 1627 self.dataOut.dataLag_cspc,
1627 1628 self.dataOut.dataLag_dc)
1628 1629 self.dataOut.flagNoData = True
1629 1630 if self.__dataReady:
1630 1631
1631 1632 if not self.ByLags:
1632 1633 if self.nChannels == 1:
1633 1634 #print("f int", avgdata_spc.shape)
1634 1635 self.dataOut.data_spc = avgdata_spc
1635 1636 self.dataOut.data_cspc = None
1636 1637 else:
1637 1638 self.dataOut.data_spc = numpy.squeeze(avgdata_spc)
1638 1639 self.dataOut.data_cspc = numpy.squeeze(avgdata_cspc)
1639 1640 self.dataOut.data_dc = avgdata_dc
1640 1641 self.dataOut.data_outlier = self.dataOutliers
1641 1642
1642 1643 else:
1643 1644 self.dataOut.dataLag_spc = avgdata_spc
1644 1645 self.dataOut.dataLag_cspc = avgdata_cspc
1645 1646 self.dataOut.dataLag_dc = avgdata_dc
1646 1647
1647 1648 self.dataOut.data_spc=self.dataOut.dataLag_spc[:,:,:,self.dataOut.LagPlot]
1648 1649 self.dataOut.data_cspc=self.dataOut.dataLag_cspc[:,:,:,self.dataOut.LagPlot]
1649 1650 self.dataOut.data_dc=self.dataOut.dataLag_dc[:,:,self.dataOut.LagPlot]
1650 1651
1651 1652
1652 1653 self.dataOut.nIncohInt *= self.n_ints
1653 1654 self.dataOut.max_nIncohInt = self.maxProfilesInt
1654 1655 #print(self.dataOut.max_nIncohInt)
1655 1656 self.dataOut.utctime = avgdatatime
1656 1657 self.dataOut.flagNoData = False
1657 1658 #print("Faraday Integration DONE...", self.dataOut.data_cspc)
1658 1659 #print(self.dataOut.flagNoData)
1659 1660 return self.dataOut
1660 1661
1661 1662
1662 1663
1663 1664 class removeInterference(Operation):
1664 1665
1665 1666 def removeInterference3(self, min_hei = None, max_hei = None):
1666 1667
1667 1668 jspectra = self.dataOut.data_spc
1668 1669 #jcspectra = self.dataOut.data_cspc
1669 1670 jnoise = self.dataOut.getNoise()
1670 1671 num_incoh = self.dataOut.max_nIncohInt
1671 1672 #print(jspectra.shape)
1672 1673 num_channel, num_prof, num_hei = jspectra.shape
1673 1674 minHei = min_hei
1674 1675 maxHei = max_hei
1675 1676 ########################################################################
1676 1677 if minHei == None or (minHei < self.dataOut.heightList[0]):
1677 1678 minHei = self.dataOut.heightList[0]
1678 1679
1679 1680 if maxHei == None or (maxHei > self.dataOut.heightList[-1]):
1680 1681 maxHei = self.dataOut.heightList[-1]
1681 1682 minIndex = 0
1682 1683 maxIndex = 0
1683 1684 heights = self.dataOut.heightList
1684 1685
1685 1686 inda = numpy.where(heights >= minHei)
1686 1687 indb = numpy.where(heights <= maxHei)
1687 1688
1688 1689 try:
1689 1690 minIndex = inda[0][0]
1690 1691 except:
1691 1692 minIndex = 0
1692 1693 try:
1693 1694 maxIndex = indb[0][-1]
1694 1695 except:
1695 1696 maxIndex = len(heights)
1696 1697
1697 1698 if (minIndex < 0) or (minIndex > maxIndex):
1698 1699 raise ValueError("some value in (%d,%d) is not valid" % (
1699 1700 minIndex, maxIndex))
1700 1701 if (maxIndex >= self.dataOut.nHeights):
1701 1702 maxIndex = self.dataOut.nHeights - 1
1702 1703
1703 1704 ########################################################################
1704 1705
1705 1706
1706 1707 #dataOut.max_nIncohInt * dataOut.nCohInt
1707 1708 norm = self.dataOut.nIncohInt /self.dataOut.max_nIncohInt
1708 1709 #print(norm.shape)
1709 1710 # Subrutina de Remocion de la Interferencia
1710 1711 for ich in range(num_channel):
1711 1712 # Se ordena los espectros segun su potencia (menor a mayor)
1712 1713 #power = jspectra[ich, mask_prof, :]
1713 1714 interf = jspectra[ich, :, minIndex:maxIndex]
1714 1715 #print(interf.shape)
1715 1716 inttef = interf.mean(axis=1)
1716 1717
1717 1718 for hei in range(num_hei):
1718 1719 temp = jspectra[ich,:, hei]
1719 1720 temp -= inttef
1720 1721 temp += jnoise[ich]*norm[ich,hei]
1721 1722 jspectra[ich,:, hei] = temp
1722 1723
1723 1724 # Guardar Resultados
1724 1725 self.dataOut.data_spc = jspectra
1725 1726 #self.dataOut.data_cspc = jcspectra
1726 1727
1727 1728 return 1
1728 1729
1729 1730 def removeInterference2(self):
1730 1731
1731 1732 cspc = self.dataOut.data_cspc
1732 1733 spc = self.dataOut.data_spc
1733 1734 Heights = numpy.arange(cspc.shape[2])
1734 1735 realCspc = numpy.abs(cspc)
1735 1736
1736 1737 for i in range(cspc.shape[0]):
1737 1738 LinePower= numpy.sum(realCspc[i], axis=0)
1738 1739 Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
1739 1740 SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
1740 1741 InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
1741 1742 InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
1742 1743 InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
1743 1744
1744 1745
1745 1746 InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
1746 1747 #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
1747 1748 if len(InterferenceRange)<int(cspc.shape[1]*0.3):
1748 1749 cspc[i,InterferenceRange,:] = numpy.NaN
1749 1750
1750 1751 self.dataOut.data_cspc = cspc
1751 1752
1752 1753 def removeInterference(self, interf = 2, hei_interf = None, nhei_interf = None, offhei_interf = None):
1753 1754
1754 1755 jspectra = self.dataOut.data_spc
1755 1756 jcspectra = self.dataOut.data_cspc
1756 1757 jnoise = self.dataOut.getNoise()
1757 1758 #num_incoh = self.dataOut.nIncohInt
1758 1759 num_incoh = self.dataOut.max_nIncohInt
1759 1760 #print("spc: ", jspectra.shape, jcspectra)
1760 1761 num_channel = jspectra.shape[0]
1761 1762 num_prof = jspectra.shape[1]
1762 1763 num_hei = jspectra.shape[2]
1763 1764
1764 1765 # hei_interf
1765 1766 if hei_interf is None:
1766 1767 count_hei = int(num_hei / 2) # a half of total ranges
1767 1768 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
1768 1769 hei_interf = numpy.asarray(hei_interf)[0]
1769 1770 #print(hei_interf)
1770 1771 # nhei_interf
1771 1772 if (nhei_interf == None):
1772 1773 nhei_interf = 5
1773 1774 if (nhei_interf < 1):
1774 1775 nhei_interf = 1
1775 1776 if (nhei_interf > count_hei):
1776 1777 nhei_interf = count_hei
1777 1778 if (offhei_interf == None):
1778 1779 offhei_interf = 0
1779 1780
1780 1781 ind_hei = list(range(num_hei))
1781 1782 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
1782 1783 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
1783 1784 mask_prof = numpy.asarray(list(range(num_prof)))
1784 1785 num_mask_prof = mask_prof.size
1785 1786 comp_mask_prof = [0, num_prof / 2]
1786 1787
1787 1788 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
1788 1789 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
1789 1790 jnoise = numpy.nan
1790 1791 noise_exist = jnoise[0] < numpy.Inf
1791 1792
1792 1793 # Subrutina de Remocion de la Interferencia
1793 1794 for ich in range(num_channel):
1794 1795 # Se ordena los espectros segun su potencia (menor a mayor)
1795 1796 power = jspectra[ich, mask_prof, :]
1796 1797 power = power[:, hei_interf]
1797 1798 power = power.sum(axis=0)
1798 1799 psort = power.ravel().argsort()
1799 1800 print(hei_interf[psort[list(range(offhei_interf, nhei_interf + offhei_interf))]])
1800 1801 # Se estima la interferencia promedio en los Espectros de Potencia empleando
1801 1802 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
1802 1803 offhei_interf, nhei_interf + offhei_interf))]]]
1803 1804
1804 1805 if noise_exist:
1805 1806 # tmp_noise = jnoise[ich] / num_prof
1806 1807 tmp_noise = jnoise[ich]
1807 1808 junkspc_interf = junkspc_interf - tmp_noise
1808 1809 #junkspc_interf[:,comp_mask_prof] = 0
1809 1810 print(junkspc_interf.shape)
1810 1811 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
1811 1812 jspc_interf = jspc_interf.transpose()
1812 1813 # Calculando el espectro de interferencia promedio
1813 1814 noiseid = numpy.where(jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
1814 1815 noiseid = noiseid[0]
1815 1816 cnoiseid = noiseid.size
1816 1817 interfid = numpy.where(jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
1817 1818 interfid = interfid[0]
1818 1819 cinterfid = interfid.size
1819 1820
1820 1821 if (cnoiseid > 0):
1821 1822 jspc_interf[noiseid] = 0
1822 1823 # Expandiendo los perfiles a limpiar
1823 1824 if (cinterfid > 0):
1824 1825 new_interfid = (
1825 1826 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
1826 1827 new_interfid = numpy.asarray(new_interfid)
1827 1828 new_interfid = {x for x in new_interfid}
1828 1829 new_interfid = numpy.array(list(new_interfid))
1829 1830 new_cinterfid = new_interfid.size
1830 1831 else:
1831 1832 new_cinterfid = 0
1832 1833
1833 1834 for ip in range(new_cinterfid):
1834 1835 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
1835 1836 jspc_interf[new_interfid[ip]] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
1836 1837
1837 1838 jspectra[ich, :, ind_hei] = jspectra[ich, :,ind_hei] - jspc_interf # Corregir indices
1838 1839
1839 1840 # Removiendo la interferencia del punto de mayor interferencia
1840 1841 ListAux = jspc_interf[mask_prof].tolist()
1841 1842 maxid = ListAux.index(max(ListAux))
1842 1843 print(cinterfid)
1843 1844 if cinterfid > 0:
1844 1845 for ip in range(cinterfid * (interf == 2) - 1):
1845 1846 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
1846 1847 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
1847 1848 cind = len(ind)
1848 1849
1849 1850 if (cind > 0):
1850 1851 jspectra[ich, interfid[ip], ind] = tmp_noise * \
1851 1852 (1 + (numpy.random.uniform(cind) - 0.5) /
1852 1853 numpy.sqrt(num_incoh))
1853 1854
1854 1855 ind = numpy.array([-2, -1, 1, 2])
1855 1856 xx = numpy.zeros([4, 4])
1856 1857
1857 1858 for id1 in range(4):
1858 1859 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1859 1860 xx_inv = numpy.linalg.inv(xx)
1860 1861 xx = xx_inv[:, 0]
1861 1862 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1862 1863 yy = jspectra[ich, mask_prof[ind], :]
1863 1864 jspectra[ich, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1864 1865
1865 1866 indAux = (jspectra[ich, :, :] < tmp_noise *
1866 1867 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
1867 1868 print(indAux)
1868 1869 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
1869 1870 (1 - 1 / numpy.sqrt(num_incoh))
1870 1871
1871 1872 # Remocion de Interferencia en el Cross Spectra
1872 1873 if jcspectra is None:
1873 1874 return jspectra, jcspectra
1874 1875 num_pairs = int(jcspectra.size / (num_prof * num_hei))
1875 1876 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
1876 1877
1877 1878 for ip in range(num_pairs):
1878 1879
1879 1880 #-------------------------------------------
1880 1881
1881 1882 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
1882 1883 cspower = cspower[:, hei_interf]
1883 1884 cspower = cspower.sum(axis=0)
1884 1885
1885 1886 cspsort = cspower.ravel().argsort()
1886 1887 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
1887 1888 offhei_interf, nhei_interf + offhei_interf))]]]
1888 1889 junkcspc_interf = junkcspc_interf.transpose()
1889 1890 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
1890 1891
1891 1892 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
1892 1893
1893 1894 median_real = int(numpy.median(numpy.real(
1894 1895 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1895 1896 median_imag = int(numpy.median(numpy.imag(
1896 1897 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1897 1898 comp_mask_prof = [int(e) for e in comp_mask_prof]
1898 1899 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
1899 1900 median_real, median_imag)
1900 1901
1901 1902 for iprof in range(num_prof):
1902 1903 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
1903 1904 jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
1904 1905
1905 1906 # Removiendo la Interferencia
1906 1907 jcspectra[ip, :, ind_hei] = jcspectra[ip,
1907 1908 :, ind_hei] - jcspc_interf
1908 1909
1909 1910 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
1910 1911 maxid = ListAux.index(max(ListAux))
1911 1912
1912 1913 ind = numpy.array([-2, -1, 1, 2])
1913 1914 xx = numpy.zeros([4, 4])
1914 1915
1915 1916 for id1 in range(4):
1916 1917 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1917 1918
1918 1919 xx_inv = numpy.linalg.inv(xx)
1919 1920 xx = xx_inv[:, 0]
1920 1921
1921 1922 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1922 1923 yy = jcspectra[ip, mask_prof[ind], :]
1923 1924 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1924 1925
1925 1926 # Guardar Resultados
1926 1927 self.dataOut.data_spc = jspectra
1927 1928 self.dataOut.data_cspc = jcspectra
1928 1929
1929 1930 return 1
1930 1931
1931 1932 def run(self, dataOut, interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None, mode=1, minHei=None, maxHei=None):
1932 1933
1933 1934 self.dataOut = dataOut
1934 1935
1935 1936 if mode == 1:
1936 1937 self.removeInterference(interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None)
1937 1938 elif mode == 2:
1938 1939 self.removeInterference2()
1939 1940 elif mode == 3:
1940 1941 self.removeInterference3(min_hei=minHei, max_hei=maxHei)
1941 1942 return self.dataOut
1942 1943
1943 1944
1944 1945 class IncohInt(Operation):
1945 1946
1946 1947 __profIndex = 0
1947 1948 __withOverapping = False
1948 1949
1949 1950 __byTime = False
1950 1951 __initime = None
1951 1952 __lastdatatime = None
1952 1953 __integrationtime = None
1953 1954
1954 1955 __buffer_spc = None
1955 1956 __buffer_cspc = None
1956 1957 __buffer_dc = None
1957 1958
1958 1959 __dataReady = False
1959 1960
1960 1961 __timeInterval = None
1961 1962 incohInt = 0
1962 1963 nOutliers = 0
1963 1964 n = None
1964 1965
1965 1966 def __init__(self):
1966 1967
1967 1968 Operation.__init__(self)
1968 1969
1969 1970 def setup(self, n=None, timeInterval=None, overlapping=False):
1970 1971 """
1971 1972 Set the parameters of the integration class.
1972 1973
1973 1974 Inputs:
1974 1975
1975 1976 n : Number of coherent integrations
1976 1977 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1977 1978 overlapping :
1978 1979
1979 1980 """
1980 1981
1981 1982 self.__initime = None
1982 1983 self.__lastdatatime = 0
1983 1984
1984 1985 self.__buffer_spc = 0
1985 1986 self.__buffer_cspc = 0
1986 1987 self.__buffer_dc = 0
1987 1988
1988 1989 self.__profIndex = 0
1989 1990 self.__dataReady = False
1990 1991 self.__byTime = False
1991 1992 self.incohInt = 0
1992 1993 self.nOutliers = 0
1993 1994 if n is None and timeInterval is None:
1994 1995 raise ValueError("n or timeInterval should be specified ...")
1995 1996
1996 1997 if n is not None:
1997 1998 self.n = int(n)
1998 1999 else:
1999 2000
2000 2001 self.__integrationtime = int(timeInterval)
2001 2002 self.n = None
2002 2003 self.__byTime = True
2003 2004
2004 2005 def putData(self, data_spc, data_cspc, data_dc):
2005 2006 """
2006 2007 Add a profile to the __buffer_spc and increase in one the __profileIndex
2007 2008
2008 2009 """
2009 2010 if data_spc.all() == numpy.nan :
2010 2011 print("nan ")
2011 2012 return
2012 2013 self.__buffer_spc += data_spc
2013 2014
2014 2015 if data_cspc is None:
2015 2016 self.__buffer_cspc = None
2016 2017 else:
2017 2018 self.__buffer_cspc += data_cspc
2018 2019
2019 2020 if data_dc is None:
2020 2021 self.__buffer_dc = None
2021 2022 else:
2022 2023 self.__buffer_dc += data_dc
2023 2024
2024 2025 self.__profIndex += 1
2025 2026
2026 2027 return
2027 2028
2028 2029 def pushData(self):
2029 2030 """
2030 2031 Return the sum of the last profiles and the profiles used in the sum.
2031 2032
2032 2033 Affected:
2033 2034
2034 2035 self.__profileIndex
2035 2036
2036 2037 """
2037 2038
2038 2039 data_spc = self.__buffer_spc
2039 2040 data_cspc = self.__buffer_cspc
2040 2041 data_dc = self.__buffer_dc
2041 2042 n = self.__profIndex
2042 2043
2043 2044 self.__buffer_spc = 0
2044 2045 self.__buffer_cspc = 0
2045 2046 self.__buffer_dc = 0
2046 2047
2047 2048
2048 2049 return data_spc, data_cspc, data_dc, n
2049 2050
2050 2051 def byProfiles(self, *args):
2051 2052
2052 2053 self.__dataReady = False
2053 2054 avgdata_spc = None
2054 2055 avgdata_cspc = None
2055 2056 avgdata_dc = None
2056 2057
2057 2058 self.putData(*args)
2058 2059
2059 2060 if self.__profIndex == self.n:
2060 2061
2061 2062 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
2062 2063 self.n = n
2063 2064 self.__dataReady = True
2064 2065
2065 2066 return avgdata_spc, avgdata_cspc, avgdata_dc
2066 2067
2067 2068 def byTime(self, datatime, *args):
2068 2069
2069 2070 self.__dataReady = False
2070 2071 avgdata_spc = None
2071 2072 avgdata_cspc = None
2072 2073 avgdata_dc = None
2073 2074
2074 2075 self.putData(*args)
2075 2076
2076 2077 if (datatime - self.__initime) >= self.__integrationtime:
2077 2078 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
2078 2079 self.n = n
2079 2080 self.__dataReady = True
2080 2081
2081 2082 return avgdata_spc, avgdata_cspc, avgdata_dc
2082 2083
2083 2084 def integrate(self, datatime, *args):
2084 2085
2085 2086 if self.__profIndex == 0:
2086 2087 self.__initime = datatime
2087 2088
2088 2089 if self.__byTime:
2089 2090 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
2090 2091 datatime, *args)
2091 2092 else:
2092 2093 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
2093 2094
2094 2095 if not self.__dataReady:
2095 2096 return None, None, None, None
2096 2097
2097 2098 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
2098 2099
2099 2100 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
2100 2101 if n == 1:
2101 2102 return dataOut
2102 2103
2103 2104 if dataOut.flagNoData == True:
2104 2105 return dataOut
2105 2106
2106 2107 dataOut.flagNoData = True
2107 2108
2108 2109 if not self.isConfig:
2109 2110 self.setup(n, timeInterval, overlapping)
2110 2111 self.isConfig = True
2111 2112
2112 2113 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
2113 2114 dataOut.data_spc,
2114 2115 dataOut.data_cspc,
2115 2116 dataOut.data_dc)
2116 2117 self.incohInt += dataOut.nIncohInt
2117 2118 self.nOutliers += dataOut.data_outlier
2118 2119 if self.__dataReady:
2119 2120 #print("prof: ",dataOut.max_nIncohInt,self.__profIndex)
2120 2121 dataOut.data_spc = avgdata_spc
2121 2122 dataOut.data_cspc = avgdata_cspc
2122 2123 dataOut.data_dc = avgdata_dc
2123 2124 dataOut.nIncohInt = self.incohInt
2124 2125 dataOut.data_outlier = self.nOutliers
2125 2126 dataOut.utctime = avgdatatime
2126 2127 dataOut.flagNoData = False
2127 2128 dataOut.max_nIncohInt = self.__profIndex
2128 2129 self.incohInt = 0
2129 2130 self.nOutliers = 0
2130 2131 self.__profIndex = 0
2131 2132 #print("IncohInt Done")
2132 2133 return dataOut
2133 2134
2134 2135 class dopplerFlip(Operation):
2135 2136
2136 2137 def run(self, dataOut):
2137 2138 # arreglo 1: (num_chan, num_profiles, num_heights)
2138 2139 self.dataOut = dataOut
2139 2140 # JULIA-oblicua, indice 2
2140 2141 # arreglo 2: (num_profiles, num_heights)
2141 2142 jspectra = self.dataOut.data_spc[2]
2142 2143 jspectra_tmp = numpy.zeros(jspectra.shape)
2143 2144 num_profiles = jspectra.shape[0]
2144 2145 freq_dc = int(num_profiles / 2)
2145 2146 # Flip con for
2146 2147 for j in range(num_profiles):
2147 2148 jspectra_tmp[num_profiles-j-1]= jspectra[j]
2148 2149 # Intercambio perfil de DC con perfil inmediato anterior
2149 2150 jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1]
2150 2151 jspectra_tmp[freq_dc]= jspectra[freq_dc]
2151 2152 # canal modificado es re-escrito en el arreglo de canales
2152 2153 self.dataOut.data_spc[2] = jspectra_tmp
2153 2154
2154 2155 return self.dataOut
Show file before | Show file after | Hide comments
@@ -1,2839 +1,3137
1 1 import sys
2 2 import numpy,math
3 3 from scipy import interpolate
4 4 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
5 5 from schainpy.model.data.jrodata import Voltage,hildebrand_sekhon
6 6 from schainpy.utils import log
7 7 from schainpy.model.io.utils import getHei_index
8 8 from time import time
9 9 import datetime
10 10 import numpy
11 11 #import copy
12 12 from schainpy.model.data import _noise
13 13
14 14 class VoltageProc(ProcessingUnit):
15 15
16 16 def __init__(self):
17 17
18 18 ProcessingUnit.__init__(self)
19 19
20 20 self.dataOut = Voltage()
21 21 self.flip = 1
22 22 self.setupReq = False
23 23
24 24 def run(self):
25 25 #print("running volt proc")
26 26 if self.dataIn.type == 'AMISR':
27 27 self.__updateObjFromAmisrInput()
28 28
29 29 if self.dataOut.buffer_empty:
30 30 if self.dataIn.type == 'Voltage':
31 31 self.dataOut.copy(self.dataIn)
32 32 #print("new volts reading")
33 33
34 34
35 35 def __updateObjFromAmisrInput(self):
36 36
37 37 self.dataOut.timeZone = self.dataIn.timeZone
38 38 self.dataOut.dstFlag = self.dataIn.dstFlag
39 39 self.dataOut.errorCount = self.dataIn.errorCount
40 40 self.dataOut.useLocalTime = self.dataIn.useLocalTime
41 41
42 42 self.dataOut.flagNoData = self.dataIn.flagNoData
43 43 self.dataOut.data = self.dataIn.data
44 44 self.dataOut.utctime = self.dataIn.utctime
45 45 self.dataOut.channelList = self.dataIn.channelList
46 46 #self.dataOut.timeInterval = self.dataIn.timeInterval
47 47 self.dataOut.heightList = self.dataIn.heightList
48 48 self.dataOut.nProfiles = self.dataIn.nProfiles
49 49
50 50 self.dataOut.nCohInt = self.dataIn.nCohInt
51 51 self.dataOut.ippSeconds = self.dataIn.ippSeconds
52 52 self.dataOut.frequency = self.dataIn.frequency
53 53
54 54 self.dataOut.azimuth = self.dataIn.azimuth
55 55 self.dataOut.zenith = self.dataIn.zenith
56 56
57 57 self.dataOut.beam.codeList = self.dataIn.beam.codeList
58 58 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
59 59 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
60 60
61 61
62 62 class selectChannels(Operation):
63 63
64 64 def run(self, dataOut, channelList=None):
65 65 self.channelList = channelList
66 66 if self.channelList == None:
67 67 print("Missing channelList")
68 68 return dataOut
69 69 channelIndexList = []
70 70
71 71 if type(dataOut.channelList) is not list: #leer array desde HDF5
72 72 try:
73 73 dataOut.channelList = dataOut.channelList.tolist()
74 74 except Exception as e:
75 75 print("Select Channels: ",e)
76 76 for channel in self.channelList:
77 77 if channel not in dataOut.channelList:
78 78 raise ValueError("Channel %d is not in %s" %(channel, str(dataOut.channelList)))
79 79
80 80 index = dataOut.channelList.index(channel)
81 81 channelIndexList.append(index)
82 82 dataOut = self.selectChannelsByIndex(dataOut,channelIndexList)
83 83 return dataOut
84 84
85 85 def selectChannelsByIndex(self, dataOut, channelIndexList):
86 86 """
87 87 Selecciona un bloque de datos en base a canales segun el channelIndexList
88 88
89 89 Input:
90 90 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
91 91
92 92 Affected:
93 93 dataOut.data
94 94 dataOut.channelIndexList
95 95 dataOut.nChannels
96 96 dataOut.m_ProcessingHeader.totalSpectra
97 97 dataOut.systemHeaderObj.numChannels
98 98 dataOut.m_ProcessingHeader.blockSize
99 99
100 100 Return:
101 101 None
102 102 """
103 103 #print("selectChannelsByIndex")
104 104 # for channelIndex in channelIndexList:
105 105 # if channelIndex not in dataOut.channelIndexList:
106 106 # raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
107 107
108 108 if dataOut.type == 'Voltage':
109 109 if dataOut.flagDataAsBlock:
110 110 """
111 111 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
112 112 """
113 113 data = dataOut.data[channelIndexList,:,:]
114 114 else:
115 115 data = dataOut.data[channelIndexList,:]
116 116
117 117 dataOut.data = data
118 118 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
119 119 dataOut.channelList = range(len(channelIndexList))
120 120
121 121 elif dataOut.type == 'Spectra':
122 122 if hasattr(dataOut, 'data_spc'):
123 123 if dataOut.data_spc is None:
124 124 raise ValueError("data_spc is None")
125 125 return dataOut
126 126 else:
127 127 data_spc = dataOut.data_spc[channelIndexList, :]
128 128 dataOut.data_spc = data_spc
129 129
130 130 # if hasattr(dataOut, 'data_dc') :# and
131 131 # if dataOut.data_dc is None:
132 132 # raise ValueError("data_dc is None")
133 133 # return dataOut
134 134 # else:
135 135 # data_dc = dataOut.data_dc[channelIndexList, :]
136 136 # dataOut.data_dc = data_dc
137 137 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
138 138 dataOut.channelList = channelIndexList
139 139 dataOut = self.__selectPairsByChannel(dataOut,channelIndexList)
140 140
141 141 return dataOut
142 142
143 143 def __selectPairsByChannel(self, dataOut, channelList=None):
144 144 #print("__selectPairsByChannel")
145 145 if channelList == None:
146 146 return
147 147
148 148 pairsIndexListSelected = []
149 149 for pairIndex in dataOut.pairsIndexList:
150 150 # First pair
151 151 if dataOut.pairsList[pairIndex][0] not in channelList:
152 152 continue
153 153 # Second pair
154 154 if dataOut.pairsList[pairIndex][1] not in channelList:
155 155 continue
156 156
157 157 pairsIndexListSelected.append(pairIndex)
158 158 if not pairsIndexListSelected:
159 159 dataOut.data_cspc = None
160 160 dataOut.pairsList = []
161 161 return
162 162
163 163 dataOut.data_cspc = dataOut.data_cspc[pairsIndexListSelected]
164 164 dataOut.pairsList = [dataOut.pairsList[i]
165 165 for i in pairsIndexListSelected]
166 166
167 167 return dataOut
168 168
169 169 class selectHeights(Operation):
170 170
171 171 def run(self, dataOut, minHei=None, maxHei=None, minIndex=None, maxIndex=None):
172 172 """
173 173 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
174 174 minHei <= height <= maxHei
175 175
176 176 Input:
177 177 minHei : valor minimo de altura a considerar
178 178 maxHei : valor maximo de altura a considerar
179 179
180 180 Affected:
181 181 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
182 182
183 183 Return:
184 184 1 si el metodo se ejecuto con exito caso contrario devuelve 0
185 185 """
186 186
187 187 self.dataOut = dataOut
188 188
189 189 if minHei and maxHei:
190 190
191 191 if (minHei < dataOut.heightList[0]):
192 192 minHei = dataOut.heightList[0]
193 193
194 194 if (maxHei > dataOut.heightList[-1]):
195 195 maxHei = dataOut.heightList[-1]
196 196
197 197 minIndex = 0
198 198 maxIndex = 0
199 199 heights = dataOut.heightList
200 200
201 201 inda = numpy.where(heights >= minHei)
202 202 indb = numpy.where(heights <= maxHei)
203 203
204 204 try:
205 205 minIndex = inda[0][0]
206 206 except:
207 207 minIndex = 0
208 208
209 209 try:
210 210 maxIndex = indb[0][-1]
211 211 except:
212 212 maxIndex = len(heights)
213 213
214 214 self.selectHeightsByIndex(minIndex, maxIndex)
215 215
216 216 return dataOut
217 217
218 218 def selectHeightsByIndex(self, minIndex, maxIndex):
219 219 """
220 220 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
221 221 minIndex <= index <= maxIndex
222 222
223 223 Input:
224 224 minIndex : valor de indice minimo de altura a considerar
225 225 maxIndex : valor de indice maximo de altura a considerar
226 226
227 227 Affected:
228 228 self.dataOut.data
229 229 self.dataOut.heightList
230 230
231 231 Return:
232 232 1 si el metodo se ejecuto con exito caso contrario devuelve 0
233 233 """
234 234
235 235 if self.dataOut.type == 'Voltage':
236 236 if (minIndex < 0) or (minIndex > maxIndex):
237 237 raise ValueError("Height index range (%d,%d) is not valid" % (minIndex, maxIndex))
238 238
239 239 if (maxIndex >= self.dataOut.nHeights):
240 240 maxIndex = self.dataOut.nHeights
241 241
242 242 #voltage
243 243 if self.dataOut.flagDataAsBlock:
244 244 """
245 245 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
246 246 """
247 247 data = self.dataOut.data[:,:, minIndex:maxIndex]
248 248 else:
249 249 data = self.dataOut.data[:, minIndex:maxIndex]
250 250
251 251 # firstHeight = self.dataOut.heightList[minIndex]
252 252
253 253 self.dataOut.data = data
254 254 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
255 255
256 256 if self.dataOut.nHeights <= 1:
257 257 raise ValueError("selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights))
258 258 elif self.dataOut.type == 'Spectra':
259 259 if (minIndex < 0) or (minIndex > maxIndex):
260 260 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (
261 261 minIndex, maxIndex))
262 262
263 263 if (maxIndex >= self.dataOut.nHeights):
264 264 maxIndex = self.dataOut.nHeights - 1
265 265
266 266 # Spectra
267 267 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
268 268
269 269 data_cspc = None
270 270 if self.dataOut.data_cspc is not None:
271 271 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
272 272
273 273 data_dc = None
274 274 if self.dataOut.data_dc is not None:
275 275 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
276 276
277 277 self.dataOut.data_spc = data_spc
278 278 self.dataOut.data_cspc = data_cspc
279 279 self.dataOut.data_dc = data_dc
280 280
281 281 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
282 282
283 283 return 1
284 284
285 285
286 286 class filterByHeights(Operation):
287 287
288 288 def run(self, dataOut, window):
289 289
290 290 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
291 291
292 292 if window == None:
293 293 window = (dataOut.radarControllerHeaderObj.txA/dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
294 294
295 295 newdelta = deltaHeight * window
296 296 r = dataOut.nHeights % window
297 297 newheights = (dataOut.nHeights-r)/window
298 298
299 299 if newheights <= 1:
300 300 raise ValueError("filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(dataOut.nHeights, window))
301 301
302 302 if dataOut.flagDataAsBlock:
303 303 """
304 304 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
305 305 """
306 306 buffer = dataOut.data[:, :, 0:int(dataOut.nHeights-r)]
307 307 buffer = buffer.reshape(dataOut.nChannels, dataOut.nProfiles, int(dataOut.nHeights/window), window)
308 308 buffer = numpy.sum(buffer,3)
309 309
310 310 else:
311 311 buffer = dataOut.data[:,0:int(dataOut.nHeights-r)]
312 312 buffer = buffer.reshape(dataOut.nChannels,int(dataOut.nHeights/window),int(window))
313 313 buffer = numpy.sum(buffer,2)
314 314
315 315 dataOut.data = buffer
316 316 dataOut.heightList = dataOut.heightList[0] + numpy.arange( newheights )*newdelta
317 317 dataOut.windowOfFilter = window
318 318
319 319 return dataOut
320 320
321 321
322 322 class setH0(Operation):
323 323
324 324 def run(self, dataOut, h0, deltaHeight = None):
325 325
326 326 if not deltaHeight:
327 327 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
328 328
329 329 nHeights = dataOut.nHeights
330 330
331 331 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
332 332
333 333 dataOut.heightList = newHeiRange
334 334
335 335 return dataOut
336 336
337 337
338 338 class deFlip(Operation):
339 339
340 340 def run(self, dataOut, channelList = []):
341 341
342 342 data = dataOut.data.copy()
343 343
344 344 if dataOut.flagDataAsBlock:
345 345 flip = self.flip
346 346 profileList = list(range(dataOut.nProfiles))
347 347
348 348 if not channelList:
349 349 for thisProfile in profileList:
350 350 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
351 351 flip *= -1.0
352 352 else:
353 353 for thisChannel in channelList:
354 354 if thisChannel not in dataOut.channelList:
355 355 continue
356 356
357 357 for thisProfile in profileList:
358 358 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
359 359 flip *= -1.0
360 360
361 361 self.flip = flip
362 362
363 363 else:
364 364 if not channelList:
365 365 data[:,:] = data[:,:]*self.flip
366 366 else:
367 367 for thisChannel in channelList:
368 368 if thisChannel not in dataOut.channelList:
369 369 continue
370 370
371 371 data[thisChannel,:] = data[thisChannel,:]*self.flip
372 372
373 373 self.flip *= -1.
374 374
375 375 dataOut.data = data
376 376
377 377 return dataOut
378 378
379 379
380 380 class setAttribute(Operation):
381 381 '''
382 382 Set an arbitrary attribute(s) to dataOut
383 383 '''
384 384
385 385 def __init__(self):
386 386
387 387 Operation.__init__(self)
388 388 self._ready = False
389 389
390 390 def run(self, dataOut, **kwargs):
391 391
392 392 for key, value in kwargs.items():
393 393 setattr(dataOut, key, value)
394 394
395 395 return dataOut
396 396
397 397
398 398 @MPDecorator
399 399 class printAttribute(Operation):
400 400 '''
401 401 Print an arbitrary attribute of dataOut
402 402 '''
403 403
404 404 def __init__(self):
405 405
406 406 Operation.__init__(self)
407 407
408 408 def run(self, dataOut, attributes):
409 409
410 410 if isinstance(attributes, str):
411 411 attributes = [attributes]
412 412 for attr in attributes:
413 413 if hasattr(dataOut, attr):
414 414 log.log(getattr(dataOut, attr), attr)
415 415
416 416 class cleanHeightsInterf(Operation):
417 __slots__ =('heights_indx', 'repeats', 'step', 'factor', 'idate', 'idxs','config','wMask')
418 def __init__(self):
419 self.repeats = 0
420 self.factor=1
421 self.wMask = None
422 self.config = False
423 self.idxs = None
424 self.heights_indx = None
417 425
418 426 def run(self, dataOut, heightsList, repeats=0, step=0, factor=1, idate=None, startH=None, endH=None):
419 427
420 428 #print(dataOut.data.shape)
421 429
422 430 startTime = datetime.datetime.combine(idate,startH)
423 431 endTime = datetime.datetime.combine(idate,endH)
424 432 currentTime = datetime.datetime.fromtimestamp(dataOut.utctime)
425 433
426 434 if currentTime < startTime or currentTime > endTime:
427 435 return dataOut
436 if not self.config:
428 437
429 wMask = numpy.asarray(factor)
430 wMask = numpy.tile(wMask,(repeats+2))
431 438 #print(wMask)
432 439 heights = [float(hei) for hei in heightsList]
433 440 for r in range(repeats):
434 441 heights += [ (h+(step*(r+1))) for h in heights]
435 442 #print(heights)
436 443 heiList = dataOut.heightList
437 heights_indx = [getHei_index(h,h,heiList)[0] for h in heights]
444 self.heights_indx = [getHei_index(h,h,heiList)[0] for h in heights]
445
446 self.wMask = numpy.asarray(factor)
447 self.wMask = numpy.tile(self.wMask,(repeats+2))
448 self.config = True
449
450 """
451 getNoisebyHildebrand(self, channel=None, ymin_index=None, ymax_index=None)
452 """
453 #print(self.noise =10*numpy.log10(dataOut.getNoisebyHildebrand(ymin_index=self.min_ref, ymax_index=self.max_ref)))
454
438 455
439 456 for ch in range(dataOut.data.shape[0]):
440 457 i = 0
441 for hei in heights_indx:
442 #print(dataOut.data[ch,hei])
458
459
460 for hei in self.heights_indx:
461 h = hei - 1
462
463
443 464 if dataOut.data.ndim < 3:
444 dataOut.data[ch,hei-1] = (dataOut.data[ch,hei-1])*wMask[i]
465 module = numpy.absolute(dataOut.data[ch,h])
466 prev_h1 = numpy.absolute(dataOut.data[ch,h-1])
467 dataOut.data[ch,h] = (dataOut.data[ch,h])/module * prev_h1
468
469 #dataOut.data[ch,hei-1] = (dataOut.data[ch,hei-1])*self.wMask[i]
445 470 else:
446 dataOut.data[ch,:,hei-1] = (dataOut.data[ch,:,hei-1])*wMask[i]
471 module = numpy.absolute(dataOut.data[ch,:,h])
472 prev_h1 = numpy.absolute(dataOut.data[ch,:,h-1])
473 dataOut.data[ch,:,h] = (dataOut.data[ch,:,h])/module * prev_h1
474 #dataOut.data[ch,:,hei-1] = (dataOut.data[ch,:,hei-1])*self.wMask[i]
447 475 #print("done")
448 476 i += 1
449 477
478
450 479 return dataOut
451 480
452 481
482
453 483 class interpolateHeights(Operation):
454 484
455 485 def run(self, dataOut, topLim, botLim):
456 486 #69 al 72 para julia
457 487 #82-84 para meteoros
458 488 if len(numpy.shape(dataOut.data))==2:
459 489 sampInterp = (dataOut.data[:,botLim-1] + dataOut.data[:,topLim+1])/2
460 490 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
461 491 #dataOut.data[:,botLim:limSup+1] = sampInterp
462 492 dataOut.data[:,botLim:topLim+1] = sampInterp
463 493 else:
464 494 nHeights = dataOut.data.shape[2]
465 495 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
466 496 y = dataOut.data[:,:,list(range(botLim))+list(range(topLim+1,nHeights))]
467 497 f = interpolate.interp1d(x, y, axis = 2)
468 498 xnew = numpy.arange(botLim,topLim+1)
469 499 ynew = f(xnew)
470 500 dataOut.data[:,:,botLim:topLim+1] = ynew
471 501
472 502 return dataOut
473 503
474 504
475 505 class CohInt(Operation):
476 506
477 507 isConfig = False
478 508 __profIndex = 0
479 509 __byTime = False
480 510 __initime = None
481 511 __lastdatatime = None
482 512 __integrationtime = None
483 513 __buffer = None
484 514 __bufferStride = []
485 515 __dataReady = False
486 516 __profIndexStride = 0
487 517 __dataToPutStride = False
488 518 n = None
489 519
490 520 def __init__(self, **kwargs):
491 521
492 522 Operation.__init__(self, **kwargs)
493 523
494 524 def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False):
495 525 """
496 526 Set the parameters of the integration class.
497 527
498 528 Inputs:
499 529
500 530 n : Number of coherent integrations
501 531 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
502 532 overlapping :
503 533 """
504 534
505 535 self.__initime = None
506 536 self.__lastdatatime = 0
507 537 self.__buffer = None
508 538 self.__dataReady = False
509 539 self.byblock = byblock
510 540 self.stride = stride
511 541
512 542 if n == None and timeInterval == None:
513 543 raise ValueError("n or timeInterval should be specified ...")
514 544
515 545 if n != None:
516 546 self.n = n
517 547 self.__byTime = False
518 548 else:
519 549 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
520 550 self.n = 9999
521 551 self.__byTime = True
522 552
523 553 if overlapping:
524 554 self.__withOverlapping = True
525 555 self.__buffer = None
526 556 else:
527 557 self.__withOverlapping = False
528 558 self.__buffer = 0
529 559
530 560 self.__profIndex = 0
531 561
532 562 def putData(self, data):
533 563
534 564 """
535 565 Add a profile to the __buffer and increase in one the __profileIndex
536 566
537 567 """
538 568
539 569 if not self.__withOverlapping:
540 570 self.__buffer += data.copy()
541 571 self.__profIndex += 1
542 572 return
543 573
544 574 #Overlapping data
545 575 nChannels, nHeis = data.shape
546 576 data = numpy.reshape(data, (1, nChannels, nHeis))
547 577
548 578 #If the buffer is empty then it takes the data value
549 579 if self.__buffer is None:
550 580 self.__buffer = data
551 581 self.__profIndex += 1
552 582 return
553 583
554 584 #If the buffer length is lower than n then stakcing the data value
555 585 if self.__profIndex < self.n:
556 586 self.__buffer = numpy.vstack((self.__buffer, data))
557 587 self.__profIndex += 1
558 588 return
559 589
560 590 #If the buffer length is equal to n then replacing the last buffer value with the data value
561 591 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
562 592 self.__buffer[self.n-1] = data
563 593 self.__profIndex = self.n
564 594 return
565 595
566 596
567 597 def pushData(self):
568 598 """
569 599 Return the sum of the last profiles and the profiles used in the sum.
570 600
571 601 Affected:
572 602
573 603 self.__profileIndex
574 604
575 605 """
576 606
577 607 if not self.__withOverlapping:
578 608 data = self.__buffer
579 609 n = self.__profIndex
580 610
581 611 self.__buffer = 0
582 612 self.__profIndex = 0
583 613
584 614 return data, n
585 615
586 616 #Integration with Overlapping
587 617 data = numpy.sum(self.__buffer, axis=0)
588 618 # print data
589 619 # raise
590 620 n = self.__profIndex
591 621
592 622 return data, n
593 623
594 624 def byProfiles(self, data):
595 625
596 626 self.__dataReady = False
597 627 avgdata = None
598 628 # n = None
599 629 # print data
600 630 # raise
601 631 self.putData(data)
602 632
603 633 if self.__profIndex == self.n:
604 634 avgdata, n = self.pushData()
605 635 self.__dataReady = True
606 636
607 637 return avgdata
608 638
609 639 def byTime(self, data, datatime):
610 640
611 641 self.__dataReady = False
612 642 avgdata = None
613 643 n = None
614 644
615 645 self.putData(data)
616 646
617 647 if (datatime - self.__initime) >= self.__integrationtime:
618 648 avgdata, n = self.pushData()
619 649 self.n = n
620 650 self.__dataReady = True
621 651
622 652 return avgdata
623 653
624 654 def integrateByStride(self, data, datatime):
625 655 # print data
626 656 if self.__profIndex == 0:
627 657 self.__buffer = [[data.copy(), datatime]]
628 658 else:
629 659 self.__buffer.append([data.copy(),datatime])
630 660 self.__profIndex += 1
631 661 self.__dataReady = False
632 662
633 663 if self.__profIndex == self.n * self.stride :
634 664 self.__dataToPutStride = True
635 665 self.__profIndexStride = 0
636 666 self.__profIndex = 0
637 667 self.__bufferStride = []
638 668 for i in range(self.stride):
639 669 current = self.__buffer[i::self.stride]
640 670 data = numpy.sum([t[0] for t in current], axis=0)
641 671 avgdatatime = numpy.average([t[1] for t in current])
642 672 # print data
643 673 self.__bufferStride.append((data, avgdatatime))
644 674
645 675 if self.__dataToPutStride:
646 676 self.__dataReady = True
647 677 self.__profIndexStride += 1
648 678 if self.__profIndexStride == self.stride:
649 679 self.__dataToPutStride = False
650 680 # print self.__bufferStride[self.__profIndexStride - 1]
651 681 # raise
652 682 return self.__bufferStride[self.__profIndexStride - 1]
653 683
654 684
655 685 return None, None
656 686
657 687 def integrate(self, data, datatime=None):
658 688
659 689 if self.__initime == None:
660 690 self.__initime = datatime
661 691
662 692 if self.__byTime:
663 693 avgdata = self.byTime(data, datatime)
664 694 else:
665 695 avgdata = self.byProfiles(data)
666 696
667 697
668 698 self.__lastdatatime = datatime
669 699
670 700 if avgdata is None:
671 701 return None, None
672 702
673 703 avgdatatime = self.__initime
674 704
675 705 deltatime = datatime - self.__lastdatatime
676 706
677 707 if not self.__withOverlapping:
678 708 self.__initime = datatime
679 709 else:
680 710 self.__initime += deltatime
681 711
682 712 return avgdata, avgdatatime
683 713
684 714 def integrateByBlock(self, dataOut):
685 715
686 716 times = int(dataOut.data.shape[1]/self.n)
687 717 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
688 718
689 719 id_min = 0
690 720 id_max = self.n
691 721
692 722 for i in range(times):
693 723 junk = dataOut.data[:,id_min:id_max,:]
694 724 avgdata[:,i,:] = junk.sum(axis=1)
695 725 id_min += self.n
696 726 id_max += self.n
697 727
698 728 timeInterval = dataOut.ippSeconds*self.n
699 729 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
700 730 self.__dataReady = True
701 731 return avgdata, avgdatatime
702 732
703 733 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
704 734
705 735 if not self.isConfig:
706 736 self.setup(n=n, stride=stride, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
707 737 self.isConfig = True
708 738
709 739 if dataOut.flagDataAsBlock:
710 740 """
711 741 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
712 742 """
713 743 avgdata, avgdatatime = self.integrateByBlock(dataOut)
714 744 dataOut.nProfiles /= self.n
715 745 else:
716 746 if stride is None:
717 747 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
718 748 else:
719 749 avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
720 750
721 751
722 752 # dataOut.timeInterval *= n
723 753 dataOut.flagNoData = True
724 754
725 755 if self.__dataReady:
726 756 dataOut.data = avgdata
727 757 if not dataOut.flagCohInt:
728 758 dataOut.nCohInt *= self.n
729 759 dataOut.flagCohInt = True
730 760 dataOut.utctime = avgdatatime
731 761 # print avgdata, avgdatatime
732 762 # raise
733 763 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
734 764 dataOut.flagNoData = False
735 765 return dataOut
736 766
737 767 class Decoder(Operation):
738 768
739 769 isConfig = False
740 770 __profIndex = 0
741 771
742 772 code = None
743 773
744 774 nCode = None
745 775 nBaud = None
746 776
747 777 def __init__(self, **kwargs):
748 778
749 779 Operation.__init__(self, **kwargs)
750 780
751 781 self.times = None
752 782 self.osamp = None
753 783 # self.__setValues = False
754 784 self.isConfig = False
755 785 self.setupReq = False
756 786 def setup(self, code, osamp, dataOut):
757 787
758 788 self.__profIndex = 0
759 789
760 790 self.code = code
761 791
762 792 self.nCode = len(code)
763 793 self.nBaud = len(code[0])
764 794 if (osamp != None) and (osamp >1):
765 795 self.osamp = osamp
766 796 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
767 797 self.nBaud = self.nBaud*self.osamp
768 798
769 799 self.__nChannels = dataOut.nChannels
770 800 self.__nProfiles = dataOut.nProfiles
771 801 self.__nHeis = dataOut.nHeights
772 802
773 803 if self.__nHeis < self.nBaud:
774 804 raise ValueError('Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud))
775 805
776 806 #Frequency
777 807 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
778 808
779 809 __codeBuffer[:,0:self.nBaud] = self.code
780 810
781 811 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
782 812
783 813 if dataOut.flagDataAsBlock:
784 814
785 815 self.ndatadec = self.__nHeis #- self.nBaud + 1
786 816
787 817 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
788 818
789 819 else:
790 820
791 821 #Time
792 822 self.ndatadec = self.__nHeis #- self.nBaud + 1
793 823
794 824 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
795 825
796 826 def __convolutionInFreq(self, data):
797 827
798 828 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
799 829
800 830 fft_data = numpy.fft.fft(data, axis=1)
801 831
802 832 conv = fft_data*fft_code
803 833
804 834 data = numpy.fft.ifft(conv,axis=1)
805 835
806 836 return data
807 837
808 838 def __convolutionInFreqOpt(self, data):
809 839
810 840 raise NotImplementedError
811 841
812 842 def __convolutionInTime(self, data):
813 843
814 844 code = self.code[self.__profIndex]
815 845 for i in range(self.__nChannels):
816 846 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
817 847
818 848 return self.datadecTime
819 849
820 850 def __convolutionByBlockInTime(self, data):
821 851
822 852 repetitions = int(self.__nProfiles / self.nCode)
823 853 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
824 854 junk = junk.flatten()
825 855 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
826 856 profilesList = range(self.__nProfiles)
827 857
828 858 for i in range(self.__nChannels):
829 859 for j in profilesList:
830 860 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
831 861 return self.datadecTime
832 862
833 863 def __convolutionByBlockInFreq(self, data):
834 864
835 865 raise NotImplementedError("Decoder by frequency fro Blocks not implemented")
836 866
837 867
838 868 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
839 869
840 870 fft_data = numpy.fft.fft(data, axis=2)
841 871
842 872 conv = fft_data*fft_code
843 873
844 874 data = numpy.fft.ifft(conv,axis=2)
845 875
846 876 return data
847 877
848 878
849 879 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
850 880
851 881 if dataOut.flagDecodeData:
852 882 print("This data is already decoded, recoding again ...")
853 883
854 884 if not self.isConfig:
855 885
856 886 if code is None:
857 887 if dataOut.code is None:
858 888 raise ValueError("Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type)
859 889
860 890 code = dataOut.code
861 891 else:
862 892 code = numpy.array(code).reshape(nCode,nBaud)
863 893 self.setup(code, osamp, dataOut)
864 894
865 895 self.isConfig = True
866 896
867 897 if mode == 3:
868 898 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
869 899
870 900 if times != None:
871 901 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
872 902
873 903 if self.code is None:
874 904 print("Fail decoding: Code is not defined.")
875 905 return
876 906
877 907 self.__nProfiles = dataOut.nProfiles
878 908 datadec = None
879 909
880 910 if mode == 3:
881 911 mode = 0
882 912
883 913 if dataOut.flagDataAsBlock:
884 914 """
885 915 Decoding when data have been read as block,
886 916 """
887 917
888 918 if mode == 0:
889 919 datadec = self.__convolutionByBlockInTime(dataOut.data)
890 920 if mode == 1:
891 921 datadec = self.__convolutionByBlockInFreq(dataOut.data)
892 922 else:
893 923 """
894 924 Decoding when data have been read profile by profile
895 925 """
896 926 if mode == 0:
897 927 datadec = self.__convolutionInTime(dataOut.data)
898 928
899 929 if mode == 1:
900 930 datadec = self.__convolutionInFreq(dataOut.data)
901 931
902 932 if mode == 2:
903 933 datadec = self.__convolutionInFreqOpt(dataOut.data)
904 934
905 935 if datadec is None:
906 936 raise ValueError("Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode)
907 937
908 938 dataOut.code = self.code
909 939 dataOut.nCode = self.nCode
910 940 dataOut.nBaud = self.nBaud
911 941
912 942 dataOut.data = datadec
913 943
914 944 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
915 945
916 946 dataOut.flagDecodeData = True #asumo q la data esta decodificada
917 947
918 948 if self.__profIndex == self.nCode-1:
919 949 self.__profIndex = 0
920 950 return dataOut
921 951
922 952 self.__profIndex += 1
923 953
924 954 return dataOut
925 955 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
926 956
927 957
928 958 class ProfileConcat(Operation):
929 959
930 960 isConfig = False
931 961 buffer = None
932 962
933 963 def __init__(self, **kwargs):
934 964
935 965 Operation.__init__(self, **kwargs)
936 966 self.profileIndex = 0
937 967
938 968 def reset(self):
939 969 self.buffer = numpy.zeros_like(self.buffer)
940 970 self.start_index = 0
941 971 self.times = 1
942 972
943 973 def setup(self, data, m, n=1):
944 974 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
945 975 self.nHeights = data.shape[1]#.nHeights
946 976 self.start_index = 0
947 977 self.times = 1
948 978
949 979 def concat(self, data):
950 980
951 981 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
952 982 self.start_index = self.start_index + self.nHeights
953 983
954 984 def run(self, dataOut, m):
955 985 dataOut.flagNoData = True
956 986
957 987 if not self.isConfig:
958 988 self.setup(dataOut.data, m, 1)
959 989 self.isConfig = True
960 990
961 991 if dataOut.flagDataAsBlock:
962 992 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
963 993
964 994 else:
965 995 self.concat(dataOut.data)
966 996 self.times += 1
967 997 if self.times > m:
968 998 dataOut.data = self.buffer
969 999 self.reset()
970 1000 dataOut.flagNoData = False
971 1001 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
972 1002 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
973 1003 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
974 1004 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
975 1005 dataOut.ippSeconds *= m
976 1006 return dataOut
977 1007
978 1008 class ProfileSelector(Operation):
979 1009
980 1010 profileIndex = None
981 1011 # Tamanho total de los perfiles
982 1012 nProfiles = None
983 1013
984 1014 def __init__(self, **kwargs):
985 1015
986 1016 Operation.__init__(self, **kwargs)
987 1017 self.profileIndex = 0
988 1018
989 1019 def incProfileIndex(self):
990 1020
991 1021 self.profileIndex += 1
992 1022
993 1023 if self.profileIndex >= self.nProfiles:
994 1024 self.profileIndex = 0
995 1025
996 1026 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
997 1027
998 1028 if profileIndex < minIndex:
999 1029 return False
1000 1030
1001 1031 if profileIndex > maxIndex:
1002 1032 return False
1003 1033
1004 1034 return True
1005 1035
1006 1036 def isThisProfileInList(self, profileIndex, profileList):
1007 1037
1008 1038 if profileIndex not in profileList:
1009 1039 return False
1010 1040
1011 1041 return True
1012 1042
1013 1043 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
1014 1044
1015 1045 """
1016 1046 ProfileSelector:
1017 1047
1018 1048 Inputs:
1019 1049 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
1020 1050
1021 1051 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
1022 1052
1023 1053 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
1024 1054
1025 1055 """
1026 1056
1027 1057 if rangeList is not None:
1028 1058 if type(rangeList[0]) not in (tuple, list):
1029 1059 rangeList = [rangeList]
1030 1060
1031 1061 dataOut.flagNoData = True
1032 1062
1033 1063 if dataOut.flagDataAsBlock:
1034 1064 """
1035 1065 data dimension = [nChannels, nProfiles, nHeis]
1036 1066 """
1037 1067 if profileList != None:
1038 1068 dataOut.data = dataOut.data[:,profileList,:]
1039 1069
1040 1070 if profileRangeList != None:
1041 1071 minIndex = profileRangeList[0]
1042 1072 maxIndex = profileRangeList[1]
1043 1073 profileList = list(range(minIndex, maxIndex+1))
1044 1074
1045 1075 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
1046 1076
1047 1077 if rangeList != None:
1048 1078
1049 1079 profileList = []
1050 1080
1051 1081 for thisRange in rangeList:
1052 1082 minIndex = thisRange[0]
1053 1083 maxIndex = thisRange[1]
1054 1084
1055 1085 profileList.extend(list(range(minIndex, maxIndex+1)))
1056 1086
1057 1087 dataOut.data = dataOut.data[:,profileList,:]
1058 1088
1059 1089 dataOut.nProfiles = len(profileList)
1060 1090 dataOut.profileIndex = dataOut.nProfiles - 1
1061 1091 dataOut.flagNoData = False
1062 1092
1063 1093 return dataOut
1064 1094
1065 1095 """
1066 1096 data dimension = [nChannels, nHeis]
1067 1097 """
1068 1098
1069 1099 if profileList != None:
1070 1100
1071 1101 if self.isThisProfileInList(dataOut.profileIndex, profileList):
1072 1102
1073 1103 self.nProfiles = len(profileList)
1074 1104 dataOut.nProfiles = self.nProfiles
1075 1105 dataOut.profileIndex = self.profileIndex
1076 1106 dataOut.flagNoData = False
1077 1107
1078 1108 self.incProfileIndex()
1079 1109 return dataOut
1080 1110
1081 1111 if profileRangeList != None:
1082 1112
1083 1113 minIndex = profileRangeList[0]
1084 1114 maxIndex = profileRangeList[1]
1085 1115
1086 1116 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1087 1117
1088 1118 self.nProfiles = maxIndex - minIndex + 1
1089 1119 dataOut.nProfiles = self.nProfiles
1090 1120 dataOut.profileIndex = self.profileIndex
1091 1121 dataOut.flagNoData = False
1092 1122
1093 1123 self.incProfileIndex()
1094 1124 return dataOut
1095 1125
1096 1126 if rangeList != None:
1097 1127
1098 1128 nProfiles = 0
1099 1129
1100 1130 for thisRange in rangeList:
1101 1131 minIndex = thisRange[0]
1102 1132 maxIndex = thisRange[1]
1103 1133
1104 1134 nProfiles += maxIndex - minIndex + 1
1105 1135
1106 1136 for thisRange in rangeList:
1107 1137
1108 1138 minIndex = thisRange[0]
1109 1139 maxIndex = thisRange[1]
1110 1140
1111 1141 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1112 1142
1113 1143 self.nProfiles = nProfiles
1114 1144 dataOut.nProfiles = self.nProfiles
1115 1145 dataOut.profileIndex = self.profileIndex
1116 1146 dataOut.flagNoData = False
1117 1147
1118 1148 self.incProfileIndex()
1119 1149
1120 1150 break
1121 1151
1122 1152 return dataOut
1123 1153
1124 1154
1125 1155 if beam != None: #beam is only for AMISR data
1126 1156 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
1127 1157 dataOut.flagNoData = False
1128 1158 dataOut.profileIndex = self.profileIndex
1129 1159
1130 1160 self.incProfileIndex()
1131 1161
1132 1162 return dataOut
1133 1163
1134 1164 raise ValueError("ProfileSelector needs profileList, profileRangeList or rangeList parameter")
1135 1165
1136 1166
1137 1167 class Reshaper(Operation):
1138 1168
1139 1169 def __init__(self, **kwargs):
1140 1170
1141 1171 Operation.__init__(self, **kwargs)
1142 1172
1143 1173 self.__buffer = None
1144 1174 self.__nitems = 0
1145 1175
1146 1176 def __appendProfile(self, dataOut, nTxs):
1147 1177
1148 1178 if self.__buffer is None:
1149 1179 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
1150 1180 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
1151 1181
1152 1182 ini = dataOut.nHeights * self.__nitems
1153 1183 end = ini + dataOut.nHeights
1154 1184
1155 1185 self.__buffer[:, ini:end] = dataOut.data
1156 1186
1157 1187 self.__nitems += 1
1158 1188
1159 1189 return int(self.__nitems*nTxs)
1160 1190
1161 1191 def __getBuffer(self):
1162 1192
1163 1193 if self.__nitems == int(1./self.__nTxs):
1164 1194
1165 1195 self.__nitems = 0
1166 1196
1167 1197 return self.__buffer.copy()
1168 1198
1169 1199 return None
1170 1200
1171 1201 def __checkInputs(self, dataOut, shape, nTxs):
1172 1202
1173 1203 if shape is None and nTxs is None:
1174 1204 raise ValueError("Reshaper: shape of factor should be defined")
1175 1205
1176 1206 if nTxs:
1177 1207 if nTxs < 0:
1178 1208 raise ValueError("nTxs should be greater than 0")
1179 1209
1180 1210 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
1181 1211 raise ValueError("nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs)))
1182 1212
1183 1213 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
1184 1214
1185 1215 return shape, nTxs
1186 1216
1187 1217 if len(shape) != 2 and len(shape) != 3:
1188 1218 raise ValueError("shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights))
1189 1219
1190 1220 if len(shape) == 2:
1191 1221 shape_tuple = [dataOut.nChannels]
1192 1222 shape_tuple.extend(shape)
1193 1223 else:
1194 1224 shape_tuple = list(shape)
1195 1225
1196 1226 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1197 1227
1198 1228 return shape_tuple, nTxs
1199 1229
1200 1230 def run(self, dataOut, shape=None, nTxs=None):
1201 1231
1202 1232 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1203 1233
1204 1234 dataOut.flagNoData = True
1205 1235 profileIndex = None
1206 1236
1207 1237 if dataOut.flagDataAsBlock:
1208 1238
1209 1239 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1210 1240 dataOut.flagNoData = False
1211 1241
1212 1242 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1213 1243
1214 1244 else:
1215 1245
1216 1246 if self.__nTxs < 1:
1217 1247
1218 1248 self.__appendProfile(dataOut, self.__nTxs)
1219 1249 new_data = self.__getBuffer()
1220 1250
1221 1251 if new_data is not None:
1222 1252 dataOut.data = new_data
1223 1253 dataOut.flagNoData = False
1224 1254
1225 1255 profileIndex = dataOut.profileIndex*nTxs
1226 1256
1227 1257 else:
1228 1258 raise ValueError("nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)")
1229 1259
1230 1260 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1231 1261
1232 1262 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1233 1263
1234 1264 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1235 1265
1236 1266 dataOut.profileIndex = profileIndex
1237 1267
1238 1268 dataOut.ippSeconds /= self.__nTxs
1239 1269
1240 1270 return dataOut
1241 1271
1242 1272 class SplitProfiles(Operation):
1243 1273
1244 1274 def __init__(self, **kwargs):
1245 1275
1246 1276 Operation.__init__(self, **kwargs)
1247 1277
1248 1278 def run(self, dataOut, n):
1249 1279
1250 1280 dataOut.flagNoData = True
1251 1281 profileIndex = None
1252 1282
1253 1283 if dataOut.flagDataAsBlock:
1254 1284
1255 1285 #nchannels, nprofiles, nsamples
1256 1286 shape = dataOut.data.shape
1257 1287
1258 1288 if shape[2] % n != 0:
1259 1289 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
1260 1290
1261 1291 new_shape = shape[0], shape[1]*n, int(shape[2]/n)
1262 1292
1263 1293 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1264 1294 dataOut.flagNoData = False
1265 1295
1266 1296 profileIndex = int(dataOut.nProfiles/n) - 1
1267 1297
1268 1298 else:
1269 1299
1270 1300 raise ValueError("Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)")
1271 1301
1272 1302 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1273 1303
1274 1304 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1275 1305
1276 1306 dataOut.nProfiles = int(dataOut.nProfiles*n)
1277 1307
1278 1308 dataOut.profileIndex = profileIndex
1279 1309
1280 1310 dataOut.ippSeconds /= n
1281 1311
1282 1312 return dataOut
1283 1313
1284 1314 class CombineProfiles(Operation):
1285 1315 def __init__(self, **kwargs):
1286 1316
1287 1317 Operation.__init__(self, **kwargs)
1288 1318
1289 1319 self.__remData = None
1290 1320 self.__profileIndex = 0
1291 1321
1292 1322 def run(self, dataOut, n):
1293 1323
1294 1324 dataOut.flagNoData = True
1295 1325 profileIndex = None
1296 1326
1297 1327 if dataOut.flagDataAsBlock:
1298 1328
1299 1329 #nchannels, nprofiles, nsamples
1300 1330 shape = dataOut.data.shape
1301 1331 new_shape = shape[0], shape[1]/n, shape[2]*n
1302 1332
1303 1333 if shape[1] % n != 0:
1304 1334 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[1]))
1305 1335
1306 1336 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1307 1337 dataOut.flagNoData = False
1308 1338
1309 1339 profileIndex = int(dataOut.nProfiles*n) - 1
1310 1340
1311 1341 else:
1312 1342
1313 1343 #nchannels, nsamples
1314 1344 if self.__remData is None:
1315 1345 newData = dataOut.data
1316 1346 else:
1317 1347 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1318 1348
1319 1349 self.__profileIndex += 1
1320 1350
1321 1351 if self.__profileIndex < n:
1322 1352 self.__remData = newData
1323 1353 #continue
1324 1354 return
1325 1355
1326 1356 self.__profileIndex = 0
1327 1357 self.__remData = None
1328 1358
1329 1359 dataOut.data = newData
1330 1360 dataOut.flagNoData = False
1331 1361
1332 1362 profileIndex = dataOut.profileIndex/n
1333 1363
1334 1364
1335 1365 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1336 1366
1337 1367 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1338 1368
1339 1369 dataOut.nProfiles = int(dataOut.nProfiles/n)
1340 1370
1341 1371 dataOut.profileIndex = profileIndex
1342 1372
1343 1373 dataOut.ippSeconds *= n
1344 1374
1345 1375 return dataOut
1346 1376
1347 1377 class PulsePairVoltage(Operation):
1348 1378 '''
1349 1379 Function PulsePair(Signal Power, Velocity)
1350 1380 The real component of Lag[0] provides Intensity Information
1351 1381 The imag component of Lag[1] Phase provides Velocity Information
1352 1382
1353 1383 Configuration Parameters:
1354 1384 nPRF = Number of Several PRF
1355 1385 theta = Degree Azimuth angel Boundaries
1356 1386
1357 1387 Input:
1358 1388 self.dataOut
1359 1389 lag[N]
1360 1390 Affected:
1361 1391 self.dataOut.spc
1362 1392 '''
1363 1393 isConfig = False
1364 1394 __profIndex = 0
1365 1395 __initime = None
1366 1396 __lastdatatime = None
1367 1397 __buffer = None
1368 1398 noise = None
1369 1399 __dataReady = False
1370 1400 n = None
1371 1401 __nch = 0
1372 1402 __nHeis = 0
1373 1403 removeDC = False
1374 1404 ipp = None
1375 1405 lambda_ = 0
1376 1406
1377 1407 def __init__(self,**kwargs):
1378 1408 Operation.__init__(self,**kwargs)
1379 1409
1380 1410 def setup(self, dataOut, n = None, removeDC=False):
1381 1411 '''
1382 1412 n= Numero de PRF's de entrada
1383 1413 '''
1384 1414 self.__initime = None
1385 1415 self.__lastdatatime = 0
1386 1416 self.__dataReady = False
1387 1417 self.__buffer = 0
1388 1418 self.__profIndex = 0
1389 1419 self.noise = None
1390 1420 self.__nch = dataOut.nChannels
1391 1421 self.__nHeis = dataOut.nHeights
1392 1422 self.removeDC = removeDC
1393 1423 self.lambda_ = 3.0e8/(9345.0e6)
1394 1424 self.ippSec = dataOut.ippSeconds
1395 1425 self.nCohInt = dataOut.nCohInt
1396 1426
1397 1427 if n == None:
1398 1428 raise ValueError("n should be specified.")
1399 1429
1400 1430 if n != None:
1401 1431 if n<2:
1402 1432 raise ValueError("n should be greater than 2")
1403 1433
1404 1434 self.n = n
1405 1435 self.__nProf = n
1406 1436
1407 1437 self.__buffer = numpy.zeros((dataOut.nChannels,
1408 1438 n,
1409 1439 dataOut.nHeights),
1410 1440 dtype='complex')
1411 1441
1412 1442 def putData(self,data):
1413 1443 '''
1414 1444 Add a profile to he __buffer and increase in one the __profiel Index
1415 1445 '''
1416 1446 self.__buffer[:,self.__profIndex,:]= data
1417 1447 self.__profIndex += 1
1418 1448 return
1419 1449
1420 1450 def pushData(self,dataOut):
1421 1451 '''
1422 1452 Return the PULSEPAIR and the profiles used in the operation
1423 1453 Affected : self.__profileIndex
1424 1454 '''
1425 1455 #----------------- Remove DC-----------------------------------
1426 1456 if self.removeDC==True:
1427 1457 mean = numpy.mean(self.__buffer,1)
1428 1458 tmp = mean.reshape(self.__nch,1,self.__nHeis)
1429 1459 dc= numpy.tile(tmp,[1,self.__nProf,1])
1430 1460 self.__buffer = self.__buffer - dc
1431 1461 #------------------Calculo de Potencia ------------------------
1432 1462 pair0 = self.__buffer*numpy.conj(self.__buffer)
1433 1463 pair0 = pair0.real
1434 1464 lag_0 = numpy.sum(pair0,1)
1435 1465 #------------------Calculo de Ruido x canal--------------------
1436 1466 self.noise = numpy.zeros(self.__nch)
1437 1467 for i in range(self.__nch):
1438 1468 daux = numpy.sort(pair0[i,:,:],axis= None)
1439 1469 self.noise[i]=hildebrand_sekhon( daux ,self.nCohInt)
1440 1470
1441 1471 self.noise = self.noise.reshape(self.__nch,1)
1442 1472 self.noise = numpy.tile(self.noise,[1,self.__nHeis])
1443 1473 noise_buffer = self.noise.reshape(self.__nch,1,self.__nHeis)
1444 1474 noise_buffer = numpy.tile(noise_buffer,[1,self.__nProf,1])
1445 1475 #------------------ Potencia recibida= P , Potencia senal = S , Ruido= N--
1446 1476 #------------------ P= S+N ,P=lag_0/N ---------------------------------
1447 1477 #-------------------- Power --------------------------------------------------
1448 1478 data_power = lag_0/(self.n*self.nCohInt)
1449 1479 #------------------ Senal ---------------------------------------------------
1450 1480 data_intensity = pair0 - noise_buffer
1451 1481 data_intensity = numpy.sum(data_intensity,axis=1)*(self.n*self.nCohInt)#*self.nCohInt)
1452 1482 #data_intensity = (lag_0-self.noise*self.n)*(self.n*self.nCohInt)
1453 1483 for i in range(self.__nch):
1454 1484 for j in range(self.__nHeis):
1455 1485 if data_intensity[i][j] < 0:
1456 1486 data_intensity[i][j] = numpy.min(numpy.absolute(data_intensity[i][j]))
1457 1487
1458 1488 #----------------- Calculo de Frecuencia y Velocidad doppler--------
1459 1489 pair1 = self.__buffer[:,:-1,:]*numpy.conjugate(self.__buffer[:,1:,:])
1460 1490 lag_1 = numpy.sum(pair1,1)
1461 1491 data_freq = (-1/(2.0*math.pi*self.ippSec*self.nCohInt))*numpy.angle(lag_1)
1462 1492 data_velocity = (self.lambda_/2.0)*data_freq
1463 1493
1464 1494 #---------------- Potencia promedio estimada de la Senal-----------
1465 1495 lag_0 = lag_0/self.n
1466 1496 S = lag_0-self.noise
1467 1497
1468 1498 #---------------- Frecuencia Doppler promedio ---------------------
1469 1499 lag_1 = lag_1/(self.n-1)
1470 1500 R1 = numpy.abs(lag_1)
1471 1501
1472 1502 #---------------- Calculo del SNR----------------------------------
1473 1503 data_snrPP = S/self.noise
1474 1504 for i in range(self.__nch):
1475 1505 for j in range(self.__nHeis):
1476 1506 if data_snrPP[i][j] < 1.e-20:
1477 1507 data_snrPP[i][j] = 1.e-20
1478 1508
1479 1509 #----------------- Calculo del ancho espectral ----------------------
1480 1510 L = S/R1
1481 1511 L = numpy.where(L<0,1,L)
1482 1512 L = numpy.log(L)
1483 1513 tmp = numpy.sqrt(numpy.absolute(L))
1484 1514 data_specwidth = (self.lambda_/(2*math.sqrt(2)*math.pi*self.ippSec*self.nCohInt))*tmp*numpy.sign(L)
1485 1515 n = self.__profIndex
1486 1516
1487 1517 self.__buffer = numpy.zeros((self.__nch, self.__nProf,self.__nHeis), dtype='complex')
1488 1518 self.__profIndex = 0
1489 1519 return data_power,data_intensity,data_velocity,data_snrPP,data_specwidth,n
1490 1520
1491 1521
1492 1522 def pulsePairbyProfiles(self,dataOut):
1493 1523
1494 1524 self.__dataReady = False
1495 1525 data_power = None
1496 1526 data_intensity = None
1497 1527 data_velocity = None
1498 1528 data_specwidth = None
1499 1529 data_snrPP = None
1500 1530 self.putData(data=dataOut.data)
1501 1531 if self.__profIndex == self.n:
1502 1532 data_power,data_intensity, data_velocity,data_snrPP,data_specwidth, n = self.pushData(dataOut=dataOut)
1503 1533 self.__dataReady = True
1504 1534
1505 1535 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth
1506 1536
1507 1537
1508 1538 def pulsePairOp(self, dataOut, datatime= None):
1509 1539
1510 1540 if self.__initime == None:
1511 1541 self.__initime = datatime
1512 1542 data_power, data_intensity, data_velocity, data_snrPP, data_specwidth = self.pulsePairbyProfiles(dataOut)
1513 1543 self.__lastdatatime = datatime
1514 1544
1515 1545 if data_power is None:
1516 1546 return None, None, None,None,None,None
1517 1547
1518 1548 avgdatatime = self.__initime
1519 1549 deltatime = datatime - self.__lastdatatime
1520 1550 self.__initime = datatime
1521 1551
1522 1552 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth, avgdatatime
1523 1553
1524 1554 def run(self, dataOut,n = None,removeDC= False, overlapping= False,**kwargs):
1525 1555
1526 1556 if not self.isConfig:
1527 1557 self.setup(dataOut = dataOut, n = n , removeDC=removeDC , **kwargs)
1528 1558 self.isConfig = True
1529 1559 data_power, data_intensity, data_velocity,data_snrPP,data_specwidth, avgdatatime = self.pulsePairOp(dataOut, dataOut.utctime)
1530 1560 dataOut.flagNoData = True
1531 1561
1532 1562 if self.__dataReady:
1533 1563 dataOut.nCohInt *= self.n
1534 1564 dataOut.dataPP_POW = data_intensity # S
1535 1565 dataOut.dataPP_POWER = data_power # P
1536 1566 dataOut.dataPP_DOP = data_velocity
1537 1567 dataOut.dataPP_SNR = data_snrPP
1538 1568 dataOut.dataPP_WIDTH = data_specwidth
1539 1569 dataOut.PRFbyAngle = self.n #numero de PRF*cada angulo rotado que equivale a un tiempo.
1540 1570 dataOut.utctime = avgdatatime
1541 1571 dataOut.flagNoData = False
1542 1572 return dataOut
1543 1573
1544 1574
1545 1575
1546 1576 # import collections
1547 1577 # from scipy.stats import mode
1548 1578 #
1549 1579 # class Synchronize(Operation):
1550 1580 #
1551 1581 # isConfig = False
1552 1582 # __profIndex = 0
1553 1583 #
1554 1584 # def __init__(self, **kwargs):
1555 1585 #
1556 1586 # Operation.__init__(self, **kwargs)
1557 1587 # # self.isConfig = False
1558 1588 # self.__powBuffer = None
1559 1589 # self.__startIndex = 0
1560 1590 # self.__pulseFound = False
1561 1591 #
1562 1592 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1563 1593 #
1564 1594 # #Read data
1565 1595 #
1566 1596 # powerdB = dataOut.getPower(channel = channel)
1567 1597 # noisedB = dataOut.getNoise(channel = channel)[0]
1568 1598 #
1569 1599 # self.__powBuffer.extend(powerdB.flatten())
1570 1600 #
1571 1601 # dataArray = numpy.array(self.__powBuffer)
1572 1602 #
1573 1603 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1574 1604 #
1575 1605 # maxValue = numpy.nanmax(filteredPower)
1576 1606 #
1577 1607 # if maxValue < noisedB + 10:
1578 1608 # #No se encuentra ningun pulso de transmision
1579 1609 # return None
1580 1610 #
1581 1611 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1582 1612 #
1583 1613 # if len(maxValuesIndex) < 2:
1584 1614 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1585 1615 # return None
1586 1616 #
1587 1617 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1588 1618 #
1589 1619 # #Seleccionar solo valores con un espaciamiento de nSamples
1590 1620 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1591 1621 #
1592 1622 # if len(pulseIndex) < 2:
1593 1623 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1594 1624 # return None
1595 1625 #
1596 1626 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1597 1627 #
1598 1628 # #remover senales que se distancien menos de 10 unidades o muestras
1599 1629 # #(No deberian existir IPP menor a 10 unidades)
1600 1630 #
1601 1631 # realIndex = numpy.where(spacing > 10 )[0]
1602 1632 #
1603 1633 # if len(realIndex) < 2:
1604 1634 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1605 1635 # return None
1606 1636 #
1607 1637 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1608 1638 # realPulseIndex = pulseIndex[realIndex]
1609 1639 #
1610 1640 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1611 1641 #
1612 1642 # print "IPP = %d samples" %period
1613 1643 #
1614 1644 # self.__newNSamples = dataOut.nHeights #int(period)
1615 1645 # self.__startIndex = int(realPulseIndex[0])
1616 1646 #
1617 1647 # return 1
1618 1648 #
1619 1649 #
1620 1650 # def setup(self, nSamples, nChannels, buffer_size = 4):
1621 1651 #
1622 1652 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1623 1653 # maxlen = buffer_size*nSamples)
1624 1654 #
1625 1655 # bufferList = []
1626 1656 #
1627 1657 # for i in range(nChannels):
1628 1658 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1629 1659 # maxlen = buffer_size*nSamples)
1630 1660 #
1631 1661 # bufferList.append(bufferByChannel)
1632 1662 #
1633 1663 # self.__nSamples = nSamples
1634 1664 # self.__nChannels = nChannels
1635 1665 # self.__bufferList = bufferList
1636 1666 #
1637 1667 # def run(self, dataOut, channel = 0):
1638 1668 #
1639 1669 # if not self.isConfig:
1640 1670 # nSamples = dataOut.nHeights
1641 1671 # nChannels = dataOut.nChannels
1642 1672 # self.setup(nSamples, nChannels)
1643 1673 # self.isConfig = True
1644 1674 #
1645 1675 # #Append new data to internal buffer
1646 1676 # for thisChannel in range(self.__nChannels):
1647 1677 # bufferByChannel = self.__bufferList[thisChannel]
1648 1678 # bufferByChannel.extend(dataOut.data[thisChannel])
1649 1679 #
1650 1680 # if self.__pulseFound:
1651 1681 # self.__startIndex -= self.__nSamples
1652 1682 #
1653 1683 # #Finding Tx Pulse
1654 1684 # if not self.__pulseFound:
1655 1685 # indexFound = self.__findTxPulse(dataOut, channel)
1656 1686 #
1657 1687 # if indexFound == None:
1658 1688 # dataOut.flagNoData = True
1659 1689 # return
1660 1690 #
1661 1691 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1662 1692 # self.__pulseFound = True
1663 1693 # self.__startIndex = indexFound
1664 1694 #
1665 1695 # #If pulse was found ...
1666 1696 # for thisChannel in range(self.__nChannels):
1667 1697 # bufferByChannel = self.__bufferList[thisChannel]
1668 1698 # #print self.__startIndex
1669 1699 # x = numpy.array(bufferByChannel)
1670 1700 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1671 1701 #
1672 1702 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1673 1703 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1674 1704 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1675 1705 #
1676 1706 # dataOut.data = self.__arrayBuffer
1677 1707 #
1678 1708 # self.__startIndex += self.__newNSamples
1679 1709 #
1680 1710 # return
1681 1711 class SSheightProfiles(Operation):
1682 1712
1683 1713 step = None
1684 1714 nsamples = None
1685 1715 bufferShape = None
1686 1716 profileShape = None
1687 1717 sshProfiles = None
1688 1718 profileIndex = None
1689 1719
1690 1720 def __init__(self, **kwargs):
1691 1721
1692 1722 Operation.__init__(self, **kwargs)
1693 1723 self.isConfig = False
1694 1724
1695 1725 def setup(self,dataOut ,step = None , nsamples = None):
1696 1726
1697 1727 if step == None and nsamples == None:
1698 1728 raise ValueError("step or nheights should be specified ...")
1699 1729
1700 1730 self.step = step
1701 1731 self.nsamples = nsamples
1702 1732 self.__nChannels = dataOut.nChannels
1703 1733 self.__nProfiles = dataOut.nProfiles
1704 1734 self.__nHeis = dataOut.nHeights
1705 1735 shape = dataOut.data.shape #nchannels, nprofiles, nsamples
1706 1736
1707 1737 residue = (shape[1] - self.nsamples) % self.step
1708 1738 if residue != 0:
1709 1739 print("The residue is %d, step=%d should be multiple of %d to avoid loss of %d samples"%(residue,step,shape[1] - self.nsamples,residue))
1710 1740
1711 1741 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1712 1742 numberProfile = self.nsamples
1713 1743 numberSamples = (shape[1] - self.nsamples)/self.step
1714 1744
1715 1745 self.bufferShape = int(shape[0]), int(numberSamples), int(numberProfile) # nchannels, nsamples , nprofiles
1716 1746 self.profileShape = int(shape[0]), int(numberProfile), int(numberSamples) # nchannels, nprofiles, nsamples
1717 1747
1718 1748 self.buffer = numpy.zeros(self.bufferShape , dtype=numpy.complex)
1719 1749 self.sshProfiles = numpy.zeros(self.profileShape, dtype=numpy.complex)
1720 1750
1721 1751 def run(self, dataOut, step, nsamples, code = None, repeat = None):
1722 1752 dataOut.flagNoData = True
1723 1753
1724 1754 profileIndex = None
1725 1755 #print("nProfiles, nHeights ",dataOut.nProfiles, dataOut.nHeights)
1726 1756 #print(dataOut.getFreqRange(1)/1000.)
1727 1757 #exit(1)
1728 1758 if dataOut.flagDataAsBlock:
1729 1759 dataOut.data = numpy.average(dataOut.data,axis=1)
1730 1760 #print("jee")
1731 1761 dataOut.flagDataAsBlock = False
1732 1762 if not self.isConfig:
1733 1763 self.setup(dataOut, step=step , nsamples=nsamples)
1734 1764 #print("Setup done")
1735 1765 self.isConfig = True
1736 1766
1737 1767
1738 1768 if code is not None:
1739 1769 code = numpy.array(code)
1740 1770 code_block = code
1741 1771
1742 1772 if repeat is not None:
1743 1773 code_block = numpy.repeat(code_block, repeats=repeat, axis=1)
1744 1774 #print(code_block.shape)
1745 1775 for i in range(self.buffer.shape[1]):
1746 1776
1747 1777 if code is not None:
1748 1778 self.buffer[:,i] = dataOut.data[:,i*self.step:i*self.step + self.nsamples]*code_block
1749 1779
1750 1780 else:
1751 1781
1752 1782 self.buffer[:,i] = dataOut.data[:,i*self.step:i*self.step + self.nsamples]#*code[dataOut.profileIndex,:]
1753 1783
1754 1784 #self.buffer[:,j,self.__nHeis-j*self.step - self.nheights:self.__nHeis-j*self.step] = numpy.flip(dataOut.data[:,j*self.step:j*self.step + self.nheights])
1755 1785
1756 1786 for j in range(self.buffer.shape[0]):
1757 1787 self.sshProfiles[j] = numpy.transpose(self.buffer[j])
1758 1788
1759 1789 profileIndex = self.nsamples
1760 1790 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1761 1791 ippSeconds = (deltaHeight*1.0e-6)/(0.15)
1762 1792 #print("ippSeconds, dH: ",ippSeconds,deltaHeight)
1763 1793 try:
1764 1794 if dataOut.concat_m is not None:
1765 1795 ippSeconds= ippSeconds/float(dataOut.concat_m)
1766 1796 #print "Profile concat %d"%dataOut.concat_m
1767 1797 except:
1768 1798 pass
1769 1799
1770 1800 dataOut.data = self.sshProfiles
1771 1801 dataOut.flagNoData = False
1772 1802 dataOut.heightList = numpy.arange(self.buffer.shape[1]) *self.step*deltaHeight + dataOut.heightList[0]
1773 1803 dataOut.nProfiles = int(dataOut.nProfiles*self.nsamples)
1774 1804
1775 1805 dataOut.profileIndex = profileIndex
1776 1806 dataOut.flagDataAsBlock = True
1777 1807 dataOut.ippSeconds = ippSeconds
1778 1808 dataOut.step = self.step
1779 1809 #print(numpy.shape(dataOut.data))
1780 1810 #exit(1)
1781 1811 #print("new data shape and time:", dataOut.data.shape, dataOut.utctime)
1782 1812
1783 1813 return dataOut
1784 1814 ################################################################################3############################3
1785 1815 ################################################################################3############################3
1786 1816 ################################################################################3############################3
1787 1817 ################################################################################3############################3
1788 1818
1789 1819 class SSheightProfiles2(Operation):
1790 1820 '''
1791 1821 Procesa por perfiles y por bloques
1792 1822 '''
1793 1823
1794 1824
1795 1825 bufferShape = None
1796 1826 profileShape = None
1797 1827 sshProfiles = None
1798 1828 profileIndex = None
1799 1829 #nsamples = None
1800 1830 #step = None
1801 1831 #deltaHeight = None
1802 1832 #init_range = None
1803 1833 __slots__ = ('step', 'nsamples', 'deltaHeight', 'init_range', 'isConfig', '__nChannels',
1804 1834 '__nProfiles', '__nHeis', 'deltaHeight', 'new_nHeights')
1805 1835
1806 1836 def __init__(self, **kwargs):
1807 1837
1808 1838 Operation.__init__(self, **kwargs)
1809 1839 self.isConfig = False
1810 1840
1811 1841 def setup(self,dataOut ,step = None , nsamples = None):
1812 1842
1813 1843 if step == None and nsamples == None:
1814 1844 raise ValueError("step or nheights should be specified ...")
1815 1845
1816 1846 self.step = step
1817 1847 self.nsamples = nsamples
1818 1848 self.__nChannels = int(dataOut.nChannels)
1819 1849 self.__nProfiles = int(dataOut.nProfiles)
1820 1850 self.__nHeis = int(dataOut.nHeights)
1821 1851
1822 1852 residue = (self.__nHeis - self.nsamples) % self.step
1823 1853 if residue != 0:
1824 1854 print("The residue is %d, step=%d should be multiple of %d to avoid loss of %d samples"%(residue,step,self.__nProfiles - self.nsamples,residue))
1825 1855
1826 1856 self.deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1827 1857 self.init_range = dataOut.heightList[0]
1828 1858 #numberProfile = self.nsamples
1829 1859 numberSamples = (self.__nHeis - self.nsamples)/self.step
1830 1860
1831 1861 self.new_nHeights = numberSamples
1832 1862
1833 1863 self.bufferShape = int(self.__nChannels), int(numberSamples), int(self.nsamples) # nchannels, nsamples , nprofiles
1834 1864 self.profileShape = int(self.__nChannels), int(self.nsamples), int(numberSamples) # nchannels, nprofiles, nsamples
1835 1865
1836 1866 self.buffer = numpy.zeros(self.bufferShape , dtype=numpy.complex)
1837 1867 self.sshProfiles = numpy.zeros(self.profileShape, dtype=numpy.complex)
1838 1868
1839 1869 def getNewProfiles(self, data, code=None, repeat=None):
1840 1870
1841 1871 if code is not None:
1842 1872 code = numpy.array(code)
1843 1873 code_block = code
1844 1874
1845 1875 if repeat is not None:
1846 1876 code_block = numpy.repeat(code_block, repeats=repeat, axis=1)
1847 1877 if data.ndim < 3:
1848 1878 data = data.reshape(self.__nChannels,1,self.__nHeis )
1849 1879 #print("buff, data, :",self.buffer.shape, data.shape,self.sshProfiles.shape, code_block.shape)
1850 1880 for ch in range(self.__nChannels):
1851 1881 for i in range(int(self.new_nHeights)): #nuevas alturas
1852 1882 if code is not None:
1853 1883 self.buffer[ch,i,:] = data[ch,:,i*self.step:i*self.step + self.nsamples]*code_block
1854 1884 else:
1855 1885 self.buffer[ch,i,:] = data[ch,:,i*self.step:i*self.step + self.nsamples]#*code[dataOut.profileIndex,:]
1856 1886
1857 1887 for j in range(self.__nChannels): #en los cananles
1858 1888 self.sshProfiles[j,:,:] = numpy.transpose(self.buffer[j,:,:])
1859 1889 #print("new profs Done")
1860 1890
1861 1891
1862 1892
1863 1893 def run(self, dataOut, step, nsamples, code = None, repeat = None):
1864 1894 # print("running")
1865 1895 if dataOut.flagNoData == True:
1866 1896 return dataOut
1867 1897 dataOut.flagNoData = True
1868 1898 #print("init data shape:", dataOut.data.shape)
1869 1899 #print("ch: {} prof: {} hs: {}".format(int(dataOut.nChannels),
1870 1900 # int(dataOut.nProfiles),int(dataOut.nHeights)))
1871 1901
1872 1902 profileIndex = None
1873 1903 # if not dataOut.flagDataAsBlock:
1874 1904 # dataOut.nProfiles = 1
1875 1905
1876 1906 if not self.isConfig:
1877 1907 self.setup(dataOut, step=step , nsamples=nsamples)
1878 1908 #print("Setup done")
1879 1909 self.isConfig = True
1880 1910
1881 1911 dataBlock = None
1882 1912
1883 1913 nprof = 1
1884 1914 if dataOut.flagDataAsBlock:
1885 1915 nprof = int(dataOut.nProfiles)
1886 1916
1887 1917 #print("dataOut nProfiles:", dataOut.nProfiles)
1888 1918 for profile in range(nprof):
1889 1919 if dataOut.flagDataAsBlock:
1890 1920 #print("read blocks")
1891 1921 self.getNewProfiles(dataOut.data[:,profile,:], code=code, repeat=repeat)
1892 1922 else:
1893 1923 #print("read profiles")
1894 1924 self.getNewProfiles(dataOut.data, code=code, repeat=repeat) #only one channe
1895 1925 if profile == 0:
1896 1926 dataBlock = self.sshProfiles.copy()
1897 1927 else: #by blocks
1898 1928 dataBlock = numpy.concatenate((dataBlock,self.sshProfiles), axis=1) #profile axis
1899 1929 #print("by blocks: ",dataBlock.shape, self.sshProfiles.shape)
1900 1930
1901 1931 profileIndex = self.nsamples
1902 1932 #deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1903 1933 ippSeconds = (self.deltaHeight*1.0e-6)/(0.15)
1904 1934
1905 1935
1906 1936 dataOut.data = dataBlock
1907 1937 #print("show me: ",self.step,self.deltaHeight, dataOut.heightList, self.new_nHeights)
1908 1938 dataOut.heightList = numpy.arange(int(self.new_nHeights)) *self.step*self.deltaHeight + self.init_range
1909 1939
1910 1940 dataOut.ippSeconds = ippSeconds
1911 1941 dataOut.step = self.step
1912 1942 dataOut.flagNoData = False
1913 1943 if dataOut.flagDataAsBlock:
1914 1944 dataOut.nProfiles = int(dataOut.nProfiles*self.nsamples)
1915 1945
1916 1946 else:
1917 1947 dataOut.nProfiles = int(self.nsamples)
1918 1948 dataOut.profileIndex = dataOut.nProfiles
1919 1949 dataOut.flagDataAsBlock = True
1920 1950
1921 1951 dataBlock = None
1922 1952
1923 1953 #print("new data shape:", dataOut.data.shape, dataOut.utctime)
1924 1954
1925 1955 return dataOut
1926 1956
1927 1957
1928 1958
1929 1959
1930 1960
1931 1961 class removeProfileByFaradayHS(Operation):
1932 1962 '''
1933 1963
1934 1964 '''
1935 1965
1936 1966 __buffer_data = []
1937 1967 __buffer_times = []
1938 1968
1939 1969 buffer = None
1940 1970
1941 1971 outliers_IDs_list = []
1942 1972
1943 1973
1944 1974 __slots__ = ('n','navg','profileMargin','thHistOutlier','minHei_idx','maxHei_idx','nHeights',
1945 1975 '__dh','first_utcBlock','__profIndex','init_prof','end_prof','lenProfileOut','nChannels',
1946 1976 '__count_exec','__initime','__dataReady','__ipp')
1947 1977 def __init__(self, **kwargs):
1948 1978
1949 1979 Operation.__init__(self, **kwargs)
1950 1980 self.isConfig = False
1951 1981
1952 1982 def setup(self,dataOut, n=None , navg=0.8, profileMargin=50,thHistOutlier=3, minHei=None, maxHei=None):
1953 1983
1954 1984 if n == None and timeInterval == None:
1955 1985 raise ValueError("nprofiles or timeInterval should be specified ...")
1956 1986
1957 1987 if n != None:
1958 1988 self.n = n
1959 1989
1960 1990 self.navg = navg
1961 1991 self.profileMargin = profileMargin
1962 1992 self.thHistOutlier = thHistOutlier
1963 1993 self.__profIndex = 0
1964 1994 self.buffer = None
1965 1995 self._ipp = dataOut.ippSeconds
1966 1996 self.n_prof_released = 0
1967 1997 self.heightList = dataOut.heightList
1968 1998 self.init_prof = 0
1969 1999 self.end_prof = 0
1970 2000 self.__count_exec = 0
1971 2001 self.__profIndex = 0
1972 2002 self.first_utcBlock = None
1973 2003 self.__dh = dataOut.heightList[1] - dataOut.heightList[0]
1974 2004 minHei = minHei
1975 2005 maxHei = maxHei
1976 2006 if minHei==None :
1977 2007 minHei = dataOut.heightList[0]
1978 2008 if maxHei==None :
1979 2009 maxHei = dataOut.heightList[-1]
1980 2010 self.minHei_idx,self.maxHei_idx = getHei_index(minHei, maxHei, dataOut.heightList)
1981 2011
1982 2012 self.nChannels = dataOut.nChannels
1983 2013 self.nHeights = dataOut.nHeights
1984 2014 self.test_counter = 0
1985 2015
1986 2016 def filterSatsProfiles(self):
1987 2017 data = self.__buffer_data
1988 2018 #print(data.shape)
1989 2019 nChannels, profiles, heights = data.shape
1990 2020 indexes=[]
1991 2021 outliers_IDs=[]
1992 2022 for c in range(nChannels):
1993 2023 for h in range(self.minHei_idx, self.maxHei_idx):
1994 power = data[c,:,h] * numpy.conjugate(data[c,:,h])
1995 power = power.real
2024 power = 10* numpy.log10((data[c,:,h] * numpy.conjugate(data[c,:,h])).real)
2025 #power = power.real
1996 2026 #power = (numpy.abs(data[c,:,h].real))
1997 2027 sortdata = numpy.sort(power, axis=None)
1998 2028 sortID=power.argsort()
1999 index = _noise.hildebrand_sekhon2(sortdata,self.navg) #0.75-> buen valor
2029 index = _noise.hildebrand_sekhon2(sortdata,self.navg)
2000 2030
2001 2031 indexes.append(index)
2002 2032 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
2003 # print(outliers_IDs)
2033
2034 # print(sortdata.min(), sortdata.max(), sortdata.mean())
2004 2035 # fig,ax = plt.subplots()
2005 2036 # #ax.set_title(str(k)+" "+str(j))
2006 2037 # x=range(len(sortdata))
2007 2038 # ax.scatter(x,sortdata)
2008 2039 # ax.axvline(index)
2009 2040 # plt.grid()
2010 2041 # plt.show()
2011 2042
2012 2043
2013 2044 outliers_IDs = outliers_IDs.astype(numpy.dtype('int64'))
2014 2045 outliers_IDs = numpy.unique(outliers_IDs)
2015 2046 outs_lines = numpy.sort(outliers_IDs)
2016 2047 # #print("outliers Ids: ", outs_lines, outs_lines.shape)
2017 2048 #hist, bin_edges = numpy.histogram(outs_lines, bins=10, density=True)
2018 2049
2019 2050
2020 2051 #Agrupando el histograma de outliers,
2021 #my_bins = numpy.linspace(0,int(profiles), int(profiles/100), endpoint=False)
2022 my_bins = numpy.linspace(0,9600, 96, endpoint=False)
2052 my_bins = numpy.linspace(0,int(profiles), int(profiles/100), endpoint=False)
2053 #my_bins = numpy.linspace(0,1600, 96, endpoint=False)
2023 2054
2024 2055 hist, bins = numpy.histogram(outs_lines,bins=my_bins)
2025 2056 hist_outliers_indexes = numpy.where(hist > self.thHistOutlier) #es outlier
2026 2057 #print(hist_outliers_indexes[0])
2027 2058 bins_outliers_indexes = [int(i) for i in bins[hist_outliers_indexes]] #
2028 2059 #print(bins_outliers_indexes)
2029 2060 outlier_loc_index = []
2030 2061
2031 2062
2032 2063 # for n in range(len(bins_outliers_indexes)-1):
2033 2064 # for k in range(bins_outliers_indexes[n]-self.profileMargin,bins_outliers_indexes[n+1]+self.profileMargin):
2034 2065 # outlier_loc_index.append(k)
2035 2066
2036 2067 outlier_loc_index = [e for n in range(len(bins_outliers_indexes)-1) for e in range(bins_outliers_indexes[n]-self.profileMargin,bins_outliers_indexes[n+1]+self.profileMargin) ]
2037 2068
2038 2069 outlier_loc_index = numpy.asarray(outlier_loc_index)
2039 2070 #print(len(numpy.unique(outlier_loc_index)), numpy.unique(outlier_loc_index))
2040 2071
2041 2072
2042 2073
2043 # x, y = numpy.meshgrid(numpy.arange(profiles), self.heightList)
2044 # fig, ax = plt.subplots(1,2,figsize=(8, 6))
2045 #
2046 # dat = data[0,:,:].real
2047 # m = numpy.nanmean(dat)
2048 # o = numpy.nanstd(dat)
2049 # #print(m, o, x.shape, y.shape)
2050 # c = ax[0].pcolormesh(x, y, dat.T, cmap ='YlGnBu', vmin = (m-2*o), vmax = (m+2*o))
2051 # ax[0].vlines(outs_lines,200,600, linestyles='dashed', label = 'outs', color='w')
2052 # fig.colorbar(c)
2053 # ax[0].vlines(outlier_loc_index,650,750, linestyles='dashed', label = 'outs', color='r')
2054 # ax[1].hist(outs_lines,bins=my_bins)
2074 x, y = numpy.meshgrid(numpy.arange(profiles), self.heightList)
2075 fig, ax = plt.subplots(1,2,figsize=(8, 6))
2076
2077 dat = data[0,:,:].real
2078 dat = 10* numpy.log10((data[0,:,:] * numpy.conjugate(data[0,:,:])).real)
2079 m = numpy.nanmean(dat)
2080 o = numpy.nanstd(dat)
2081 #print(m, o, x.shape, y.shape)
2082 c = ax[0].pcolormesh(x, y, dat.T, cmap ='YlGnBu', vmin = (m-2*o), vmax = (m+2*o))
2083 ax[0].vlines(outs_lines,200,600, linestyles='dashed', label = 'outs', color='w')
2084 fig.colorbar(c)
2085 ax[0].vlines(outlier_loc_index,650,750, linestyles='dashed', label = 'outs', color='r')
2086 ax[1].hist(outs_lines,bins=my_bins)
2087 plt.show()
2088
2089
2090 self.outliers_IDs_list = numpy.unique(outlier_loc_index)
2091 print("outs list: ", self.outliers_IDs_list)
2092 return data
2093
2094 def filterSatsProfiles2(self):
2095 data = self.__buffer_data
2096 #print(data.shape)
2097 nChannels, profiles, heights = data.shape
2098 indexes=numpy.zeros([], dtype=int)
2099 outliers_IDs=[]
2100 for c in range(nChannels):
2101 noise_ref =10* numpy.log10((data[c,:,550:600] * numpy.conjugate(data[c,:,550:600])).real)
2102 print("Noise ",noise_ref.mean())
2103 for h in range(self.minHei_idx, self.maxHei_idx):
2104 power = 10* numpy.log10((data[c,:,h] * numpy.conjugate(data[c,:,h])).real)
2105 #power = power.real
2106 #power = (numpy.abs(data[c,:,h].real))
2107 #sortdata = numpy.sort(power, axis=None)
2108 #sortID=power.argsort()
2109 #print(sortID)
2110 th = 60 + 10
2111 index = numpy.where(power > th )
2112 if index[0].size > 10 and index[0].size < int(0.8*profiles):
2113 indexes = numpy.append(indexes, index[0])
2114 #print(index[0])
2115 #print(index[0])
2116
2117 # fig,ax = plt.subplots()
2118 # #ax.set_title(str(k)+" "+str(j))
2119 # x=range(len(power))
2120 # ax.scatter(x,power)
2121 # #ax.axvline(index)
2122 # plt.grid()
2055 2123 # plt.show()
2124 #print(indexes)
2125
2126 #outliers_IDs = outliers_IDs.astype(numpy.dtype('int64'))
2127 #outliers_IDs = numpy.unique(outliers_IDs)
2128
2129 outs_lines = numpy.unique(indexes)
2130 print("outliers Ids: ", outs_lines, outs_lines.shape)
2131 #hist, bin_edges = numpy.histogram(outs_lines, bins=10, density=True)
2132
2133
2134 #Agrupando el histograma de outliers,
2135 my_bins = numpy.linspace(0,int(profiles), int(profiles/100), endpoint=False)
2136 #my_bins = numpy.linspace(0,1600, 96, endpoint=False)
2137
2138 hist, bins = numpy.histogram(outs_lines,bins=my_bins)
2139 hist_outliers_indexes = numpy.where(hist > self.thHistOutlier) #es outlier
2140 #print(hist_outliers_indexes[0])
2141 bins_outliers_indexes = [int(i) for i in bins[hist_outliers_indexes]] #
2142 #print(bins_outliers_indexes)
2143 outlier_loc_index = []
2144
2145
2146
2147 outlier_loc_index = [e for n in range(len(bins_outliers_indexes)-1) for e in range(bins_outliers_indexes[n]-self.profileMargin,bins_outliers_indexes[n+1]+self.profileMargin) ]
2148
2149 outlier_loc_index = numpy.asarray(outlier_loc_index)
2150 outlier_loc_index = outlier_loc_index[~numpy.all(outlier_loc_index < 0)]
2151
2152 print("outliers final: ", outlier_loc_index)
2153
2154 x, y = numpy.meshgrid(numpy.arange(profiles), self.heightList)
2155 fig, ax = plt.subplots(1,2,figsize=(8, 6))
2156
2157 dat = data[0,:,:].real
2158 dat = 10* numpy.log10((data[0,:,:] * numpy.conjugate(data[0,:,:])).real)
2159 m = numpy.nanmean(dat)
2160 o = numpy.nanstd(dat)
2161 #print(m, o, x.shape, y.shape)
2162 c = ax[0].pcolormesh(x, y, dat.T, cmap ='YlGnBu', vmin = (m-2*o), vmax = (m+2*o))
2163 ax[0].vlines(outs_lines,200,600, linestyles='dashed', label = 'outs', color='w')
2164 fig.colorbar(c)
2165 ax[0].vlines(outlier_loc_index,650,750, linestyles='dashed', label = 'outs', color='r')
2166 ax[1].hist(outs_lines,bins=my_bins)
2167 plt.show()
2056 2168
2057 2169
2058 2170 self.outliers_IDs_list = numpy.unique(outlier_loc_index)
2171 print("outs list: ", self.outliers_IDs_list)
2059 2172 return data
2060 2173
2061 2174 def cleanSpikesFFT2D(self):
2062 2175 incoh_int = 10
2063 2176 norm_img = 75
2064 2177 import matplotlib.pyplot as plt
2065 2178 import datetime
2066 2179 import cv2
2067 2180 data = self.__buffer_data
2068 2181 print("cleaning shape inpt: ",data.shape)
2069 2182 self.__buffer_data = []
2070 2183
2071 2184
2072 2185 channels , profiles, heights = data.shape
2073 2186 len_split_prof = profiles / incoh_int
2074 2187
2075 2188
2076 2189 for ch in range(channels):
2077 2190 data_10 = numpy.split(data[ch, :, self.minHei_idx:], incoh_int, axis=0) # división de los perfiles
2078 2191 print("splited data: ",len(data_10)," -> ", data_10[0].shape)
2079 2192 int_img = None
2080 2193 i_count = 0
2081 2194 n_x, n_y = data_10[0].shape
2082 2195 for s_data in data_10: #porciones de espectro
2083 2196 spectrum = numpy.fft.fft2(s_data, axes=(0,1))
2084 2197 z = numpy.abs(spectrum)
2085 2198 mg = z[2:n_y,:] #omitir dc y adjunto
2086 2199 dat = numpy.log10(mg.T)
2087 2200 i_count += 1
2088 2201 if i_count == 1:
2089 2202 int_img = dat
2090 2203 else:
2091 2204 int_img += dat
2092 2205 #print(i_count)
2093 2206
2094 2207 min, max = int_img.min(), int_img.max()
2095 2208 int_img = ((int_img-min)*255/(max-min)).astype(numpy.uint8)
2096 2209
2097 2210 cv2.imshow('integrated image', int_img) #numpy.fft.fftshift(img))
2098 2211 cv2.waitKey(0)
2099 2212 #####################################################################
2100 2213 kernel_h = numpy.zeros((3,3)) #
2101 2214 kernel_h[0, :] = -2
2102 2215 kernel_h[1, :] = 3
2103 2216 kernel_h[2, :] = -2
2104 2217
2105 2218
2106 2219 kernel_5h = numpy.zeros((5,5)) #
2107 2220 kernel_5h[0, :] = -2
2108 2221 kernel_5h[1, :] = -1
2109 2222 kernel_5h[2, :] = 5
2110 2223 kernel_5h[3, :] = -1
2111 2224 kernel_5h[4, :] = -2
2112 2225
2113 2226 #####################################################################
2114 2227 sharp_img = cv2.filter2D(src=int_img, ddepth=-1, kernel=kernel_5h)
2115 2228 # cv2.imshow('sharp image h ', sharp_img)
2116 2229 # cv2.waitKey(0)
2117 2230 #####################################################################
2118 2231 horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5,1)) #11
2119 2232 #####################################################################
2120 2233 detected_lines_h = cv2.morphologyEx(sharp_img, cv2.MORPH_OPEN, horizontal_kernel, iterations=1)
2121 2234 # cv2.imshow('lines horizontal', detected_lines_h) #numpy.fft.fftshift(detected_lines_h))
2122 2235 # cv2.waitKey(0)
2123 2236 #####################################################################
2124 2237 ret, detected_lines_h = cv2.threshold(detected_lines_h, 200, 255, cv2.THRESH_BINARY)#
2125 2238 cv2.imshow('binary img', detected_lines_h) #numpy.fft.fftshift(detected_lines_h))
2126 2239 cv2.waitKey(0)
2127 2240 #####################################################################
2128 2241 cnts_h, h0 = cv2.findContours(detected_lines_h, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
2129 2242 #####################################################################
2130 2243 h_line_index = []
2131 2244 v_line_index = []
2132 2245
2133 2246 #cnts_h += cnts_h_s #combine large and small lines
2134 2247
2135 2248 # line indexes x1, x2, y
2136 2249 for c in cnts_h:
2137 2250 #print(c)
2138 2251 if len(c) < 3: #contorno linea
2139 2252 x1 = c[0][0][0]
2140 2253 x2 = c[1][0][0]
2141 2254 if x1 > 5 and x2 < (n_x-5) :
2142 2255 start = incoh_int + (x1 * incoh_int)
2143 2256 end = incoh_int + (x2 * incoh_int)
2144 2257 h_line_index.append( [start, end, c[0][0][1]] )
2145 2258
2146 2259 #print("x1, x2, y", c[0][0][0],c[1][0][0], c[0][0][1])
2147 2260 else: #contorno poligono
2148 2261 pairs = numpy.asarray([c[n][0] for n in range(len(c))])
2149 2262 y = numpy.unique(pairs[:,1])
2150 2263 x = numpy.unique(pairs[:,0])
2151 2264 #print(x)
2152 2265 for yk in y:
2153 2266 x0 = x[0]
2154 2267 if x0 < 8:
2155 2268 x0 = 10
2156 2269 #print(x[0], x[-1], yk)
2157 2270 h_line_index.append( [x0, x[-1], yk])
2158 2271 #print("x1, x2, y ->p ", x[0], x[-1], yk)
2159 2272 ###################################################################
2160 2273 #print("Cleaning")
2161 2274 # # clean Spectrum
2162 2275 spectrum = numpy.fft.fft2(data[ch,:,self.minHei_idx:], axes=(0,1))
2163 2276 z = numpy.abs(spectrum)
2164 2277 phase = numpy.angle(spectrum)
2165 2278 print("Total Horizontal", len(h_line_index))
2166 2279 if len(h_line_index) < 75 :
2167 2280 for x1, x2, y in h_line_index:
2168 2281 print(x1, x2, y)
2169 2282 z[x1:x2,y] = 0
2170 2283
2171 2284
2172 2285 spcCleaned = z * numpy.exp(1j*phase)
2173 2286
2174 2287 dat2 = numpy.log10(z[1:-1,:].T)
2175 2288 min, max =dat2.min(), dat2.max()
2176 2289 print(min, max)
2177 2290 img2 = ((dat2-min)*255/(max-min)).astype(numpy.uint8)
2178 2291 cv2.imshow('cleaned', img2) #numpy.fft.fftshift(img_cleaned))
2179 2292 cv2.waitKey(0)
2180 2293 cv2.destroyAllWindows()
2181 2294
2182 2295 data[ch,:,self.minHei_idx:] = numpy.fft.ifft2(spcCleaned, axes=(0,1))
2183 2296
2184 2297
2185 2298 #print("cleanOutliersByBlock Done", data.shape)
2186 2299 self.__buffer_data = data
2187 2300 return data
2188 2301
2189 2302
2190 2303
2191 2304
2192 2305 def cleanOutliersByBlock(self):
2193 2306 import matplotlib.pyplot as plt
2194 2307 import datetime
2195 2308 import cv2
2196 2309 #print(self.__buffer_data[0].shape)
2197 2310 data = self.__buffer_data#.copy()
2198 2311 print("cleaning shape inpt: ",data.shape)
2199 2312 self.__buffer_data = []
2200 2313
2201 2314
2202 2315 spectrum = numpy.fft.fft2(data[:,:,self.minHei_idx:], axes=(1,2))
2203 2316 print("spc : ",spectrum.shape)
2204 2317 (nch,nsamples, nh) = spectrum.shape
2205 2318 data2 = None
2206 2319 #print(data.shape)
2207 2320 cleanedBlock = None
2208 2321 spectrum2 = spectrum.copy()
2209 2322 for ch in range(nch):
2210 2323 dh = self.__dh
2211 2324 dt1 = (dh*1.0e-6)/(0.15)
2212 2325 dt2 = self.__buffer_times[1]-self.__buffer_times[0]
2213 2326
2214 2327 freqv = numpy.fft.fftfreq(nh, d=dt1)
2215 2328 freqh = numpy.fft.fftfreq(self.n, d=dt2)
2216 2329
2217 2330 z = numpy.abs(spectrum[ch,:,:])
2218 2331 phase = numpy.angle(spectrum[ch,:,:])
2219 2332 z1 = z[0,:]
2220 2333 #print("shape z: ", z.shape, nsamples)
2221 2334
2222 2335 dat = numpy.log10(z[1:nsamples,:].T)
2223 2336
2224 2337 pdat = numpy.log10(phase.T)
2225 2338 #print("dat mean",dat.mean())
2226 2339
2227 2340 mean, min, max = dat.mean(), dat.min(), dat.max()
2228 2341 img = ((dat-min)*200/(max-min)).astype(numpy.uint8)
2229 2342
2230 2343 # print(img.shape)
2231 2344 cv2.imshow('image', img) #numpy.fft.fftshift(img))
2232 2345 cv2.waitKey(0)
2233 2346
2234 2347
2235 2348 ''' #FUNCIONA LINEAS PEQUEÑAS
2236 2349 kernel_5h = numpy.zeros((5,3)) #
2237 2350 kernel_5h[0, :] = 2
2238 2351 kernel_5h[1, :] = 1
2239 2352 kernel_5h[2, :] = 0
2240 2353 kernel_5h[3, :] = -1
2241 2354 kernel_5h[4, :] = -2
2242 2355
2243 2356 sharp_imgh = cv2.filter2D(src=img, ddepth=-1, kernel=kernel_5h)
2244 2357 cv2.imshow('sharp image h',sharp_imgh)
2245 2358 cv2.waitKey(0)
2246 2359 horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (20,1))
2247 2360
2248 2361 detected_lines_h = cv2.morphologyEx(sharp_imgh, cv2.MORPH_OPEN, horizontal_kernel, iterations=1)
2249 2362 #detected_lines_h = cv2.medianBlur(detected_lines_h, 3)
2250 2363 #detected_lines_h = cv2.filter2D(src=img, ddepth=-1, kernel=kernel)
2251 2364 cv2.imshow('lines h gray', detected_lines_h)
2252 2365 cv2.waitKey(0)
2253 2366 reth, detected_lines_h = cv2.threshold(detected_lines_h, 90, 255, cv2.THRESH_BINARY)
2254 2367 cv2.imshow('lines h ', detected_lines_h)
2255 2368 cv2.waitKey(0)
2256 2369 '''
2257 2370
2258 2371
2259 2372 '''
2260 2373 kernel_3h = numpy.zeros((3,10)) #10
2261 2374 kernel_3h[0, :] = -1
2262 2375 kernel_3h[1, :] = 2
2263 2376 kernel_3h[2, :] = -1
2264 2377
2265 2378
2266 2379 kernel_h = numpy.zeros((3,20)) #20
2267 2380 kernel_h[0, :] = -1
2268 2381 kernel_h[1, :] = 2
2269 2382 kernel_h[2, :] = -1
2270 2383
2271 2384 kernel_v = numpy.zeros((30,3)) #30
2272 2385 kernel_v[:, 0] = -1
2273 2386 kernel_v[:, 1] = 2
2274 2387 kernel_v[:, 2] = -1
2275 2388
2276 2389 kernel_4h = numpy.zeros((4,20)) #
2277 2390 kernel_4h[0, :] = 1
2278 2391 kernel_4h[1, :] = 0
2279 2392 kernel_4h[2, :] = 0
2280 2393 kernel_4h[3, :] = -1
2281 2394
2282 2395 kernel_5h = numpy.zeros((5,30)) #
2283 2396 kernel_5h[0, :] = 2
2284 2397 kernel_5h[1, :] = 1
2285 2398 kernel_5h[2, :] = 0
2286 2399 kernel_5h[3, :] = -1
2287 2400 kernel_5h[4, :] = -2
2288 2401
2289 2402
2290 2403 sharp_img0 = cv2.filter2D(src=img, ddepth=-1, kernel=kernel_3h)
2291 2404 # cv2.imshow('sharp image small h',sharp_img0) # numpy.fft.fftshift(sharp_img1))
2292 2405 # cv2.waitKey(0)
2293 2406
2294 2407 sharp_img1 = cv2.filter2D(src=img, ddepth=-1, kernel=kernel_h)
2295 2408 # cv2.imshow('sharp image h',sharp_img1) # numpy.fft.fftshift(sharp_img1))
2296 2409 # cv2.waitKey(0)
2297 2410
2298 2411 sharp_img2 = cv2.filter2D(src=img, ddepth=-1, kernel=kernel_v)
2299 2412 # cv2.imshow('sharp image v', sharp_img2) #numpy.fft.fftshift(sharp_img2))
2300 2413 # cv2.waitKey(0)
2301 2414
2302 2415 sharp_imgw = cv2.filter2D(src=img, ddepth=-1, kernel=kernel_4h)
2303 2416 # cv2.imshow('sharp image h wide', sharp_imgw) #numpy.fft.fftshift(sharp_img2))
2304 2417 # cv2.waitKey(0)
2305 2418
2306 2419 sharp_imgwl = cv2.filter2D(src=img, ddepth=-1, kernel=kernel_5h, borderType = cv2.BORDER_ISOLATED)
2307 2420 cv2.imshow('sharp image h long wide', sharp_imgwl) #numpy.fft.fftshift(sharp_img2))
2308 2421 cv2.waitKey(0)
2309 2422
2310 2423 # cv2.imwrite('/home/soporte/Data/AMISR14/ISR/spc/spc/sharp_h.jpg', sharp_img1)
2311 2424 # cv2.imwrite('/home/soporte/Data/AMISR14/ISR/spc/spc/sharp_v.jpg', sharp_img2)
2312 2425 # cv2.imwrite('/home/soporte/Data/AMISR14/ISR/spc/spc/input_img.jpg', img)
2313 2426
2314 2427 ########################small horizontal
2315 2428 horizontal_kernel_s = cv2.getStructuringElement(cv2.MORPH_RECT, (11,1)) #11
2316 2429 ######################## horizontal
2317 2430 horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (30,1)) #30
2318 2431 ######################## vertical
2319 2432 vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1,50)) #50
2320 2433 ######################## horizontal wide
2321 2434 horizontal_kernel_w = cv2.getStructuringElement(cv2.MORPH_RECT, (30,1)) # 30
2322 2435
2323 2436 horizontal_kernel_expand = cv2.getStructuringElement(cv2.MORPH_RECT, (3,3)) #
2324 2437
2325 2438 horizontal_kernel_wl = cv2.getStructuringElement(cv2.MORPH_RECT, (50,1)) #
2326 2439
2327 2440 detected_lines_h_s = cv2.morphologyEx(sharp_img0, cv2.MORPH_OPEN, horizontal_kernel_s, iterations=7) #7
2328 2441 detected_lines_h = cv2.morphologyEx(sharp_img1, cv2.MORPH_OPEN, horizontal_kernel, iterations=7) #7
2329 2442 detected_lines_v = cv2.morphologyEx(sharp_img2, cv2.MORPH_OPEN, vertical_kernel, iterations=7) #7
2330 2443 detected_lines_h_w = cv2.morphologyEx(sharp_imgw, cv2.MORPH_OPEN, horizontal_kernel_w, iterations=5) #5
2331 2444
2332 2445 detected_lines_h_wl = cv2.morphologyEx(sharp_imgwl, cv2.MORPH_OPEN, horizontal_kernel_wl, iterations=5) #
2333 2446 detected_lines_h_wl = cv2.filter2D(src=detected_lines_h_wl, ddepth=-1, kernel=horizontal_kernel_expand)
2334 2447
2335 2448 # cv2.imshow('lines h small gray', detected_lines_h_s) #numpy.fft.fftshift(detected_lines_h))
2336 2449 # cv2.waitKey(0)
2337 2450 # cv2.imshow('lines h gray', detected_lines_h) #numpy.fft.fftshift(detected_lines_h))
2338 2451 # cv2.waitKey(0)
2339 2452 # cv2.imshow('lines v gray', detected_lines_v) #numpy.fft.fftshift(detected_lines_h))
2340 2453 # cv2.waitKey(0)
2341 2454 # cv2.imshow('lines h wide gray', detected_lines_h_w) #numpy.fft.fftshift(detected_lines_h))
2342 2455 # cv2.waitKey(0)
2343 2456 cv2.imshow('lines h long wide gray', detected_lines_h_wl) #numpy.fft.fftshift(detected_lines_h))
2344 2457 cv2.waitKey(0)
2345 2458
2346 2459 reth_s, detected_lines_h_s = cv2.threshold(detected_lines_h_s, 85, 255, cv2.THRESH_BINARY)# 85
2347 2460 reth, detected_lines_h = cv2.threshold(detected_lines_h, 30, 255, cv2.THRESH_BINARY) #30
2348 2461 retv, detected_lines_v = cv2.threshold(detected_lines_v, 30, 255, cv2.THRESH_BINARY) #30
2349 2462 reth_w, detected_lines_h_w = cv2.threshold(detected_lines_h_w, 35, 255, cv2.THRESH_BINARY)#
2350 2463 reth_wl, detected_lines_h_wl = cv2.threshold(detected_lines_h_wl, 200, 255, cv2.THRESH_BINARY)#
2351 2464
2352 2465 cnts_h_s, h0 = cv2.findContours(detected_lines_h_s, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
2353 2466 cnts_h, h1 = cv2.findContours(detected_lines_h, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
2354 2467 cnts_v, h2 = cv2.findContours(detected_lines_v, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
2355 2468 cnts_h_w, h3 = cv2.findContours(detected_lines_h_w, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
2356 2469 cnts_h_wl, h4 = cv2.findContours(detected_lines_h_wl, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
2357 2470 #print("horizontal ", cnts_h)
2358 2471 #print("vertical ", cnts_v)
2359 2472 # cnts, h = cv2.findContours(detected_lines_h, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
2360 2473 # print(cnts)
2361 2474 # cv2.imshow('lines h wide', detected_lines_h_w) #numpy.fft.fftshift(detected_lines_h))
2362 2475 # cv2.waitKey(0)
2363 2476 cv2.imshow('lines h wide long ', detected_lines_h_wl) #numpy.fft.fftshift(detected_lines_v))
2364 2477 cv2.waitKey(0)
2365 2478 # cv2.imshow('lines h small', detected_lines_h_s) #numpy.fft.fftshift(detected_lines_h))
2366 2479 # cv2.waitKey(0)
2367 2480 # cv2.imshow('lines h ', detected_lines_h) #numpy.fft.fftshift(detected_lines_h))
2368 2481 # cv2.waitKey(0)
2369 2482 # cv2.imshow('lines v ', detected_lines_v) #numpy.fft.fftshift(detected_lines_v))
2370 2483 # cv2.waitKey(0)
2371 2484
2372 2485 # cv2.imwrite('/home/soporte/Data/AMISR14/ISR/spc/spc/lines_h.jpg', detected_lines_h)
2373 2486 # cv2.imwrite('/home/soporte/Data/AMISR14/ISR/spc/spc/lines_v.jpg', detected_lines_v)
2374 2487
2375 2488 #cnts = cnts[0] if len(cnts) == 2 else cnts[1]
2376 2489 #y_line_index = numpy.asarray([ [c[0][0][0],c[1][0][0], c[0][0][1]] for c in cnts_v ])
2377 2490 h_line_index = []
2378 2491 v_line_index = []
2379 2492
2380 2493 cnts_h += cnts_h_s #combine large and small lines
2381 2494
2382 2495 # line indexes x1, x2, y
2383 2496 for c in cnts_h:
2384 2497 #print(c)
2385 2498 if len(c) < 3: #contorno linea
2386 2499 x1 = c[0][0][0]
2387 2500 if x1 < 8:
2388 2501 x1 = 10
2389 2502 h_line_index.append( [x1,c[1][0][0], c[0][0][1]] )
2390 2503 #print("x1, x2, y", c[0][0][0],c[1][0][0], c[0][0][1])
2391 2504 else: #contorno poligono
2392 2505 pairs = numpy.asarray([c[n][0] for n in range(len(c))])
2393 2506 y = numpy.unique(pairs[:,1])
2394 2507 x = numpy.unique(pairs[:,0])
2395 2508 #print(x)
2396 2509 for yk in y:
2397 2510 x0 = x[0]
2398 2511 if x0 < 8:
2399 2512 x0 = 10
2400 2513 #print(x[0], x[-1], yk)
2401 2514 h_line_index.append( [x0, x[-1], yk])
2402 2515 #print("x1, x2, y ->p ", x[0], x[-1], yk)
2403 2516 for c in cnts_h_w:
2404 2517 #print(c)
2405 2518 if len(c) < 3: #contorno linea
2406 2519 x1 = c[0][0][0]
2407 2520 if x1 < 8:
2408 2521 x1 = 10
2409 2522 y = c[0][0][1] - 2 # se incrementa 2 líneas x el filtro
2410 2523 h_line_index.append( [x1,c[1][0][0],y] )
2411 2524 #print("x1, x2, y", c[0][0][0],c[1][0][0], c[0][0][1])
2412 2525 else: #contorno poligono
2413 2526 pairs = numpy.asarray([c[n][0] for n in range(len(c))])
2414 2527 y = numpy.unique(pairs[:,1])
2415 2528 x = numpy.unique(pairs[:,0])
2416 2529 #print(x)
2417 2530 for yk in y:
2418 2531
2419 2532 x0 = x[0]
2420 2533 if x0 < 8:
2421 2534 x0 = 10
2422 2535 h_line_index.append( [x0, x[-1], yk-2])
2423 2536
2424 2537 for c in cnts_h_wl: # # revisar
2425 2538 #print(c)
2426 2539 if len(c) < 3: #contorno linea
2427 2540 x1 = c[0][0][0]
2428 2541 if x1 < 8:
2429 2542 x1 = 10
2430 2543 y = c[0][0][1] - 2 # se incrementa 2 líneas x el filtro
2431 2544 h_line_index.append( [x1,c[1][0][0],y] )
2432 2545 #print("x1, x2, y", c[0][0][0],c[1][0][0], c[0][0][1])
2433 2546 else: #contorno poligono
2434 2547 pairs = numpy.asarray([c[n][0] for n in range(len(c))])
2435 2548 y = numpy.unique(pairs[:,1])
2436 2549 x = numpy.unique(pairs[:,0])
2437 2550 for yk in range(y[-1]-y[0]):
2438 2551 y_k = yk +y[0]
2439 2552
2440 2553 x0 = x[0]
2441 2554 if x0 < 8:
2442 2555 x0 = 10
2443 2556 h_line_index.append( [x0, x[-1], y_k-2])
2444 2557
2445 2558 print([[c[0][0][1],c[1][0][1], c[0][0][0] ] for c in cnts_v])
2446 2559 # line indexes y1, y2, x
2447 2560 for c in cnts_v:
2448 2561 if len(c) < 3: #contorno linea
2449 2562 v_line_index.append( [c[0][0][1],c[1][0][1], c[0][0][0] ] )
2450 2563 else: #contorno poligono
2451 2564 pairs = numpy.asarray([c[n][0] for n in range(len(c))])
2452 2565 #print(pairs)
2453 2566 y = numpy.unique(pairs[:,1])
2454 2567 x = numpy.unique(pairs[:,0])
2455 2568
2456 2569 for xk in x:
2457 2570 #print(x[0], x[-1], yk)
2458 2571 v_line_index.append( [y[0],y[-1], xk])
2459 2572
2460 2573 ###################################################################
2461 2574 # # clean Horizontal
2462 2575 print("Total Horizontal", len(h_line_index))
2463 2576 if len(h_line_index) < 75 :
2464 2577 for x1, x2, y in h_line_index:
2465 2578 #print("cleaning ",x1, x2, y)
2466 2579 len_line = x2 - x1
2467 2580 if y > 10 and y < (nh -10):
2468 2581 # if y != (nh-1):
2469 2582 # list = [ ((z[n, y-1] + z[n,y+1])/2) for n in range(len_line)]
2470 2583 # else:
2471 2584 # list = [ ((z[n, y-1] + z[n,0])/2) for n in range(len_line)]
2472 2585 #
2473 2586 # z[x1:x2,y] = numpy.asarray(list)
2474 2587 z[x1-5:x2+5,y:y+1] = 0
2475 2588
2476 2589 # clean vertical
2477 2590 for y1, y2, x in v_line_index:
2478 2591 len_line = y2 - y1
2479 2592 #print(x)
2480 2593 if x > 0 and x < (nsamples-2):
2481 2594 # if x != (nsamples-1):
2482 2595 # list = [ ((z[x-2, n] + z[x+2,n])/2) for n in range(len_line)]
2483 2596 # else:
2484 2597 # list = [ ((z[x-2, n] + z[1,n])/2) for n in range(len_line)]
2485 2598 #
2486 2599 # #z[x-1:x+1,y1:y2] = numpy.asarray(list)
2487 2600 #
2488 2601 z[x+1,y1:y2] = 0
2489 2602
2490 2603 '''
2491 2604 #z[: ,[215, 217, 221, 223, 225, 340, 342, 346, 348, 350, 465, 467, 471, 473, 475]]=0
2492 2605 z[1: ,[112, 114, 118, 120, 122, 237, 239, 245, 247, 249, 362, 364, 368, 370, 372]]=0
2493 2606 # z[: ,217]=0
2494 2607 # z[: ,221]=0
2495 2608 # z[: ,223]=0
2496 2609 # z[: ,225]=0
2497 2610
2498 2611 dat2 = numpy.log10(z.T)
2499 2612 #print(dat2)
2500 2613 max = dat2.max()
2501 2614 #print(" min, max ", max, min)
2502 2615 img2 = ((dat2-min)*255/(max-min)).astype(numpy.uint8)
2503 2616 #img_cleaned = img2.copy()
2504 2617 #cv2.drawContours(img2, cnts_h, -1, (255,255,255), 1)
2505 2618 #cv2.drawContours(img2, cnts_v, -1, (255,255,255), 1)
2506 2619
2507 2620
2508 2621 spcCleaned = z * numpy.exp(1j*phase)
2509 2622 #print(spcCleaned)
2510 2623
2511 2624
2512 2625 # cv2.imshow('image contours', img2) #numpy.fft.fftshift(img))
2513 2626 # cv2.waitKey(0)
2514 2627
2515 2628 cv2.imshow('cleaned', img2) #numpy.fft.fftshift(img_cleaned))
2516 2629 cv2.waitKey(0)
2517 2630 # # cv2.imwrite('/home/soporte/Data/AMISR14/ISR/spc/spc/cleaned_{}.jpg'.format(self.test_counter), img2)
2518 2631 cv2.destroyAllWindows()
2519 2632 # self.test_counter += 1
2520 2633
2521 2634
2522 2635 #print("DC difference " ,z1 - z[0,:])
2523 2636
2524 2637 # m = numpy.mean(dat)
2525 2638 # o = numpy.std(dat)
2526 2639 # print("mean ", m, " std ", o)
2527 2640 # fig, ax = plt.subplots(1,2,figsize=(12, 6))
2528 2641 # #X, Y = numpy.meshgrid(numpy.sort(freqh),numpy.sort(freqv))
2529 2642 # X, Y = numpy.meshgrid(numpy.fft.fftshift(freqh),numpy.fft.fftshift(freqv))
2530 2643 #
2531 2644 # colormap = 'jet'
2532 2645 # #c = ax[0].pcolormesh(x, y, dat, cmap =colormap, vmin = (m-2*o)/2, vmax = (m+2*o))
2533 2646 # #c = ax[0].pcolormesh(X, Y, numpy.fft.fftshift(dat), cmap =colormap, vmin = 6.5, vmax = 6.8)
2534 2647 # c = ax[0].pcolormesh(X, Y, numpy.fft.fftshift(dat), cmap =colormap, vmin = (m-2*o), vmax = (m+1.5*o))
2535 2648 # fig.colorbar(c, ax=ax[0])
2536 2649 #
2537 2650 #
2538 2651 # #c = ax.pcolor( z.T , cmap ='gray', vmin = (m-2*o), vmax = (m+2*o))
2539 2652 # #date_time = datetime.datetime.fromtimestamp(self.__buffer_times[0]).strftime('%Y-%m-%d %H:%M:%S.%f')
2540 2653 # #strftime('%Y-%m-%d %H:%M:%S')
2541 2654 # #ax[0].set_title('Spectrum magnitude '+date_time)
2542 2655 # #fig.canvas.set_window_title('Spectrum magnitude {} '.format(self.n)+date_time)
2543 2656 # #print("aqui estoy2",dat2[:,:,0].shape)
2544 2657 # #c = ax[1].pcolormesh(X, Y, numpy.fft.fftshift(pdat), cmap =colormap, vmin = 4.2, vmax = 5.0)
2545 2658 # c = ax[0].pcolormesh(X, Y, numpy.fft.fftshift(dat2), cmap =colormap, vmin = (m-2*o), vmax = (m+1.5*o))
2546 2659 # #c = ax[1].pcolormesh(X, Y, numpy.fft.fftshift(pdat), cmap =colormap ) #, vmin = 0.0, vmax = 0.5)
2547 2660 # #c = ax[1].pcolormesh(x, y, dat2[:,:,0], cmap =colormap, vmin = (m-2*o)/2, vmax = (m+2*o)-1)
2548 2661 # #print("aqui estoy3")
2549 2662 # fig.colorbar(c, ax=ax[1])
2550 2663 # plt.show()
2551 2664
2552 2665 spectrum[ch,:,:] = spcCleaned
2553 2666
2554 2667 #print(data2.shape)
2555 2668
2556 2669
2557 2670
2558 2671 data[:,:,self.minHei_idx:] = numpy.fft.ifft2(spectrum, axes=(1,2))
2559 2672
2560 2673 #print("cleanOutliersByBlock Done", data.shape)
2561 2674 self.__buffer_data = data
2562 2675 return data
2563 2676
2564 2677
2565 2678
2566 2679 def fillBuffer(self, data, datatime):
2567 2680
2568 2681 if self.__profIndex == 0:
2569 2682 self.__buffer_data = data.copy()
2570 2683
2571 2684 else:
2572 2685 self.__buffer_data = numpy.concatenate((self.__buffer_data,data), axis=1)#en perfiles
2573 2686 self.__profIndex += 1
2574 2687 self.__buffer_times.append(datatime)
2575 2688
2576 2689 def getData(self, data, datatime=None):
2577 2690
2578 2691 if self.__profIndex == 0:
2579 2692 self.__initime = datatime
2580 2693
2581 2694
2582 2695 self.__dataReady = False
2583 2696
2584 2697 self.fillBuffer(data, datatime)
2585 2698 dataBlock = None
2586 2699
2587 2700 if self.__profIndex == self.n:
2588 2701 #print("apnd : ",data)
2589 2702 #dataBlock = self.cleanOutliersByBlock()
2590 2703 #dataBlock = self.cleanSpikesFFT2D()
2591 dataBlock = self.filterSatsProfiles()
2704 dataBlock = self.filterSatsProfiles2()
2592 2705 self.__dataReady = True
2593 2706
2594 2707 return dataBlock
2595 2708
2596 2709 if dataBlock is None:
2597 2710 return None, None
2598 2711
2599 2712
2600 2713
2601 2714 return dataBlock
2602 2715
2603 2716 def releaseBlock(self):
2604 2717
2605 2718 if self.n % self.lenProfileOut != 0:
2606 2719 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n))
2607 2720 return None
2608 2721
2609 2722 data = self.buffer[:,self.init_prof:self.end_prof:,:] #ch, prof, alt
2610 2723
2611 2724 self.init_prof = self.end_prof
2612 2725 self.end_prof += self.lenProfileOut
2613 2726 #print("data release shape: ",dataOut.data.shape, self.end_prof)
2614 2727 self.n_prof_released += 1
2615 2728
2616 2729
2617 2730 #print("f_no_data ", dataOut.flagNoData)
2618 2731 return data
2619 2732
2620 2733 def run(self, dataOut, n=None, navg=0.8, nProfilesOut=1, profile_margin=50,th_hist_outlier=3,minHei=None, maxHei=None):
2621 2734 #print("run op buffer 2D",dataOut.ippSeconds)
2622 2735 # self.nChannels = dataOut.nChannels
2623 2736 # self.nHeights = dataOut.nHeights
2624 2737
2625 2738 if not self.isConfig:
2626 2739 #print("init p idx: ", dataOut.profileIndex )
2627 2740 self.setup(dataOut,n=n, navg=navg,profileMargin=profile_margin,
2628 2741 thHistOutlier=th_hist_outlier,minHei=minHei, maxHei=maxHei)
2629 2742 self.isConfig = True
2630 2743
2631 2744 dataBlock = None
2632 2745
2633 2746 if not dataOut.buffer_empty: #hay datos acumulados
2634 2747
2635 2748 if self.init_prof == 0:
2636 2749 self.n_prof_released = 0
2637 2750 self.lenProfileOut = nProfilesOut
2638 2751 dataOut.flagNoData = False
2639 2752 #print("tp 2 ",dataOut.data.shape)
2640 2753
2641 2754 self.init_prof = 0
2642 2755 self.end_prof = self.lenProfileOut
2643 2756
2644 2757 dataOut.nProfiles = self.lenProfileOut
2645 2758 if nProfilesOut == 1:
2646 2759 dataOut.flagDataAsBlock = False
2647 2760 else:
2648 2761 dataOut.flagDataAsBlock = True
2649 2762 #print("prof: ",self.init_prof)
2650 2763 dataOut.flagNoData = False
2651 2764 if numpy.isin(self.n_prof_released, self.outliers_IDs_list):
2652 #print("omitting: ", self.n_prof_released)
2765 print("omitting: ", self.n_prof_released)
2653 2766 dataOut.flagNoData = True
2654 2767 dataOut.ippSeconds = self._ipp
2655 2768 dataOut.utctime = self.first_utcBlock + self.init_prof*self._ipp
2656 2769 # print("time: ", dataOut.utctime, self.first_utcBlock, self.init_prof,self._ipp,dataOut.ippSeconds)
2657 2770 #dataOut.data = self.releaseBlock()
2658 2771 #########################################################3
2659 2772 if self.n % self.lenProfileOut != 0:
2660 2773 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n))
2661 2774 return None
2662 2775
2663 2776 dataOut.data = self.buffer[:,self.init_prof:self.end_prof:,:] #ch, prof, alt
2664 2777
2665 2778 self.init_prof = self.end_prof
2666 2779 self.end_prof += self.lenProfileOut
2667 # #print("data release shape: ",dataOut.data.shape, self.end_prof, dataOut.flagNoData)
2780 #print("data release shape: ",dataOut.data.shape, self.end_prof, dataOut.flagNoData)
2668 2781 self.n_prof_released += 1
2669 2782
2670 2783 if self.end_prof >= (self.n +self.lenProfileOut):
2671 2784
2672 2785 self.init_prof = 0
2673 2786 self.__profIndex = 0
2674 2787 self.buffer = None
2675 2788 dataOut.buffer_empty = True
2676 2789 self.outliers_IDs_list = []
2677 2790 self.n_prof_released = 0
2678 2791 dataOut.flagNoData = False #enviar ultimo aunque sea outlier :(
2679 2792 #print("cleaning...", dataOut.buffer_empty)
2680 2793 dataOut.profileIndex = 0 #self.lenProfileOut
2681 2794 ####################################################################
2682 2795 return dataOut
2683 2796
2684 2797
2685 2798 #print("tp 223 ",dataOut.data.shape)
2686 2799 dataOut.flagNoData = True
2687 2800
2688 2801
2689 2802
2690 2803 try:
2691 2804 #dataBlock = self.getData(dataOut.data.reshape(self.nChannels,1,self.nHeights), dataOut.utctime)
2692 2805 dataBlock = self.getData(numpy.reshape(dataOut.data,(self.nChannels,1,self.nHeights)), dataOut.utctime)
2693 2806 self.__count_exec +=1
2694 2807 except Exception as e:
2695 2808 print("Error getting profiles data",self.__count_exec )
2696 2809 print(e)
2697 2810 sys.exit()
2698 2811
2699 2812 if self.__dataReady:
2700 2813 #print("omitting: ", len(self.outliers_IDs_list))
2701 2814 self.__count_exec = 0
2702 2815 #dataOut.data =
2703 2816 #self.buffer = numpy.flip(dataBlock, axis=1)
2704 2817 self.buffer = dataBlock
2705 2818 self.first_utcBlock = self.__initime
2706 2819 dataOut.utctime = self.__initime
2707 2820 dataOut.nProfiles = self.__profIndex
2708 2821 #dataOut.flagNoData = False
2709 2822 self.init_prof = 0
2710 2823 self.__profIndex = 0
2711 2824 self.__initime = None
2712 2825 dataBlock = None
2713 2826 self.__buffer_times = []
2714 2827 dataOut.error = False
2715 2828 dataOut.useInputBuffer = True
2716 2829 dataOut.buffer_empty = False
2717 2830 #print("1 ch: {} prof: {} hs: {}".format(int(dataOut.nChannels),int(dataOut.nProfiles),int(dataOut.nHeights)))
2718 2831
2719 2832
2720 2833
2721 2834 #print(self.__count_exec)
2722 2835
2723 2836 return dataOut
2724 2837
2838
2725 2839 class RemoveProfileSats(Operation):
2726 2840 '''
2727 Omite los perfiles contaminados con señal de satelites,
2841 Omite los perfiles contaminados con señal de satélites, usando una altura de referencia
2728 2842 In: minHei = min_sat_range
2729 2843 max_sat_range
2730 2844 min_hei_ref
2731 2845 max_hei_ref
2732 2846 th = diference between profiles mean, ref and sats
2733 2847 Out:
2734 2848 profile clean
2735 2849 '''
2736 2850
2737 isConfig = False
2738 min_sats = 0
2739 max_sats = 999999999
2740 min_ref= 0
2741 max_ref= 9999999999
2742 needReshape = False
2743 count = 0
2744 thdB = 0
2745 byRanges = False
2746 min_sats = None
2747 max_sats = None
2748 noise = 0
2749 2851
2852 __buffer_data = []
2853 __buffer_times = []
2854
2855 buffer = None
2856
2857 outliers_IDs_list = []
2858
2859
2860 __slots__ = ('n','navg','profileMargin','thHistOutlier','minHei_idx','maxHei_idx','nHeights',
2861 'first_utcBlock','__profIndex','init_prof','end_prof','lenProfileOut','nChannels',
2862 '__count_exec','__initime','__dataReady','__ipp', 'minRef', 'maxRef', 'thdB')
2750 2863 def __init__(self, **kwargs):
2751 2864
2752 2865 Operation.__init__(self, **kwargs)
2753 2866 self.isConfig = False
2754 2867
2868 def setup(self,dataOut, n=None , navg=0.8, profileMargin=50,thHistOutlier=3,
2869 minHei=None, maxHei=None, minRef=None, maxRef=None, thdB=10):
2870
2871 if n == None and timeInterval == None:
2872 raise ValueError("nprofiles or timeInterval should be specified ...")
2755 2873
2756 def setup(self, dataOut, minHei, maxHei, minRef, maxRef, th, thdB, rangeHeiList):
2874 if n != None:
2875 self.n = n
2757 2876
2758 if rangeHeiList!=None:
2759 self.byRanges = True
2760 else:
2761 if minHei==None or maxHei==None :
2762 raise ValueError("Parameters heights are required")
2763 if minRef==None or maxRef==None:
2764 raise ValueError("Parameters heights are required")
2765
2766 if self.byRanges:
2767 self.min_sats = []
2768 self.max_sats = []
2769 for min,max in rangeHeiList:
2770 a,b = getHei_index(min, max, dataOut.heightList)
2771 self.min_sats.append(a)
2772 self.max_sats.append(b)
2773 else:
2774 self.min_sats, self.max_sats = getHei_index(minHei, maxHei, dataOut.heightList)
2877 self.navg = navg
2878 self.profileMargin = profileMargin
2879 self.thHistOutlier = thHistOutlier
2880 self.__profIndex = 0
2881 self.buffer = None
2882 self._ipp = dataOut.ippSeconds
2883 self.n_prof_released = 0
2884 self.heightList = dataOut.heightList
2885 self.init_prof = 0
2886 self.end_prof = 0
2887 self.__count_exec = 0
2888 self.__profIndex = 0
2889 self.first_utcBlock = None
2890 #self.__dh = dataOut.heightList[1] - dataOut.heightList[0]
2891 minHei = minHei
2892 maxHei = maxHei
2893 if minHei==None :
2894 minHei = dataOut.heightList[0]
2895 if maxHei==None :
2896 maxHei = dataOut.heightList[-1]
2897 self.minHei_idx,self.maxHei_idx = getHei_index(minHei, maxHei, dataOut.heightList)
2775 2898 self.min_ref, self.max_ref = getHei_index(minRef, maxRef, dataOut.heightList)
2776 self.th = th
2899 self.nChannels = dataOut.nChannels
2900 self.nHeights = dataOut.nHeights
2901 self.test_counter = 0
2777 2902 self.thdB = thdB
2778 self.isConfig = True
2779 2903
2904 def filterSatsProfiles(self):
2905 data = self.__buffer_data
2906 #print(data.shape)
2907 nChannels, profiles, heights = data.shape
2908 indexes=numpy.zeros([], dtype=int)
2909 outliers_IDs=[]
2910 for c in range(nChannels):
2911 #print(self.min_ref,self.max_ref)
2912 noise_ref = 10* numpy.log10((data[c,:,self.min_ref:self.max_ref] * numpy.conjugate(data[c,:,self.min_ref:self.max_ref])).real)
2913 #print("Noise ",numpy.percentile(noise_ref,95))
2914 p95 = numpy.percentile(noise_ref,95)
2915 noise_ref = noise_ref.mean()
2916 #print("Noise ",noise_ref
2780 2917
2781 def compareRanges(self,data, minHei,maxHei):
2782 2918
2783 # ref = data[0,self.min_ref:self.max_ref] * numpy.conjugate(data[0,self.min_ref:self.max_ref])
2784 # p_ref = 10*numpy.log10(ref.real)
2785 # m_ref = numpy.mean(p_ref)
2919 for h in range(self.minHei_idx, self.maxHei_idx):
2920 power = 10* numpy.log10((data[c,:,h] * numpy.conjugate(data[c,:,h])).real)
2921 #th = noise_ref + self.thdB
2922 th = noise_ref + 1.5*(p95-noise_ref)
2923 index = numpy.where(power > th )
2924 if index[0].size > 10 and index[0].size < int(self.navg*profiles):
2925 indexes = numpy.append(indexes, index[0])
2926 #print(index[0])
2927 #print(index[0])
2786 2928
2787 m_ref = self.noise
2929 # fig,ax = plt.subplots()
2930 # #ax.set_title(str(k)+" "+str(j))
2931 # x=range(len(power))
2932 # ax.scatter(x,power)
2933 # #ax.axvline(index)
2934 # plt.grid()
2935 # plt.show()
2936 #print(indexes)
2788 2937
2789 sats = data[0,minHei:maxHei] * numpy.conjugate(data[0,minHei:maxHei])
2790 p_sats = 10*numpy.log10(sats.real)
2791 m_sats = numpy.mean(p_sats)
2938 #outliers_IDs = outliers_IDs.astype(numpy.dtype('int64'))
2939 #outliers_IDs = numpy.unique(outliers_IDs)
2792 2940
2793 if m_sats > (m_ref + self.th): #and (m_sats > self.thdB):
2794 #print("msats: ",m_sats," \tmRef: ", m_ref, "\t",(m_sats - m_ref))
2795 #print("Removing profiles...")
2796 return False
2941 outs_lines = numpy.unique(indexes)
2797 2942
2798 return True
2799 2943
2800 def isProfileClean(self, data):
2801 '''
2802 Analiza solo 1 canal, y descarta todos...
2803 '''
2944 #Agrupando el histograma de outliers,
2945 my_bins = numpy.linspace(0,int(profiles), int(profiles/100), endpoint=True)
2804 2946
2805 clean = True
2806 2947
2807 if self.byRanges:
2948 hist, bins = numpy.histogram(outs_lines,bins=my_bins)
2949 hist_outliers_indexes = numpy.where(hist > self.thHistOutlier) #es outlier
2950 hist_outliers_indexes = hist_outliers_indexes[0]
2951 if len(hist_outliers_indexes>0):
2952 hist_outliers_indexes = numpy.append(hist_outliers_indexes,hist_outliers_indexes[-1]+1)
2953 #print(hist_outliers_indexes)
2954 #print(bins, hist_outliers_indexes)
2955 bins_outliers_indexes = [int(i) for i in (bins[hist_outliers_indexes])] #
2956
2957 outlier_loc_index = [e for n in range(len(bins_outliers_indexes)) for e in range(bins_outliers_indexes[n]-self.profileMargin,bins_outliers_indexes[n]+ profiles//100 + self.profileMargin) ]
2958 outlier_loc_index = numpy.asarray(outlier_loc_index)
2959
2960 #print("outliers Ids: ", outlier_loc_index, outlier_loc_index.shape)
2961 outlier_loc_index = outlier_loc_index[ (outlier_loc_index >= 0) & (outlier_loc_index<profiles)]
2962 #print("outliers final: ", outlier_loc_index)
2963
2964 # from matplotlib import pyplot as plt
2965 # x, y = numpy.meshgrid(numpy.arange(profiles), self.heightList)
2966 # fig, ax = plt.subplots(1,2,figsize=(8, 6))
2967 # dat = data[0,:,:].real
2968 # dat = 10* numpy.log10((data[0,:,:] * numpy.conjugate(data[0,:,:])).real)
2969 # m = numpy.nanmean(dat)
2970 # o = numpy.nanstd(dat)
2971 # #print(m, o, x.shape, y.shape)
2972 # c = ax[0].pcolormesh(x, y, dat.T, cmap ='YlGnBu', vmin = (m-2*o), vmax = (m+2*o))
2973 # ax[0].vlines(outs_lines,200,600, linestyles='dashed', label = 'outs', color='w')
2974 # fig.colorbar(c)
2975 # ax[0].vlines(outlier_loc_index,650,750, linestyles='dashed', label = 'outs', color='r')
2976 # ax[1].hist(outs_lines,bins=my_bins)
2977 # plt.show()
2978
2979
2980 self.outliers_IDs_list = outlier_loc_index
2981 #print("outs list: ", self.outliers_IDs_list)
2982 return data
2983
2984
2985
2986 def fillBuffer(self, data, datatime):
2987
2988 if self.__profIndex == 0:
2989 self.__buffer_data = data.copy()
2808 2990
2809 for n in range(len(self.min_sats)):
2810 c = self.compareRanges(data,self.min_sats[n],self.max_sats[n])
2811 clean = clean and c
2812 2991 else:
2992 self.__buffer_data = numpy.concatenate((self.__buffer_data,data), axis=1)#en perfiles
2993 self.__profIndex += 1
2994 self.__buffer_times.append(datatime)
2813 2995
2814 clean = (self.compareRanges(data, self.min_sats,self.max_sats))
2996 def getData(self, data, datatime=None):
2815 2997
2816 return clean
2998 if self.__profIndex == 0:
2999 self.__initime = datatime
2817 3000
2818 3001
3002 self.__dataReady = False
2819 3003
2820 def run(self, dataOut, minHei=None, maxHei=None, minRef=None, maxRef=None, th=5, thdB=65, rangeHeiList=None):
2821 dataOut.flagNoData = True
3004 self.fillBuffer(data, datatime)
3005 dataBlock = None
3006
3007 if self.__profIndex == self.n:
3008 #print("apnd : ",data)
3009 dataBlock = self.filterSatsProfiles()
3010 self.__dataReady = True
3011
3012 return dataBlock
3013
3014 if dataBlock is None:
3015 return None, None
3016
3017
3018
3019 return dataBlock
3020
3021 def releaseBlock(self):
3022
3023 if self.n % self.lenProfileOut != 0:
3024 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n))
3025 return None
3026
3027 data = self.buffer[:,self.init_prof:self.end_prof:,:] #ch, prof, alt
3028
3029 self.init_prof = self.end_prof
3030 self.end_prof += self.lenProfileOut
3031 #print("data release shape: ",dataOut.data.shape, self.end_prof)
3032 self.n_prof_released += 1
3033
3034 return data
3035
3036 def run(self, dataOut, n=None, navg=0.75, nProfilesOut=1, profile_margin=50,
3037 th_hist_outlier=3,minHei=None, maxHei=None, minRef=None, maxRef=None, thdB=10):
2822 3038
2823 3039 if not self.isConfig:
2824 self.setup(dataOut, minHei, maxHei, minRef, maxRef, th, thdB, rangeHeiList)
3040 #print("init p idx: ", dataOut.profileIndex )
3041 self.setup(dataOut,n=n, navg=navg,profileMargin=profile_margin,thHistOutlier=th_hist_outlier,
3042 minHei=minHei, maxHei=maxHei, minRef=minRef, maxRef=maxRef, thdB=thdB)
2825 3043 self.isConfig = True
2826 #print(self.min_sats,self.max_sats)
2827 if dataOut.flagDataAsBlock:
2828 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
2829 3044
3045 dataBlock = None
3046
3047 if not dataOut.buffer_empty: #hay datos acumulados
3048
3049 if self.init_prof == 0:
3050 self.n_prof_released = 0
3051 self.lenProfileOut = nProfilesOut
3052 dataOut.flagNoData = False
3053 #print("tp 2 ",dataOut.data.shape)
3054
3055 self.init_prof = 0
3056 self.end_prof = self.lenProfileOut
3057
3058 dataOut.nProfiles = self.lenProfileOut
3059 if nProfilesOut == 1:
3060 dataOut.flagDataAsBlock = False
2830 3061 else:
2831 self.noise =10*numpy.log10(dataOut.getNoisebyHildebrand(ymin_index=self.min_ref, ymax_index=self.max_ref))
2832 if not self.isProfileClean(dataOut.data):
3062 dataOut.flagDataAsBlock = True
3063 #print("prof: ",self.init_prof)
3064 dataOut.flagNoData = False
3065 if numpy.isin(self.n_prof_released, self.outliers_IDs_list):
3066 #print("omitting: ", self.n_prof_released)
3067 dataOut.flagNoData = True
3068 dataOut.ippSeconds = self._ipp
3069 dataOut.utctime = self.first_utcBlock + self.init_prof*self._ipp
3070 # print("time: ", dataOut.utctime, self.first_utcBlock, self.init_prof,self._ipp,dataOut.ippSeconds)
3071 #dataOut.data = self.releaseBlock()
3072 #########################################################3
3073 if self.n % self.lenProfileOut != 0:
3074 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n))
3075 return None
3076
3077 dataOut.data = self.buffer[:,self.init_prof:self.end_prof:,:] #ch, prof, alt
3078
3079 self.init_prof = self.end_prof
3080 self.end_prof += self.lenProfileOut
3081 #print("data release shape: ",dataOut.data.shape, self.end_prof, dataOut.flagNoData)
3082 self.n_prof_released += 1
3083
3084 if self.end_prof >= (self.n +self.lenProfileOut):
3085
3086 self.init_prof = 0
3087 self.__profIndex = 0
3088 self.buffer = None
3089 dataOut.buffer_empty = True
3090 self.outliers_IDs_list = []
3091 self.n_prof_released = 0
3092 dataOut.flagNoData = False #enviar ultimo aunque sea outlier :(
3093 #print("cleaning...", dataOut.buffer_empty)
3094 dataOut.profileIndex = 0 #self.lenProfileOut
3095 ####################################################################
2833 3096 return dataOut
2834 #dataOut.data = numpy.full((dataOut.nChannels,dataOut.nHeights),numpy.NAN)
2835 #self.count += 1
2836 3097
2837 dataOut.flagNoData = False
3098
3099 #print("tp 223 ",dataOut.data.shape)
3100 dataOut.flagNoData = True
3101
3102
3103
3104 try:
3105 #dataBlock = self.getData(dataOut.data.reshape(self.nChannels,1,self.nHeights), dataOut.utctime)
3106 dataBlock = self.getData(numpy.reshape(dataOut.data,(self.nChannels,1,self.nHeights)), dataOut.utctime)
3107 self.__count_exec +=1
3108 except Exception as e:
3109 print("Error getting profiles data",self.__count_exec )
3110 print(e)
3111 sys.exit()
3112
3113 if self.__dataReady:
3114 #print("omitting: ", len(self.outliers_IDs_list))
3115 self.__count_exec = 0
3116 #dataOut.data =
3117 #self.buffer = numpy.flip(dataBlock, axis=1)
3118 self.buffer = dataBlock
3119 self.first_utcBlock = self.__initime
3120 dataOut.utctime = self.__initime
3121 dataOut.nProfiles = self.__profIndex
3122 #dataOut.flagNoData = False
3123 self.init_prof = 0
3124 self.__profIndex = 0
3125 self.__initime = None
3126 dataBlock = None
3127 self.__buffer_times = []
3128 dataOut.error = False
3129 dataOut.useInputBuffer = True
3130 dataOut.buffer_empty = False
3131 #print("1 ch: {} prof: {} hs: {}".format(int(dataOut.nChannels),int(dataOut.nProfiles),int(dataOut.nHeights)))
3132
3133
3134
3135 #print(self.__count_exec)
2838 3136
2839 3137 return dataOut
General Comments 0
You need to be logged in to leave comments. Login now