##// END OF EJS Templates
schain
RSS
simple plot amisr-drift?
joabAM -
r1671:65a6f6a9ba7b
parent child
Show More
Side by Side Unified
Context file:
r1671:65a6f6a9ba7b Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
1 NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,1166 +1,1167
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Definition of diferent Data objects for different types of data
6 6
7 7 Here you will find the diferent data objects for the different types
8 8 of data, this data objects must be used as dataIn or dataOut objects in
9 9 processing units and operations. Currently the supported data objects are:
10 10 Voltage, Spectra, SpectraHeis, Fits, Correlation and Parameters
11 11 """
12 12
13 13 import copy
14 14 import numpy
15 15 import datetime
16 16 import json
17 17
18 18 import schainpy.admin
19 19 from schainpy.utils import log
20 20 from .jroheaderIO import SystemHeader, RadarControllerHeader,ProcessingHeader
21 21 from schainpy.model.data import _noise
22 22 SPEED_OF_LIGHT = 3e8
23 23
24 24 def getNumpyDtype(dataTypeCode):
25 25
26 26 if dataTypeCode == 0:
27 27 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
28 28 elif dataTypeCode == 1:
29 29 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
30 30 elif dataTypeCode == 2:
31 31 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
32 32 elif dataTypeCode == 3:
33 33 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
34 34 elif dataTypeCode == 4:
35 35 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
36 36 elif dataTypeCode == 5:
37 37 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
38 38 else:
39 39 raise ValueError('dataTypeCode was not defined')
40 40
41 41 return numpyDtype
42 42
43 43
44 44 def getDataTypeCode(numpyDtype):
45 45
46 46 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
47 47 datatype = 0
48 48 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
49 49 datatype = 1
50 50 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
51 51 datatype = 2
52 52 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
53 53 datatype = 3
54 54 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
55 55 datatype = 4
56 56 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
57 57 datatype = 5
58 58 else:
59 59 datatype = None
60 60
61 61 return datatype
62 62
63 63
64 64 def hildebrand_sekhon(data, navg):
65 65 """
66 66 This method is for the objective determination of the noise level in Doppler spectra. This
67 67 implementation technique is based on the fact that the standard deviation of the spectral
68 68 densities is equal to the mean spectral density for white Gaussian noise
69 69
70 70 Inputs:
71 71 Data : heights
72 72 navg : numbers of averages
73 73
74 74 Return:
75 75 mean : noise's level
76 76 """
77 77
78 78 sortdata = numpy.sort(data, axis=None)
79 79 '''
80 80 lenOfData = len(sortdata)
81 81 nums_min = lenOfData*0.5
82 82
83 83 if nums_min <= 5:
84 84
85 85 nums_min = 5
86 86
87 87 sump = 0.
88 88 sumq = 0.
89 89
90 90 j = 0
91 91 cont = 1
92 92
93 93 while((cont == 1)and(j < lenOfData)):
94 94
95 95 sump += sortdata[j]
96 96 sumq += sortdata[j]**2
97 97
98 98 if j > nums_min:
99 99 rtest = float(j)/(j-1) + 1.0/navg
100 100 if ((sumq*j) > (rtest*sump**2)):
101 101 j = j - 1
102 102 sump = sump - sortdata[j]
103 103 sumq = sumq - sortdata[j]**2
104 104 cont = 0
105 105
106 106 j += 1
107 107
108 108 lnoise = sump / j
109 109 return lnoise
110 110 '''
111 111 return _noise.hildebrand_sekhon(sortdata, navg)
112 112
113 113
114 114 class Beam:
115 115
116 116 def __init__(self):
117 117 self.codeList = []
118 118 self.azimuthList = []
119 119 self.zenithList = []
120 120
121 121
122 122
123 123 class GenericData(object):
124 124
125 125 flagNoData = True
126 126
127 127 def copy(self, inputObj=None):
128 128
129 129 if inputObj == None:
130 130 return copy.deepcopy(self)
131 131
132 132 for key in list(inputObj.__dict__.keys()):
133 133
134 134 attribute = inputObj.__dict__[key]
135 135
136 136 # If this attribute is a tuple or list
137 137 if type(inputObj.__dict__[key]) in (tuple, list):
138 138 self.__dict__[key] = attribute[:]
139 139 continue
140 140
141 141 # If this attribute is another object or instance
142 142 if hasattr(attribute, '__dict__'):
143 143 self.__dict__[key] = attribute.copy()
144 144 continue
145 145
146 146 self.__dict__[key] = inputObj.__dict__[key]
147 147
148 148 def deepcopy(self):
149 149
150 150 return copy.deepcopy(self)
151 151
152 152 def isEmpty(self):
153 153
154 154 return self.flagNoData
155 155
156 156 def isReady(self):
157 157
158 158 return not self.flagNoData
159 159
160 160
161 161 class JROData(GenericData):
162 162
163 163 useInputBuffer = False
164 164 buffer_empty = True
165 165
166 166 systemHeaderObj = SystemHeader()
167 167 radarControllerHeaderObj = RadarControllerHeader()
168 168 type = None
169 169 datatype = None # dtype but in string
170 170 nProfiles = None
171 171 heightList = None
172 172 channelList = None
173 173 flagDiscontinuousBlock = False
174 174 useLocalTime = False
175 175 utctime = None
176 176 timeZone = None
177 177 dstFlag = None
178 178 errorCount = None
179 179 blocksize = None
180 180 flagDecodeData = False # asumo q la data no esta decodificada
181 181 flagDeflipData = False # asumo q la data no esta sin flip
182 182 flagShiftFFT = False
183 183 nCohInt = None
184 184 windowOfFilter = 1
185 185 C = 3e8
186 186 frequency = 49.92e6
187 187 realtime = False
188 188 beacon_heiIndexList = None
189 189 last_block = None
190 190 blocknow = None
191 191 azimuth = None
192 192 zenith = None
193 193
194 194 ippFactor = 1 #Added to correct the freq and vel range for AMISR data
195 195
196 196 profileIndex = None
197 197 error = None
198 198 data = None
199 199 nmodes = None
200 200 metadata_list = ['heightList', 'timeZone', 'type']
201 201 codeList = []
202 202 azimuthList = []
203 203 elevationList = []
204 204 last_noise = None
205 205 __ipp = None
206 206 __ippSeconds = None
207 207 sampled_heightsFFT = None
208 208 pulseLength_TxA = None
209 209 deltaHeight = None
210 210 __code = None
211 211 __nCode = None
212 212 __nBaud = None
213 213 unitsDescription = "The units of the parameters are according to the International System of units (Seconds, Meter, Hertz, ...), except \
214 214 the parameters related to distances such as heightList, or heightResolution wich are in Km"
215 215
216 216
217 217 def __str__(self):
218 218
219 219 return '{} - {}'.format(self.type, self.datatime())
220 220
221 221 def getNoise(self):
222 222
223 223 raise NotImplementedError
224 224
225 225 @property
226 226 def nChannels(self):
227 227
228 228 return len(self.channelList)
229 229
230 230 @property
231 231 def channelIndexList(self):
232 232
233 233 return list(range(self.nChannels))
234 234
235 235 @property
236 236 def nHeights(self):
237 237
238 238 return len(self.heightList)
239 239
240 240 def getDeltaH(self):
241 241
242 242 return self.heightList[1] - self.heightList[0]
243 243
244 244 @property
245 245 def ltctime(self):
246 246
247 247 if self.useLocalTime:
248 248 return self.utctime - self.timeZone * 60
249 249
250 250 return self.utctime
251 251
252 252 @property
253 253 def datatime(self):
254 254
255 255 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
256 256 return datatimeValue
257 257
258 258 def getTimeRange(self):
259 259
260 260 datatime = []
261 261
262 262 datatime.append(self.ltctime)
263 263 datatime.append(self.ltctime + self.timeInterval + 1)
264 264
265 265 datatime = numpy.array(datatime)
266 266
267 267 return datatime
268 268
269 269 def getFmaxTimeResponse(self):
270 270
271 271 period = (10**-6) * self.getDeltaH() / (0.15)
272 272
273 273 PRF = 1. / (period * self.nCohInt)
274 274
275 275 fmax = PRF
276 276
277 277 return fmax
278 278
279 279 def getFmax(self):
280 280 PRF = 1. / (self.__ippSeconds * self.nCohInt)
281 281
282 282 fmax = PRF
283 283 return fmax
284 284
285 285 def getVmax(self):
286 286
287 287 _lambda = self.C / self.frequency
288 288
289 289 vmax = self.getFmax() * _lambda / 2
290 290
291 291 return vmax
292 292
293 293 @property
294 294 def ippSeconds(self):
295 295 '''
296 296 '''
297 297 #return self.radarControllerHeaderObj.ippSeconds
298 298 return self.__ippSeconds
299 299
300 300 @ippSeconds.setter
301 301 def ippSeconds(self, ippSeconds):
302 302 '''
303 303 '''
304 304 #self.radarControllerHeaderObj.ippSeconds = ippSeconds
305 305 self.__ippSeconds = ippSeconds
306 306 self.__ipp = ippSeconds*SPEED_OF_LIGHT/2000.0
307 307
308 308
309 309 @property
310 310 def code(self):
311 311 '''
312 312 '''
313 313 return self.__code
314 314
315 315 @code.setter
316 316 def code(self, code):
317 317 '''
318 318 '''
319 319 self.__code = code
320 320 #
321 321 @property
322 322 def nCode(self):
323 323 '''
324 324 '''
325 325 return self.__nCode
326 326
327 327 @nCode.setter
328 328 def nCode(self, ncode):
329 329 '''
330 330 '''
331 331 self.__nCode = ncode
332 332
333 333 @property
334 334 def nBaud(self):
335 335 '''
336 336 '''
337 337 return self.__nBaud
338 338
339 339 @nBaud.setter
340 340 def nBaud(self, nbaud):
341 341 '''
342 342 '''
343 343 self.__nBaud = nbaud
344 344
345 345 @property
346 346 def ipp(self):
347 347 '''
348 348 '''
349 349 return self.__ipp
350 350 #return self.radarControllerHeaderObj.ipp
351 351
352 352 @ipp.setter
353 353 def ipp(self, ipp):
354 354 '''
355 355 '''
356 356 self.__ipp = ipp
357 357 #self.radarControllerHeaderObj.ipp = ipp
358 358
359 359 @property
360 360 def metadata(self):
361 361 '''
362 362 '''
363 363
364 364 return {attr: getattr(self, attr) for attr in self.metadata_list}
365 365
366 366
367 367 class Voltage(JROData):
368 368
369 369 dataPP_POW = None
370 370 dataPP_DOP = None
371 371 dataPP_WIDTH = None
372 372 dataPP_SNR = None
373 373 flagProfilesByRange = False
374 374 nProfilesByRange = None
375 375 max_nIncohInt = 1
376 376 def __init__(self):
377 377 '''
378 378 Constructor
379 379 '''
380 380
381 381 self.useLocalTime = True
382 382 self.radarControllerHeaderObj = RadarControllerHeader()
383 383 self.systemHeaderObj = SystemHeader()
384 384 self.processingHeaderObj = ProcessingHeader()
385 385 self.type = "Voltage"
386 386 self.data = None
387 387 self.nProfiles = None
388 388 self.heightList = None
389 389 self.channelList = None
390 390 self.flagNoData = True
391 391 self.flagDiscontinuousBlock = False
392 392 self.utctime = None
393 393 self.timeZone = 0
394 394 self.dstFlag = None
395 395 self.errorCount = None
396 396 self.nCohInt = None
397 397 self.blocksize = None
398 398 self.flagCohInt = False
399 399 self.flagDecodeData = False # asumo q la data no esta decodificada
400 400 self.flagDeflipData = False # asumo q la data no esta sin flip
401 401 self.flagShiftFFT = False
402 402 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
403 403 self.profileIndex = 0
404 404 self.ippFactor=1
405 405 self.metadata_list = ['type', 'heightList', 'timeZone', 'nProfiles', 'channelList', 'nCohInt',
406 406 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp']
407 407
408 408 def getNoisebyHildebrand(self, channel=None, ymin_index=None, ymax_index=None):
409 409 """
410 410 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
411 411
412 412 Return:
413 413 noiselevel
414 414 """
415 415
416 416 if channel != None:
417 417 data = self.data[channel,ymin_index:ymax_index]
418 418 nChannels = 1
419 419 else:
420 420 data = self.data[:,ymin_index:ymax_index]
421 421 nChannels = self.nChannels
422 422
423 423 noise = numpy.zeros(nChannels)
424 424 power = data * numpy.conjugate(data)
425 425
426 426 for thisChannel in range(nChannels):
427 427 if nChannels == 1:
428 428 daux = power[:].real
429 429 else:
430 430 daux = power[thisChannel, :].real
431 431 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
432 432
433 433 return noise
434 434
435 435 def getNoise(self, type=1, channel=None,ymin_index=None, ymax_index=None):
436 436
437 437 if type == 1:
438 438 noise = self.getNoisebyHildebrand(channel,ymin_index, ymax_index)
439 439
440 440 return noise
441 441
442 442 def getPower(self, channel=None):
443 443
444 444 if channel != None:
445 445 data = self.data[channel]
446 446 else:
447 447 data = self.data
448 448
449 449 power = data * numpy.conjugate(data)
450 450 powerdB = 10 * numpy.log10(power.real)
451 451 powerdB = numpy.squeeze(powerdB)
452 452
453 453 return powerdB
454 454
455 455 @property
456 456 def timeInterval(self):
457 457
458 458 return self.ippSeconds * self.nCohInt
459 459
460 460 noise = property(getNoise, "I'm the 'nHeights' property.")
461 461
462 462
463 463 class Spectra(JROData):
464 464
465 465 data_outlier = None
466 466 flagProfilesByRange = False
467 467 nProfilesByRange = None
468 468
469 469 def __init__(self):
470 470 '''
471 471 Constructor
472 472 '''
473 473
474 474 self.data_dc = None
475 475 self.data_spc = None
476 476 self.data_cspc = None
477 477 self.useLocalTime = True
478 478 self.radarControllerHeaderObj = RadarControllerHeader()
479 479 self.systemHeaderObj = SystemHeader()
480 480 self.processingHeaderObj = ProcessingHeader()
481 481 self.type = "Spectra"
482 482 self.timeZone = 0
483 483 self.nProfiles = None
484 484 self.heightList = None
485 485 self.channelList = None
486 486 self.pairsList = None
487 487 self.flagNoData = True
488 488 self.flagDiscontinuousBlock = False
489 489 self.utctime = None
490 490 self.nCohInt = None
491 491 self.nIncohInt = None
492 492 self.blocksize = None
493 493 self.nFFTPoints = None
494 494 self.wavelength = None
495 495 self.flagDecodeData = False # asumo q la data no esta decodificada
496 496 self.flagDeflipData = False # asumo q la data no esta sin flip
497 497 self.flagShiftFFT = False
498 498 self.ippFactor = 1
499 499 self.beacon_heiIndexList = []
500 500 self.noise_estimation = None
501 501 self.codeList = []
502 502 self.azimuthList = []
503 503 self.elevationList = []
504 504 self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt',
505 505 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp','nIncohInt', 'nFFTPoints', 'nProfiles']
506 506
507 507
508 508
509 509
510 510 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
511 511 """
512 512 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
513 513
514 514 Return:
515 515 noiselevel
516 516 """
517 517 # if hasattr(self.nIncohInt, "__len__"): #nIncohInt is a matrix
518 518 #
519 519 # heis = self.data_spc.shape[2]
520 520 #
521 521 # noise = numpy.zeros((self.nChannels, heis))
522 522 # for hei in range(heis):
523 523 # for channel in range(self.nChannels):
524 524 # daux = self.data_spc[channel, xmin_index:xmax_index, hei]
525 525 #
526 526 # noise[channel,hei] = hildebrand_sekhon(daux, self.nIncohInt[channel,hei])
527 527 #
528 528 # else:
529 529 # noise = numpy.zeros(self.nChannels)
530 530 # for channel in range(self.nChannels):
531 531 # daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
532 532 #
533 533 # noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
534 534 noise = numpy.zeros(self.nChannels)
535 535
536 536 for channel in range(self.nChannels):
537 537 daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
538 538
539 539 noise[channel] = hildebrand_sekhon(daux, self.max_nIncohInt[channel])
540 540
541 541 return noise
542 542
543 543 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
544 544
545 545 if self.noise_estimation is not None:
546 546 # this was estimated by getNoise Operation defined in jroproc_spectra.py
547 547 return self.noise_estimation
548 548 else:
549 549 noise = self.getNoisebyHildebrand(
550 550 xmin_index, xmax_index, ymin_index, ymax_index)
551 551 return noise
552 552
553 553 def getFreqRangeTimeResponse(self, extrapoints=0):
554 554
555 555 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
556 556 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
557 557
558 558 return freqrange
559 559
560 560 def getAcfRange(self, extrapoints=0):
561 561
562 562 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
563 563 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
564 564
565 565 return freqrange
566 566
567 567 def getFreqRange(self, extrapoints=0):
568 568
569 569 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
570 570 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
571 571
572 572 return freqrange
573 573
574 574 def getVelRange(self, extrapoints=0):
575 575
576 576 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
577 577 velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
578 578
579 579 if self.nmodes:
580 580 return velrange/self.nmodes
581 581 else:
582 582 return velrange
583 583
584 584 @property
585 585 def nPairs(self):
586 586
587 587 return len(self.pairsList)
588 588
589 589 @property
590 590 def pairsIndexList(self):
591 591
592 592 return list(range(self.nPairs))
593 593
594 594 @property
595 595 def normFactor(self):
596 596
597 597 pwcode = 1
598 598 if self.flagDecodeData:
599 599 try:
600 600 pwcode = numpy.sum(self.code[0]**2)
601 601 except Exception as e:
602 602 log.warning("Failed pwcode read, setting to 1")
603 603 pwcode = 1
604 604 # print(self.flagDecodeData, pwcode)
605 605 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
606 606 normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
607 607 if self.flagProfilesByRange:
608 608 normFactor *= (self.nProfilesByRange/self.nProfilesByRange.max())
609 609 #print("normFactor: ", normFactor)
610 610 return normFactor
611 611
612 612 @property
613 613 def flag_cspc(self):
614 614
615 615 if self.data_cspc is None:
616 616 return True
617 617
618 618 return False
619 619
620 620 @property
621 621 def flag_dc(self):
622 622
623 623 if self.data_dc is None:
624 624 return True
625 625
626 626 return False
627 627
628 628 @property
629 629 def timeInterval(self):
630 630
631 631 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
632 632 if self.nmodes:
633 633 return self.nmodes*timeInterval
634 634 else:
635 635 return timeInterval
636 636
637 637 def getPower(self):
638 638
639 639 factor = self.normFactor
640 640 power = numpy.zeros( (self.nChannels,self.nHeights) )
641 641 for ch in range(self.nChannels):
642 642 z = None
643 643 if hasattr(factor,'shape'):
644 644 if factor.ndim > 1:
645 645 z = self.data_spc[ch]/factor[ch]
646 646 else:
647 647 z = self.data_spc[ch]/factor
648 648 else:
649 649 z = self.data_spc[ch]/factor
650 650 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
651 651 avg = numpy.average(z, axis=0)
652 652 power[ch] = 10 * numpy.log10(avg)
653 653 return power
654 654
655 655 @property
656 656 def max_nIncohInt(self):
657 657
658 658 ints = numpy.zeros(self.nChannels)
659 659 for ch in range(self.nChannels):
660 660 if hasattr(self.nIncohInt,'shape'):
661 661 if self.nIncohInt.ndim > 1:
662 662 ints[ch,] = self.nIncohInt[ch].max()
663 663 else:
664 664 ints[ch,] = self.nIncohInt
665 665 self.nIncohInt = int(self.nIncohInt)
666 666 else:
667 667 ints[ch,] = self.nIncohInt
668 668
669 669 return ints
670 670
671 671
672 672 def getCoherence(self, pairsList=None, phase=False):
673 673
674 674 z = []
675 675 if pairsList is None:
676 676 pairsIndexList = self.pairsIndexList
677 677 else:
678 678 pairsIndexList = []
679 679 for pair in pairsList:
680 680 if pair not in self.pairsList:
681 681 raise ValueError("Pair %s is not in dataOut.pairsList" % (
682 682 pair))
683 683 pairsIndexList.append(self.pairsList.index(pair))
684 684 for i in range(len(pairsIndexList)):
685 685 pair = self.pairsList[pairsIndexList[i]]
686 686 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
687 687 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
688 688 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
689 689 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
690 690 if phase:
691 691 data = numpy.arctan2(avgcoherenceComplex.imag,
692 692 avgcoherenceComplex.real) * 180 / numpy.pi
693 693 else:
694 694 data = numpy.abs(avgcoherenceComplex)
695 695
696 696 z.append(data)
697 697
698 698 return numpy.array(z)
699 699
700 700 def setValue(self, value):
701 701
702 702 print("This property should not be initialized", value)
703 703
704 704 return
705 705
706 706 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
707 707
708 708
709 709 class SpectraHeis(Spectra):
710 710
711 711 def __init__(self):
712 712
713 713 self.radarControllerHeaderObj = RadarControllerHeader()
714 714 self.systemHeaderObj = SystemHeader()
715 715 self.type = "SpectraHeis"
716 716 self.nProfiles = None
717 717 self.heightList = None
718 718 self.channelList = None
719 719 self.flagNoData = True
720 720 self.flagDiscontinuousBlock = False
721 721 self.utctime = None
722 722 self.blocksize = None
723 723 self.profileIndex = 0
724 724 self.nCohInt = 1
725 725 self.nIncohInt = 1
726 726
727 727 @property
728 728 def normFactor(self):
729 729 pwcode = 1
730 730 if self.flagDecodeData:
731 731 pwcode = numpy.sum(self.code[0]**2)
732 732
733 733 normFactor = self.nIncohInt * self.nCohInt * pwcode
734 734
735 735 return normFactor
736 736
737 737 @property
738 738 def timeInterval(self):
739 739
740 740 return self.ippSeconds * self.nCohInt * self.nIncohInt
741 741
742 742
743 743 class Fits(JROData):
744 744
745 745 def __init__(self):
746 746
747 747 self.type = "Fits"
748 748 self.nProfiles = None
749 749 self.heightList = None
750 750 self.channelList = None
751 751 self.flagNoData = True
752 752 self.utctime = None
753 753 self.nCohInt = 1
754 754 self.nIncohInt = 1
755 755 self.useLocalTime = True
756 756 self.profileIndex = 0
757 757 self.timeZone = 0
758 758
759 759 def getTimeRange(self):
760 760
761 761 datatime = []
762 762
763 763 datatime.append(self.ltctime)
764 764 datatime.append(self.ltctime + self.timeInterval)
765 765
766 766 datatime = numpy.array(datatime)
767 767
768 768 return datatime
769 769
770 770 def getChannelIndexList(self):
771 771
772 772 return list(range(self.nChannels))
773 773
774 774 def getNoise(self, type=1):
775 775
776 776
777 777 if type == 1:
778 778 noise = self.getNoisebyHildebrand()
779 779
780 780 if type == 2:
781 781 noise = self.getNoisebySort()
782 782
783 783 if type == 3:
784 784 noise = self.getNoisebyWindow()
785 785
786 786 return noise
787 787
788 788 @property
789 789 def timeInterval(self):
790 790
791 791 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
792 792
793 793 return timeInterval
794 794
795 795 @property
796 796 def ippSeconds(self):
797 797 '''
798 798 '''
799 799 return self.ipp_sec
800 800
801 801 noise = property(getNoise, "I'm the 'nHeights' property.")
802 802
803 803
804 804 class Correlation(JROData):
805 805
806 806 def __init__(self):
807 807 '''
808 808 Constructor
809 809 '''
810 810 self.radarControllerHeaderObj = RadarControllerHeader()
811 811 self.systemHeaderObj = SystemHeader()
812 812 self.type = "Correlation"
813 813 self.data = None
814 814 self.dtype = None
815 815 self.nProfiles = None
816 816 self.heightList = None
817 817 self.channelList = None
818 818 self.flagNoData = True
819 819 self.flagDiscontinuousBlock = False
820 820 self.utctime = None
821 821 self.timeZone = 0
822 822 self.dstFlag = None
823 823 self.errorCount = None
824 824 self.blocksize = None
825 825 self.flagDecodeData = False # asumo q la data no esta decodificada
826 826 self.flagDeflipData = False # asumo q la data no esta sin flip
827 827 self.pairsList = None
828 828 self.nPoints = None
829 829
830 830 def getPairsList(self):
831 831
832 832 return self.pairsList
833 833
834 834 def getNoise(self, mode=2):
835 835
836 836 indR = numpy.where(self.lagR == 0)[0][0]
837 837 indT = numpy.where(self.lagT == 0)[0][0]
838 838
839 839 jspectra0 = self.data_corr[:, :, indR, :]
840 840 jspectra = copy.copy(jspectra0)
841 841
842 842 num_chan = jspectra.shape[0]
843 843 num_hei = jspectra.shape[2]
844 844
845 845 freq_dc = jspectra.shape[1] / 2
846 846 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
847 847
848 848 if ind_vel[0] < 0:
849 849 ind_vel[list(range(0, 1))] = ind_vel[list(
850 850 range(0, 1))] + self.num_prof
851 851
852 852 if mode == 1:
853 853 jspectra[:, freq_dc, :] = (
854 854 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
855 855
856 856 if mode == 2:
857 857
858 858 vel = numpy.array([-2, -1, 1, 2])
859 859 xx = numpy.zeros([4, 4])
860 860
861 861 for fil in range(4):
862 862 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
863 863
864 864 xx_inv = numpy.linalg.inv(xx)
865 865 xx_aux = xx_inv[0, :]
866 866
867 867 for ich in range(num_chan):
868 868 yy = jspectra[ich, ind_vel, :]
869 869 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
870 870
871 871 junkid = jspectra[ich, freq_dc, :] <= 0
872 872 cjunkid = sum(junkid)
873 873
874 874 if cjunkid.any():
875 875 jspectra[ich, freq_dc, junkid.nonzero()] = (
876 876 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
877 877
878 878 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
879 879
880 880 return noise
881 881
882 882 @property
883 883 def timeInterval(self):
884 884
885 885 return self.ippSeconds * self.nCohInt * self.nProfiles
886 886
887 887 def splitFunctions(self):
888 888
889 889 pairsList = self.pairsList
890 890 ccf_pairs = []
891 891 acf_pairs = []
892 892 ccf_ind = []
893 893 acf_ind = []
894 894 for l in range(len(pairsList)):
895 895 chan0 = pairsList[l][0]
896 896 chan1 = pairsList[l][1]
897 897
898 898 # Obteniendo pares de Autocorrelacion
899 899 if chan0 == chan1:
900 900 acf_pairs.append(chan0)
901 901 acf_ind.append(l)
902 902 else:
903 903 ccf_pairs.append(pairsList[l])
904 904 ccf_ind.append(l)
905 905
906 906 data_acf = self.data_cf[acf_ind]
907 907 data_ccf = self.data_cf[ccf_ind]
908 908
909 909 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
910 910
911 911 @property
912 912 def normFactor(self):
913 913 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
914 914 acf_pairs = numpy.array(acf_pairs)
915 915 normFactor = numpy.zeros((self.nPairs, self.nHeights))
916 916
917 917 for p in range(self.nPairs):
918 918 pair = self.pairsList[p]
919 919
920 920 ch0 = pair[0]
921 921 ch1 = pair[1]
922 922
923 923 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
924 924 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
925 925 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
926 926
927 927 return normFactor
928 928
929 929
930 930 class Parameters(Spectra):
931 931
932 932 radarControllerHeaderTxt=None #header Controller like text
933 933 groupList = None # List of Pairs, Groups, etc
934 934 data_param = None # Parameters obtained
935 935 data_pre = None # Data Pre Parametrization
936 936 data_SNR = None # Signal to Noise Ratio
937 937 data_outlier = None
938 data_vdrift = None
938 939 abscissaList = None # Abscissa, can be velocities, lags or time
939 940 utctimeInit = None # Initial UTC time
940 941 paramInterval = None # Time interval to calculate Parameters in seconds
941 942 useLocalTime = True
942 943 # Fitting
943 944 data_error = None # Error of the estimation
944 945 constants = None
945 946 library = None
946 947 # Output signal
947 948 outputInterval = None # Time interval to calculate output signal in seconds
948 949 data_output = None # Out signal
949 950 nAvg = None
950 951 noise_estimation = None
951 952 GauSPC = None # Fit gaussian SPC
952 953 txPower = None
953 954 flagProfilesByRange = False
954 955 nProfilesByRange = None
955 956
956 957 def __init__(self):
957 958 '''
958 959 Constructor
959 960 '''
960 961 self.radarControllerHeaderObj = RadarControllerHeader()
961 962 self.systemHeaderObj = SystemHeader()
962 963 self.processingHeaderObj = ProcessingHeader()
963 964 self.type = "Parameters"
964 965 self.timeZone = 0
965 966
966 967 def getTimeRange1(self, interval):
967 968
968 969 datatime = []
969 970
970 971 if self.useLocalTime:
971 972 time1 = self.utctimeInit - self.timeZone * 60
972 973 else:
973 974 time1 = self.utctimeInit
974 975
975 976 datatime.append(time1)
976 977 datatime.append(time1 + interval)
977 978 datatime = numpy.array(datatime)
978 979
979 980 return datatime
980 981
981 982 @property
982 983 def timeInterval(self):
983 984
984 985 if hasattr(self, 'timeInterval1'):
985 986 return self.timeInterval1
986 987 else:
987 988 return self.paramInterval
988 989
989 990 def setValue(self, value):
990 991
991 992 print("This property should not be initialized")
992 993
993 994 return
994 995
995 996 def getNoise(self):
996 997
997 998 return self.spc_noise
998 999
999 1000 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
1000 1001
1001 1002
1002 1003 class PlotterData(object):
1003 1004 '''
1004 1005 Object to hold data to be plotted
1005 1006 '''
1006 1007
1007 1008 MAXNUMX = 200
1008 1009 MAXNUMY = 200
1009 1010
1010 1011 def __init__(self, code, exp_code, localtime=True):
1011 1012
1012 1013 self.key = code
1013 1014 self.exp_code = exp_code
1014 1015 self.ready = False
1015 1016 self.flagNoData = False
1016 1017 self.localtime = localtime
1017 1018 self.data = {}
1018 1019 self.meta = {}
1019 1020 self.__heights = []
1020 1021
1021 1022 def __str__(self):
1022 1023 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
1023 1024 return 'Data[{}][{}]'.format(';'.join(dum), len(self.times))
1024 1025
1025 1026 def __len__(self):
1026 1027 return len(self.data)
1027 1028
1028 1029 def __getitem__(self, key):
1029 1030 if isinstance(key, int):
1030 1031 return self.data[self.times[key]]
1031 1032 elif isinstance(key, str):
1032 1033 ret = numpy.array([self.data[x][key] for x in self.times])
1033 1034 if ret.ndim > 1:
1034 1035 ret = numpy.swapaxes(ret, 0, 1)
1035 1036 return ret
1036 1037
1037 1038 def __contains__(self, key):
1038 1039 return key in self.data[self.min_time]
1039 1040
1040 1041 def setup(self):
1041 1042 '''
1042 1043 Configure object
1043 1044 '''
1044 1045 self.type = ''
1045 1046 self.ready = False
1046 1047 del self.data
1047 1048 self.data = {}
1048 1049 self.__heights = []
1049 1050 self.__all_heights = set()
1050 1051
1051 1052 def shape(self, key):
1052 1053 '''
1053 1054 Get the shape of the one-element data for the given key
1054 1055 '''
1055 1056
1056 1057 if len(self.data[self.min_time][key]):
1057 1058 return self.data[self.min_time][key].shape
1058 1059 return (0,)
1059 1060
1060 1061 def update(self, data, tm, meta={}):
1061 1062 '''
1062 1063 Update data object with new dataOut
1063 1064 '''
1064 1065
1065 1066 self.data[tm] = data
1066 1067
1067 1068 for key, value in meta.items():
1068 1069 setattr(self, key, value)
1069 1070
1070 1071 def normalize_heights(self):
1071 1072 '''
1072 1073 Ensure same-dimension of the data for different heighList
1073 1074 '''
1074 1075
1075 1076 H = numpy.array(list(self.__all_heights))
1076 1077 H.sort()
1077 1078 for key in self.data:
1078 1079 shape = self.shape(key)[:-1] + H.shape
1079 1080 for tm, obj in list(self.data[key].items()):
1080 1081 h = self.__heights[self.times.tolist().index(tm)]
1081 1082 if H.size == h.size:
1082 1083 continue
1083 1084 index = numpy.where(numpy.in1d(H, h))[0]
1084 1085 dummy = numpy.zeros(shape) + numpy.nan
1085 1086 if len(shape) == 2:
1086 1087 dummy[:, index] = obj
1087 1088 else:
1088 1089 dummy[index] = obj
1089 1090 self.data[key][tm] = dummy
1090 1091
1091 1092 self.__heights = [H for tm in self.times]
1092 1093
1093 1094 def jsonify(self, tm, plot_name, plot_type, decimate=False):
1094 1095 '''
1095 1096 Convert data to json
1096 1097 '''
1097 1098
1098 1099 meta = {}
1099 1100 meta['xrange'] = []
1100 1101 dy = int(len(self.yrange)/self.MAXNUMY) + 1
1101 1102 tmp = self.data[tm][self.key]
1102 1103 shape = tmp.shape
1103 1104 if len(shape) == 2:
1104 1105 data = self.roundFloats(self.data[tm][self.key][::, ::dy].tolist())
1105 1106 elif len(shape) == 3:
1106 1107 dx = int(self.data[tm][self.key].shape[1]/self.MAXNUMX) + 1
1107 1108 data = self.roundFloats(
1108 1109 self.data[tm][self.key][::, ::dx, ::dy].tolist())
1109 1110 meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1110 1111 else:
1111 1112 data = self.roundFloats(self.data[tm][self.key].tolist())
1112 1113
1113 1114 ret = {
1114 1115 'plot': plot_name,
1115 1116 'code': self.exp_code,
1116 1117 'time': float(tm),
1117 1118 'data': data,
1118 1119 }
1119 1120 meta['type'] = plot_type
1120 1121 meta['interval'] = float(self.interval)
1121 1122 meta['localtime'] = self.localtime
1122 1123 meta['yrange'] = self.roundFloats(self.yrange[::dy].tolist())
1123 1124 meta.update(self.meta)
1124 1125 ret['metadata'] = meta
1125 1126 return json.dumps(ret)
1126 1127
1127 1128 @property
1128 1129 def times(self):
1129 1130 '''
1130 1131 Return the list of times of the current data
1131 1132 '''
1132 1133
1133 1134 ret = [t for t in self.data]
1134 1135 ret.sort()
1135 1136 return numpy.array(ret)
1136 1137
1137 1138 @property
1138 1139 def min_time(self):
1139 1140 '''
1140 1141 Return the minimun time value
1141 1142 '''
1142 1143
1143 1144 return self.times[0]
1144 1145
1145 1146 @property
1146 1147 def max_time(self):
1147 1148 '''
1148 1149 Return the maximun time value
1149 1150 '''
1150 1151
1151 1152 return self.times[-1]
1152 1153
1153 1154 # @property
1154 1155 # def heights(self):
1155 1156 # '''
1156 1157 # Return the list of heights of the current data
1157 1158 # '''
1158 1159
1159 1160 # return numpy.array(self.__heights[-1])
1160 1161
1161 1162 @staticmethod
1162 1163 def roundFloats(obj):
1163 1164 if isinstance(obj, list):
1164 1165 return list(map(PlotterData.roundFloats, obj))
1165 1166 elif isinstance(obj, float):
1166 1167 return round(obj, 2)
Show file before | Show file after | Hide comments
@@ -1,419 +1,419
1 1 import os
2 2 import datetime
3 3 import numpy
4 4
5 5 from schainpy.model.graphics.jroplot_base import Plot, plt
6 6 from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot
7 7 from schainpy.utils import log
8 8
9 9 EARTH_RADIUS = 6.3710e3
10 10
11 11
12 12 def ll2xy(lat1, lon1, lat2, lon2):
13 13
14 14 p = 0.017453292519943295
15 15 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
16 16 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
17 17 r = 12742 * numpy.arcsin(numpy.sqrt(a))
18 18 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
19 19 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
20 20 theta = -theta + numpy.pi/2
21 21 return r*numpy.cos(theta), r*numpy.sin(theta)
22 22
23 23
24 24 def km2deg(km):
25 25 '''
26 26 Convert distance in km to degrees
27 27 '''
28 28
29 29 return numpy.rad2deg(km/EARTH_RADIUS)
30 30
31 31
32 32
33 33 class SpectralMomentsPlot(SpectraPlot):
34 34 '''
35 35 Plot for Spectral Moments
36 36 '''
37 37 CODE = 'spc_moments'
38 38 colormap = 'jet'
39 39 plot_type = 'pcolor'
40 40
41 41
42 42 class SnrPlot(RTIPlot):
43 43 '''
44 44 Plot for SNR Data
45 45 '''
46 46
47 47 CODE = 'snr'
48 48 colormap = 'jet'
49 49
50 50 def update(self, dataOut):
51 51 if len(self.channelList) == 0:
52 52 self.update_list(dataOut)
53 53
54 54 meta = {}
55 55 data = {
56 56 'snr': 10 * numpy.log10(dataOut.data_snr)
57 57 }
58 58 return data, meta
59 59
60 60 class DopplerPlot(RTIPlot):
61 61 '''
62 62 Plot for DOPPLER Data (1st moment)
63 63 '''
64 64
65 65 CODE = 'dop'
66 66 colormap = 'jet'
67 67
68 68 def update(self, dataOut):
69 69 self.update_list(dataOut)
70 70 data = {
71 71 'dop': 10*numpy.log10(dataOut.data_dop)
72 72 }
73 73
74 74 return data, {}
75 75
76 76 class PowerPlot(RTIPlot):
77 77 '''
78 78 Plot for Power Data (0 moment)
79 79 '''
80 80
81 81 CODE = 'pow'
82 82 colormap = 'jet'
83 83
84 84 def update(self, dataOut):
85 85 self.update_list(dataOut)
86 86 data = {
87 87 'pow': 10*numpy.log10(dataOut.data_pow)
88 88 }
89 89 try:
90 90 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
91 91 except:
92 92 pass
93 93 return data, {}
94 94
95 95 class SpectralWidthPlot(RTIPlot):
96 96 '''
97 97 Plot for Spectral Width Data (2nd moment)
98 98 '''
99 99
100 100 CODE = 'width'
101 101 colormap = 'jet'
102 102
103 103 def update(self, dataOut):
104 104 self.update_list(dataOut)
105 105 data = {
106 106 'width': dataOut.data_width
107 107 }
108 108 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
109 109 return data, {}
110 110
111 111 class SkyMapPlot(Plot):
112 112 '''
113 113 Plot for meteors detection data
114 114 '''
115 115
116 116 CODE = 'param'
117 117
118 118 def setup(self):
119 119
120 120 self.ncols = 1
121 121 self.nrows = 1
122 122 self.width = 7.2
123 123 self.height = 7.2
124 124 self.nplots = 1
125 125 self.xlabel = 'Zonal Zenith Angle (deg)'
126 126 self.ylabel = 'Meridional Zenith Angle (deg)'
127 127 self.polar = True
128 128 self.ymin = -180
129 129 self.ymax = 180
130 130 self.colorbar = False
131 131
132 132 def plot(self):
133 133
134 134 arrayParameters = numpy.concatenate(self.data['param'])
135 135 error = arrayParameters[:, -1]
136 136 indValid = numpy.where(error == 0)[0]
137 137 finalMeteor = arrayParameters[indValid, :]
138 138 finalAzimuth = finalMeteor[:, 3]
139 139 finalZenith = finalMeteor[:, 4]
140 140
141 141 x = finalAzimuth * numpy.pi / 180
142 142 y = finalZenith
143 143
144 144 ax = self.axes[0]
145 145
146 146 if ax.firsttime:
147 147 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
148 148 else:
149 149 ax.plot.set_data(x, y)
150 150
151 151 dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
152 152 dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
153 153 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
154 154 dt2,
155 155 len(x))
156 156 self.titles[0] = title
157 157
158 158
159 159 class GenericRTIPlot(Plot):
160 160 '''
161 161 Plot for data_xxxx object
162 162 '''
163 163
164 164 CODE = 'param'
165 165 colormap = 'viridis'
166 166 plot_type = 'pcolorbuffer'
167 167
168 168 def setup(self):
169 169 self.xaxis = 'time'
170 170 self.ncols = 1
171 171 self.nrows = self.data.shape('param')[0]
172 172 self.nplots = self.nrows
173 173 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
174 174
175 175 if not self.xlabel:
176 176 self.xlabel = 'Time'
177 177
178 178 self.ylabel = 'Height [km]'
179 179 if not self.titles:
180 180 self.titles = ['Param {}'.format(x) for x in range(self.nrows)]
181 181
182 182 def update(self, dataOut):
183 183
184 184 data = {
185 185 'param' : numpy.concatenate([getattr(dataOut, attr) for attr in self.attr_data], axis=0)
186 186 }
187 187
188 188 meta = {}
189 189
190 190 return data, meta
191 191
192 192 def plot(self):
193 193 # self.data.normalize_heights()
194 194 self.x = self.data.times
195 195 self.y = self.data.yrange
196 196 self.z = self.data['param']
197 197
198 198 self.z = numpy.ma.masked_invalid(self.z)
199 199
200 200 if self.decimation is None:
201 201 x, y, z = self.fill_gaps(self.x, self.y, self.z)
202 202 else:
203 203 x, y, z = self.fill_gaps(*self.decimate())
204 204
205 205 for n, ax in enumerate(self.axes):
206 206
207 207 self.zmax = self.zmax if self.zmax is not None else numpy.max(
208 208 self.z[n])
209 209 self.zmin = self.zmin if self.zmin is not None else numpy.min(
210 210 self.z[n])
211 211
212 212 if ax.firsttime:
213 213 if self.zlimits is not None:
214 214 self.zmin, self.zmax = self.zlimits[n]
215 215
216 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
216 ax.plt = ax.pcolormesh(x, y, z[n].T ,
217 217 vmin=self.zmin,
218 218 vmax=self.zmax,
219 219 cmap=self.cmaps[n]
220 220 )
221 221 else:
222 222 if self.zlimits is not None:
223 223 self.zmin, self.zmax = self.zlimits[n]
224 224 ax.collections.remove(ax.collections[0])
225 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
225 ax.plt = ax.pcolormesh(x, y, z[n].T ,
226 226 vmin=self.zmin,
227 227 vmax=self.zmax,
228 228 cmap=self.cmaps[n]
229 229 )
230 230
231 231
232 232 class PolarMapPlot(Plot):
233 233 '''
234 234 Plot for weather radar
235 235 '''
236 236
237 237 CODE = 'param'
238 238 colormap = 'seismic'
239 239
240 240 def setup(self):
241 241 self.ncols = 1
242 242 self.nrows = 1
243 243 self.width = 9
244 244 self.height = 8
245 245 self.mode = self.data.meta['mode']
246 246 if self.channels is not None:
247 247 self.nplots = len(self.channels)
248 248 self.nrows = len(self.channels)
249 249 else:
250 250 self.nplots = self.data.shape(self.CODE)[0]
251 251 self.nrows = self.nplots
252 252 self.channels = list(range(self.nplots))
253 253 if self.mode == 'E':
254 254 self.xlabel = 'Longitude'
255 255 self.ylabel = 'Latitude'
256 256 else:
257 257 self.xlabel = 'Range (km)'
258 258 self.ylabel = 'Height (km)'
259 259 self.bgcolor = 'white'
260 260 self.cb_labels = self.data.meta['units']
261 261 self.lat = self.data.meta['latitude']
262 262 self.lon = self.data.meta['longitude']
263 263 self.xmin, self.xmax = float(
264 264 km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
265 265 self.ymin, self.ymax = float(
266 266 km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
267 267 # self.polar = True
268 268
269 269 def plot(self):
270 270
271 271 for n, ax in enumerate(self.axes):
272 272 data = self.data['param'][self.channels[n]]
273 273
274 274 zeniths = numpy.linspace(
275 275 0, self.data.meta['max_range'], data.shape[1])
276 276 if self.mode == 'E':
277 277 azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
278 278 r, theta = numpy.meshgrid(zeniths, azimuths)
279 279 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
280 280 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
281 281 x = km2deg(x) + self.lon
282 282 y = km2deg(y) + self.lat
283 283 else:
284 284 azimuths = numpy.radians(self.data.yrange)
285 285 r, theta = numpy.meshgrid(zeniths, azimuths)
286 286 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
287 287 self.y = zeniths
288 288
289 289 if ax.firsttime:
290 290 if self.zlimits is not None:
291 291 self.zmin, self.zmax = self.zlimits[n]
292 292 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
293 293 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
294 294 vmin=self.zmin,
295 295 vmax=self.zmax,
296 296 cmap=self.cmaps[n])
297 297 else:
298 298 if self.zlimits is not None:
299 299 self.zmin, self.zmax = self.zlimits[n]
300 300 ax.collections.remove(ax.collections[0])
301 301 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
302 302 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
303 303 vmin=self.zmin,
304 304 vmax=self.zmax,
305 305 cmap=self.cmaps[n])
306 306
307 307 if self.mode == 'A':
308 308 continue
309 309
310 310 # plot district names
311 311 f = open('/data/workspace/schain_scripts/distrito.csv')
312 312 for line in f:
313 313 label, lon, lat = [s.strip() for s in line.split(',') if s]
314 314 lat = float(lat)
315 315 lon = float(lon)
316 316 # ax.plot(lon, lat, '.b', ms=2)
317 317 ax.text(lon, lat, label.decode('utf8'), ha='center',
318 318 va='bottom', size='8', color='black')
319 319
320 320 # plot limites
321 321 limites = []
322 322 tmp = []
323 323 for line in open('/data/workspace/schain_scripts/lima.csv'):
324 324 if '#' in line:
325 325 if tmp:
326 326 limites.append(tmp)
327 327 tmp = []
328 328 continue
329 329 values = line.strip().split(',')
330 330 tmp.append((float(values[0]), float(values[1])))
331 331 for points in limites:
332 332 ax.add_patch(
333 333 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
334 334
335 335 # plot Cuencas
336 336 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
337 337 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
338 338 values = [line.strip().split(',') for line in f]
339 339 points = [(float(s[0]), float(s[1])) for s in values]
340 340 ax.add_patch(Polygon(points, ec='b', fc='none'))
341 341
342 342 # plot grid
343 343 for r in (15, 30, 45, 60):
344 344 ax.add_artist(plt.Circle((self.lon, self.lat),
345 345 km2deg(r), color='0.6', fill=False, lw=0.2))
346 346 ax.text(
347 347 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
348 348 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
349 349 '{}km'.format(r),
350 350 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
351 351
352 352 if self.mode == 'E':
353 353 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
354 354 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
355 355 else:
356 356 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
357 357 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
358 358
359 359 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
360 360 self.titles = ['{} {}'.format(
361 361 self.data.parameters[x], title) for x in self.channels]
362 362
363 363
364 364
365 365
366 366
367 367
368 368
369 369
370 370
371 371
372 372 class TxPowerPlot(Plot):
373 373 '''
374 374 Plot for Power of transmission
375 375 '''
376 376
377 377 CODE = 'tx_power'
378 378 plot_type = 'scatterbuffer'
379 379
380 380 def setup(self):
381 381 self.xaxis = 'time'
382 382 self.ncols = 1
383 383 self.nrows = 1
384 384 self.nplots = 1
385 385 self.ylabel = 'Power [kW]'
386 386 self.xlabel = 'Time'
387 387 self.titles = ['TX power']
388 388 self.colorbar = False
389 389 self.plots_adjust.update({'right': 0.85 })
390 390 #if not self.titles:
391 391 self.titles = ['TX Power Plot']
392 392
393 393 def update(self, dataOut):
394 394
395 395 data = {}
396 396 meta = {}
397 397
398 398 data['tx_power'] = dataOut.txPower/1000
399 399 meta['yrange'] = numpy.array([])
400 400 #print(dataOut.txPower/1000)
401 401 return data, meta
402 402
403 403 def plot(self):
404 404
405 405 x = self.data.times
406 406 xmin = self.data.min_time
407 407 xmax = xmin + self.xrange * 60 * 60
408 408 Y = self.data['tx_power']
409 409
410 410 if self.axes[0].firsttime:
411 411 if self.ymin is None: self.ymin = 0
412 412 if self.ymax is None: self.ymax = numpy.nanmax(Y) + 5
413 413 if self.ymax == 5:
414 414 self.ymax = 250
415 415 self.ymin = 100
416 416 self.axes[0].plot(x, Y, lw=1, label='Power')
417 417 plt.legend(bbox_to_anchor=(1.18, 1.0))
418 418 else:
419 419 self.axes[0].lines[0].set_data(x, Y) No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1725 +1,1722
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Classes to plot Spectra data
6 6
7 7 """
8 8
9 9 import os
10 10 import numpy
11 11
12 12 from schainpy.model.graphics.jroplot_base import Plot, plt, log
13 13 from itertools import combinations
14 14 from matplotlib.ticker import LinearLocator
15 15
16 16 from schainpy.model.utils.BField import BField
17 17 from scipy.interpolate import splrep
18 18 from scipy.interpolate import splev
19 19
20 20 from matplotlib import __version__ as plt_version
21 21
22 22 if plt_version >='3.3.4':
23 23 EXTRA_POINTS = 0
24 24 else:
25 25 EXTRA_POINTS = 1
26 26
27 27 class SpectraPlot(Plot):
28 28 '''
29 29 Plot for Spectra data
30 30 '''
31 31
32 32 CODE = 'spc'
33 33 colormap = 'jet'
34 34 plot_type = 'pcolor'
35 35 buffering = False
36 36 channelList = []
37 37 elevationList = []
38 38 azimuthList = []
39 39
40 40 def setup(self):
41 41
42 42 self.nplots = len(self.data.channels)
43 43 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
44 44 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
45 45 self.height = 3.4 * self.nrows
46 46
47 47 self.cb_label = 'dB'
48 48 if self.showprofile:
49 49 self.width = 5.2 * self.ncols
50 50 else:
51 51 self.width = 4.2* self.ncols
52 52 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.12})
53 53 self.ylabel = 'Range [km]'
54 54
55 55
56 56 def update_list(self,dataOut):
57 57 if len(self.channelList) == 0:
58 58 self.channelList = dataOut.channelList
59 59 if len(self.elevationList) == 0:
60 60 self.elevationList = dataOut.elevationList
61 61 if len(self.azimuthList) == 0:
62 62 self.azimuthList = dataOut.azimuthList
63 63
64 64 def update(self, dataOut):
65 65
66 66 self.update_list(dataOut)
67 67 data = {}
68 68 meta = {}
69 69
70 70 #data['rti'] = dataOut.getPower()
71 71 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
72 72 noise = 10*numpy.log10(dataOut.getNoise()/norm)
73 73
74 <<<<<<< HEAD
75 =======
76 74
77 75
78 >>>>>>> 37cccf17c7b80521b59b978cb30e4ab2e6f37fce
79 76 z = numpy.zeros((dataOut.nChannels, dataOut.nFFTPoints, dataOut.nHeights))
80 77 for ch in range(dataOut.nChannels):
81 78 if hasattr(dataOut.normFactor,'ndim'):
82 79 if dataOut.normFactor.ndim > 1:
83 80 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
84 81
85 82 else:
86 83 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
87 84 else:
88 85 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
89 86
90 87
91 88 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
92 89 spc = 10*numpy.log10(z)
93 90
94 91 data['spc'] = spc
95 92 #print(spc[0].shape)
96 93 data['rti'] = spc.mean(axis=1)
97 94 data['noise'] = noise
98 95 meta['xrange'] = (dataOut.getFreqRange(EXTRA_POINTS)/1000., dataOut.getAcfRange(EXTRA_POINTS), dataOut.getVelRange(EXTRA_POINTS))
99 96 if self.CODE == 'spc_moments':
100 97 data['moments'] = dataOut.moments
101 98
102 99 return data, meta
103 100
104 101 def plot(self):
105 102 if self.xaxis == "frequency":
106 103 x = self.data.xrange[0]
107 104 self.xlabel = "Frequency (kHz)"
108 105 elif self.xaxis == "time":
109 106 x = self.data.xrange[1]
110 107 self.xlabel = "Time (ms)"
111 108 else:
112 109 x = self.data.xrange[2]
113 110 self.xlabel = "Velocity (m/s)"
114 111
115 112 if self.CODE == 'spc_moments':
116 113 x = self.data.xrange[2]
117 114 self.xlabel = "Velocity (m/s)"
118 115
119 116 self.titles = []
120 117 y = self.data.yrange
121 118 self.y = y
122 119
123 120 data = self.data[-1]
124 121 z = data['spc']
125 122 #print(z.shape, x.shape, y.shape)
126 123 for n, ax in enumerate(self.axes):
127 124 noise = self.data['noise'][n][0]
128 125 #print(noise)
129 126 if self.CODE == 'spc_moments':
130 127 mean = data['moments'][n, 1]
131 128 if ax.firsttime:
132 129 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
133 130 self.xmin = self.xmin if self.xmin else -self.xmax
134 131 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
135 132 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
136 133 ax.plt = ax.pcolormesh(x, y, z[n].T,
137 134 vmin=self.zmin,
138 135 vmax=self.zmax,
139 136 cmap=plt.get_cmap(self.colormap)
140 137 )
141 138
142 139 if self.showprofile:
143 140 ax.plt_profile = self.pf_axes[n].plot(
144 141 data['rti'][n], y)[0]
145 142 # ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
146 143 # color="k", linestyle="dashed", lw=1)[0]
147 144 if self.CODE == 'spc_moments':
148 145 ax.plt_mean = ax.plot(mean, y, color='k')[0]
149 146 else:
150 147 ax.plt.set_array(z[n].T.ravel())
151 148 if self.showprofile:
152 149 ax.plt_profile.set_data(data['rti'][n], y)
153 150 #ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
154 151 if self.CODE == 'spc_moments':
155 152 ax.plt_mean.set_data(mean, y)
156 153 if len(self.azimuthList) > 0 and len(self.elevationList) > 0:
157 154 self.titles.append('CH {}: {:2.1f}elv {:2.1f}az {:3.2f}dB'.format(self.channelList[n], noise, self.elevationList[n], self.azimuthList[n]))
158 155 else:
159 156 self.titles.append('CH {}: {:3.2f}dB'.format(self.channelList[n], noise))
160 157
161 158
162 159 class CrossSpectraPlot(Plot):
163 160
164 161 CODE = 'cspc'
165 162 colormap = 'jet'
166 163 plot_type = 'pcolor'
167 164 zmin_coh = None
168 165 zmax_coh = None
169 166 zmin_phase = None
170 167 zmax_phase = None
171 168 realChannels = None
172 169 crossPairs = None
173 170
174 171 def setup(self):
175 172
176 173 self.ncols = 4
177 174 self.nplots = len(self.data.pairs) * 2
178 175 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
179 176 self.width = 3.1 * self.ncols
180 177 self.height = 2.6 * self.nrows
181 178 self.ylabel = 'Range [km]'
182 179 self.showprofile = False
183 180 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
184 181
185 182 def update(self, dataOut):
186 183
187 184 data = {}
188 185 meta = {}
189 186
190 187 spc = dataOut.data_spc
191 188 cspc = dataOut.data_cspc
192 189 meta['xrange'] = (dataOut.getFreqRange(EXTRA_POINTS)/1000., dataOut.getAcfRange(EXTRA_POINTS), dataOut.getVelRange(EXTRA_POINTS))
193 190 rawPairs = list(combinations(list(range(dataOut.nChannels)), 2))
194 191 meta['pairs'] = rawPairs
195 192
196 193 if self.crossPairs == None:
197 194 self.crossPairs = dataOut.pairsList
198 195
199 196 tmp = []
200 197
201 198 for n, pair in enumerate(meta['pairs']):
202 199
203 200 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
204 201 coh = numpy.abs(out)
205 202 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
206 203 tmp.append(coh)
207 204 tmp.append(phase)
208 205
209 206 data['cspc'] = numpy.array(tmp)
210 207
211 208 return data, meta
212 209
213 210 def plot(self):
214 211
215 212 if self.xaxis == "frequency":
216 213 x = self.data.xrange[0]
217 214 self.xlabel = "Frequency (kHz)"
218 215 elif self.xaxis == "time":
219 216 x = self.data.xrange[1]
220 217 self.xlabel = "Time (ms)"
221 218 else:
222 219 x = self.data.xrange[2]
223 220 self.xlabel = "Velocity (m/s)"
224 221
225 222 self.titles = []
226 223
227 224 y = self.data.yrange
228 225 self.y = y
229 226
230 227 data = self.data[-1]
231 228 cspc = data['cspc']
232 229
233 230 for n in range(len(self.data.pairs)):
234 231
235 232 pair = self.crossPairs[n]
236 233
237 234 coh = cspc[n*2]
238 235 phase = cspc[n*2+1]
239 236 ax = self.axes[2 * n]
240 237
241 238 if ax.firsttime:
242 239 ax.plt = ax.pcolormesh(x, y, coh.T,
243 240 vmin=self.zmin_coh,
244 241 vmax=self.zmax_coh,
245 242 cmap=plt.get_cmap(self.colormap_coh)
246 243 )
247 244 else:
248 245 ax.plt.set_array(coh.T.ravel())
249 246 self.titles.append(
250 247 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
251 248
252 249 ax = self.axes[2 * n + 1]
253 250 if ax.firsttime:
254 251 ax.plt = ax.pcolormesh(x, y, phase.T,
255 252 vmin=-180,
256 253 vmax=180,
257 254 cmap=plt.get_cmap(self.colormap_phase)
258 255 )
259 256 else:
260 257 ax.plt.set_array(phase.T.ravel())
261 258
262 259 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
263 260
264 261
265 262 class RTIPlot(Plot):
266 263 '''
267 264 Plot for RTI data
268 265 '''
269 266
270 267 CODE = 'rti'
271 268 colormap = 'jet'
272 269 plot_type = 'pcolorbuffer'
273 270 titles = None
274 271 channelList = []
275 272 elevationList = []
276 273 azimuthList = []
277 274
278 275 def setup(self):
279 276 self.xaxis = 'time'
280 277 self.ncols = 1
281 278 #print("dataChannels ",self.data.channels)
282 279 self.nrows = len(self.data.channels)
283 280 self.nplots = len(self.data.channels)
284 281 self.ylabel = 'Range [km]'
285 282 #self.xlabel = 'Time'
286 283 self.cb_label = 'dB'
287 284 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
288 285 self.titles = ['{} Channel {}'.format(
289 286 self.CODE.upper(), x) for x in range(self.nplots)]
290 287
291 288 def update_list(self,dataOut):
292 289
293 290 if len(self.channelList) == 0:
294 291 self.channelList = dataOut.channelList
295 292 if len(self.elevationList) == 0:
296 293 self.elevationList = dataOut.elevationList
297 294 if len(self.azimuthList) == 0:
298 295 self.azimuthList = dataOut.azimuthList
299 296
300 297
301 298 def update(self, dataOut):
302 299 if len(self.channelList) == 0:
303 300 self.update_list(dataOut)
304 301 data = {}
305 302 meta = {}
306 303
307 304 data['rti'] = dataOut.getPower()
308 305
309 306 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
310 307 noise = 10*numpy.log10(dataOut.getNoise()/norm)
311 308 data['noise'] = noise
312 309
313 310 return data, meta
314 311
315 312 def plot(self):
316 313
317 314 self.x = self.data.times
318 315 self.y = self.data.yrange
319 316 #print(" x, y: ",self.x, self.y)
320 317 self.z = self.data[self.CODE]
321 318 self.z = numpy.array(self.z, dtype=float)
322 319 self.z = numpy.ma.masked_invalid(self.z)
323 320
324 321 try:
325 322 if self.channelList != None:
326 323 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
327 324 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
328 325 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
329 326 else:
330 327 self.titles = ['{} Channel {}'.format(
331 328 self.CODE.upper(), x) for x in self.channelList]
332 329 except:
333 330 if self.channelList.any() != None:
334 331 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
335 332 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
336 333 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
337 334 else:
338 335 self.titles = ['{} Channel {}'.format(
339 336 self.CODE.upper(), x) for x in self.channelList]
340 337
341 338 if self.decimation is None:
342 339 x, y, z = self.fill_gaps(self.x, self.y, self.z)
343 340 else:
344 341 x, y, z = self.fill_gaps(*self.decimate())
345 342
346 343 #dummy_var = self.axes #Extrañamente esto actualiza el valor axes
347 344 for n, ax in enumerate(self.axes):
348 345 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
349 346 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
350 347 data = self.data[-1]
351 348
352 349 if ax.firsttime:
353 350 if (n+1) == len(self.channelList):
354 351 ax.set_xlabel('Time')
355 352 ax.plt = ax.pcolormesh(x, y, z[n].T,
356 353 vmin=self.zmin,
357 354 vmax=self.zmax,
358 355 cmap=plt.get_cmap(self.colormap)
359 356 )
360 357 if self.showprofile:
361 358 ax.plot_profile = self.pf_axes[n].plot(data[self.CODE][n], self.y)[0]
362 359 if "noise" in self.data:
363 360
364 361 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
365 362 color="k", linestyle="dashed", lw=1)[0]
366 363 else:
367 364 ax.collections.remove(ax.collections[0])
368 365 ax.plt = ax.pcolormesh(x, y, z[n].T,
369 366 vmin=self.zmin,
370 367 vmax=self.zmax,
371 368 cmap=plt.get_cmap(self.colormap)
372 369 )
373 370 if self.showprofile:
374 371 ax.plot_profile.set_data(data[self.CODE][n], self.y)
375 372 if "noise" in self.data:
376 373 ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
377 374
378 375
379 376 class CoherencePlot(RTIPlot):
380 377 '''
381 378 Plot for Coherence data
382 379 '''
383 380
384 381 CODE = 'coh'
385 382 titles = None
386 383
387 384 def setup(self):
388 385 self.xaxis = 'time'
389 386 self.ncols = 1
390 387 self.nrows = len(self.data.pairs)
391 388 self.nplots = len(self.data.pairs)
392 389 self.ylabel = 'Range [km]'
393 390 #self.xlabel = 'Time'
394 391 self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1,'right':0.95})
395 392 if self.CODE == 'coh':
396 393 self.cb_label = ''
397 394 self.titles = [
398 395 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
399 396 else:
400 397 self.cb_label = 'Degrees'
401 398 self.titles = [
402 399 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
403 400
404 401
405 402 def update(self, dataOut):
406 403
407 404 data = {}
408 405 meta = {}
409 406 data['coh'] = dataOut.getCoherence()
410 407 meta['pairs'] = dataOut.pairsList
411 408
412 409
413 410 return data, meta
414 411
415 412 def plot(self):
416 413
417 414 self.x = self.data.times
418 415 self.y = self.data.yrange
419 416 self.z = self.data[self.CODE]
420 417
421 418 self.z = numpy.ma.masked_invalid(self.z)
422 419
423 420 if self.decimation is None:
424 421 x, y, z = self.fill_gaps(self.x, self.y, self.z)
425 422 else:
426 423 x, y, z = self.fill_gaps(*self.decimate())
427 424
428 425 for n, ax in enumerate(self.axes):
429 426 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
430 427 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
431 428 if ax.firsttime:
432 429 if (n+1) == len(self.channelList):
433 430 ax.set_xlabel('Time')
434 431 ax.plt = ax.pcolormesh(x, y, z[n].T,
435 432 vmin=self.zmin,
436 433 vmax=self.zmax,
437 434 cmap=plt.get_cmap(self.colormap)
438 435 )
439 436 if self.showprofile:
440 437 ax.plot_profile = self.pf_axes[n].plot(
441 438 self.data[self.CODE][n][-1], self.y)[0]
442 439 # ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
443 440 # color="k", linestyle="dashed", lw=1)[0]
444 441 else:
445 442 ax.collections.remove(ax.collections[0])
446 443 ax.plt = ax.pcolormesh(x, y, z[n].T,
447 444 vmin=self.zmin,
448 445 vmax=self.zmax,
449 446 cmap=plt.get_cmap(self.colormap)
450 447 )
451 448 if self.showprofile:
452 449 ax.plot_profile.set_data(self.data[self.CODE][n][-1], self.y)
453 450 # ax.plot_noise.set_data(numpy.repeat(
454 451 # self.data['noise'][n][-1], len(self.y)), self.y)
455 452
456 453
457 454
458 455 class PhasePlot(CoherencePlot):
459 456 '''
460 457 Plot for Phase map data
461 458 '''
462 459
463 460 CODE = 'phase'
464 461 colormap = 'seismic'
465 462
466 463 def update(self, dataOut):
467 464
468 465 data = {}
469 466 meta = {}
470 467 data['phase'] = dataOut.getCoherence(phase=True)
471 468 meta['pairs'] = dataOut.pairsList
472 469
473 470 return data, meta
474 471
475 472 class NoisePlot(Plot):
476 473 '''
477 474 Plot for noise
478 475 '''
479 476
480 477 CODE = 'noise'
481 478 plot_type = 'scatterbuffer'
482 479
483 480 def setup(self):
484 481 self.xaxis = 'time'
485 482 self.ncols = 1
486 483 self.nrows = 1
487 484 self.nplots = 1
488 485 self.ylabel = 'Intensity [dB]'
489 486 self.xlabel = 'Time'
490 487 self.titles = ['Noise']
491 488 self.colorbar = False
492 489 self.plots_adjust.update({'right': 0.85 })
493 490 #if not self.titles:
494 491 self.titles = ['Noise Plot']
495 492
496 493 def update(self, dataOut):
497 494
498 495 data = {}
499 496 meta = {}
500 497 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
501 498 noise = 10*numpy.log10(dataOut.getNoise())
502 499 noise = noise.reshape(dataOut.nChannels, 1)
503 500 data['noise'] = noise
504 501 meta['yrange'] = numpy.array([])
505 502
506 503 return data, meta
507 504
508 505 def plot(self):
509 506
510 507 x = self.data.times
511 508 xmin = self.data.min_time
512 509 xmax = xmin + self.xrange * 60 * 60
513 510 Y = self.data['noise']
514 511
515 512 if self.axes[0].firsttime:
516 513 if self.ymin is None: self.ymin = numpy.nanmin(Y) - 5
517 514 if self.ymax is None: self.ymax = numpy.nanmax(Y) + 5
518 515 for ch in self.data.channels:
519 516 y = Y[ch]
520 517 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
521 518 plt.legend(bbox_to_anchor=(1.18, 1.0))
522 519 else:
523 520 for ch in self.data.channels:
524 521 y = Y[ch]
525 522 self.axes[0].lines[ch].set_data(x, y)
526 523
527 524
528 525 class PowerProfilePlot(Plot):
529 526
530 527 CODE = 'pow_profile'
531 528 plot_type = 'scatter'
532 529
533 530 def setup(self):
534 531
535 532 self.ncols = 1
536 533 self.nrows = 1
537 534 self.nplots = 1
538 535 self.height = 4
539 536 self.width = 3
540 537 self.ylabel = 'Range [km]'
541 538 self.xlabel = 'Intensity [dB]'
542 539 self.titles = ['Power Profile']
543 540 self.colorbar = False
544 541
545 542 def update(self, dataOut):
546 543
547 544 data = {}
548 545 meta = {}
549 546 data[self.CODE] = dataOut.getPower()
550 547
551 548 return data, meta
552 549
553 550 def plot(self):
554 551
555 552 y = self.data.yrange
556 553 self.y = y
557 554
558 555 x = self.data[-1][self.CODE]
559 556
560 557 if self.xmin is None: self.xmin = numpy.nanmin(x)*0.9
561 558 if self.xmax is None: self.xmax = numpy.nanmax(x)*1.1
562 559
563 560 if self.axes[0].firsttime:
564 561 for ch in self.data.channels:
565 562 self.axes[0].plot(x[ch], y, lw=1, label='Ch{}'.format(ch))
566 563 plt.legend()
567 564 else:
568 565 for ch in self.data.channels:
569 566 self.axes[0].lines[ch].set_data(x[ch], y)
570 567
571 568
572 569 class SpectraCutPlot(Plot):
573 570
574 571 CODE = 'spc_cut'
575 572 plot_type = 'scatter'
576 573 buffering = False
577 574 heights = []
578 575 channelList = []
579 576 maintitle = "Spectra Cuts"
580 577 flag_setIndex = False
581 578
582 579 def setup(self):
583 580
584 581 self.nplots = len(self.data.channels)
585 582 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
586 583 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
587 584 self.width = 4.5 * self.ncols + 2.5
588 585 self.height = 4.8 * self.nrows
589 586 self.ylabel = 'Power [dB]'
590 587 self.colorbar = False
591 588 self.plots_adjust.update({'left':0.1, 'hspace':0.3, 'right': 0.9, 'bottom':0.08})
592 589
593 590 if len(self.selectedHeightsList) > 0:
594 591 self.maintitle = "Spectra Cut"# for %d km " %(int(self.selectedHeight))
595 592
596 593
597 594
598 595 def update(self, dataOut):
599 596 if len(self.channelList) == 0:
600 597 self.channelList = dataOut.channelList
601 598
602 599 self.heights = dataOut.heightList
603 600 #print("sels: ",self.selectedHeightsList)
604 601 if len(self.selectedHeightsList)>0 and not self.flag_setIndex:
605 602
606 603 for sel_height in self.selectedHeightsList:
607 604 index_list = numpy.where(self.heights >= sel_height)
608 605 index_list = index_list[0]
609 606 self.height_index.append(index_list[0])
610 607 #print("sels i:"", self.height_index)
611 608 self.flag_setIndex = True
612 609 #print(self.height_index)
613 610 data = {}
614 611 meta = {}
615 612
616 613 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter#*dataOut.nFFTPoints
617 614 n0 = 10*numpy.log10(dataOut.getNoise()/norm)
618 615 noise = numpy.repeat(n0,(dataOut.nFFTPoints*dataOut.nHeights)).reshape(dataOut.nChannels,dataOut.nFFTPoints,dataOut.nHeights)
619 616
620 617
621 618 z = []
622 619 for ch in range(dataOut.nChannels):
623 620 if hasattr(dataOut.normFactor,'shape'):
624 621 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
625 622 else:
626 623 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
627 624
628 625 z = numpy.asarray(z)
629 626 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
630 627 spc = 10*numpy.log10(z)
631 628
632 629
633 630 data['spc'] = spc - noise
634 631 meta['xrange'] = (dataOut.getFreqRange(EXTRA_POINTS)/1000., dataOut.getAcfRange(EXTRA_POINTS), dataOut.getVelRange(EXTRA_POINTS))
635 632
636 633 return data, meta
637 634
638 635 def plot(self):
639 636 if self.xaxis == "frequency":
640 637 x = self.data.xrange[0][0:]
641 638 self.xlabel = "Frequency (kHz)"
642 639 elif self.xaxis == "time":
643 640 x = self.data.xrange[1]
644 641 self.xlabel = "Time (ms)"
645 642 else:
646 643 x = self.data.xrange[2]
647 644 self.xlabel = "Velocity (m/s)"
648 645
649 646 self.titles = []
650 647
651 648 y = self.data.yrange
652 649 z = self.data[-1]['spc']
653 650 #print(z.shape)
654 651 if len(self.height_index) > 0:
655 652 index = self.height_index
656 653 else:
657 654 index = numpy.arange(0, len(y), int((len(y))/9))
658 655 #print("inde x ", index, self.axes)
659 656
660 657 for n, ax in enumerate(self.axes):
661 658
662 659 if ax.firsttime:
663 660
664 661
665 662 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
666 663 self.xmin = self.xmin if self.xmin else -self.xmax
667 664 self.ymin = self.ymin if self.ymin else numpy.nanmin(z)
668 665 self.ymax = self.ymax if self.ymax else numpy.nanmax(z)
669 666
670 667
671 668 ax.plt = ax.plot(x, z[n, :, index].T)
672 669 labels = ['Range = {:2.1f}km'.format(y[i]) for i in index]
673 670 self.figures[0].legend(ax.plt, labels, loc='center right', prop={'size': 8})
674 671 ax.minorticks_on()
675 672 ax.grid(which='major', axis='both')
676 673 ax.grid(which='minor', axis='x')
677 674 else:
678 675 for i, line in enumerate(ax.plt):
679 676 line.set_data(x, z[n, :, index[i]])
680 677
681 678
682 679 self.titles.append('CH {}'.format(self.channelList[n]))
683 680 plt.suptitle(self.maintitle, fontsize=10)
684 681
685 682
686 683 class BeaconPhase(Plot):
687 684
688 685 __isConfig = None
689 686 __nsubplots = None
690 687
691 688 PREFIX = 'beacon_phase'
692 689
693 690 def __init__(self):
694 691 Plot.__init__(self)
695 692 self.timerange = 24*60*60
696 693 self.isConfig = False
697 694 self.__nsubplots = 1
698 695 self.counter_imagwr = 0
699 696 self.WIDTH = 800
700 697 self.HEIGHT = 400
701 698 self.WIDTHPROF = 120
702 699 self.HEIGHTPROF = 0
703 700 self.xdata = None
704 701 self.ydata = None
705 702
706 703 self.PLOT_CODE = BEACON_CODE
707 704
708 705 self.FTP_WEI = None
709 706 self.EXP_CODE = None
710 707 self.SUB_EXP_CODE = None
711 708 self.PLOT_POS = None
712 709
713 710 self.filename_phase = None
714 711
715 712 self.figfile = None
716 713
717 714 self.xmin = None
718 715 self.xmax = None
719 716
720 717 def getSubplots(self):
721 718
722 719 ncol = 1
723 720 nrow = 1
724 721
725 722 return nrow, ncol
726 723
727 724 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
728 725
729 726 self.__showprofile = showprofile
730 727 self.nplots = nplots
731 728
732 729 ncolspan = 7
733 730 colspan = 6
734 731 self.__nsubplots = 2
735 732
736 733 self.createFigure(id = id,
737 734 wintitle = wintitle,
738 735 widthplot = self.WIDTH+self.WIDTHPROF,
739 736 heightplot = self.HEIGHT+self.HEIGHTPROF,
740 737 show=show)
741 738
742 739 nrow, ncol = self.getSubplots()
743 740
744 741 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
745 742
746 743 def save_phase(self, filename_phase):
747 744 f = open(filename_phase,'w+')
748 745 f.write('\n\n')
749 746 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
750 747 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
751 748 f.close()
752 749
753 750 def save_data(self, filename_phase, data, data_datetime):
754 751 f=open(filename_phase,'a')
755 752 timetuple_data = data_datetime.timetuple()
756 753 day = str(timetuple_data.tm_mday)
757 754 month = str(timetuple_data.tm_mon)
758 755 year = str(timetuple_data.tm_year)
759 756 hour = str(timetuple_data.tm_hour)
760 757 minute = str(timetuple_data.tm_min)
761 758 second = str(timetuple_data.tm_sec)
762 759 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
763 760 f.close()
764 761
765 762 def plot(self):
766 763 log.warning('TODO: Not yet implemented...')
767 764
768 765 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
769 766 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
770 767 timerange=None,
771 768 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
772 769 server=None, folder=None, username=None, password=None,
773 770 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
774 771
775 772 if dataOut.flagNoData:
776 773 return dataOut
777 774
778 775 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
779 776 return
780 777
781 778 if pairsList == None:
782 779 pairsIndexList = dataOut.pairsIndexList[:10]
783 780 else:
784 781 pairsIndexList = []
785 782 for pair in pairsList:
786 783 if pair not in dataOut.pairsList:
787 784 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
788 785 pairsIndexList.append(dataOut.pairsList.index(pair))
789 786
790 787 if pairsIndexList == []:
791 788 return
792 789
793 790 # if len(pairsIndexList) > 4:
794 791 # pairsIndexList = pairsIndexList[0:4]
795 792
796 793 hmin_index = None
797 794 hmax_index = None
798 795
799 796 if hmin != None and hmax != None:
800 797 indexes = numpy.arange(dataOut.nHeights)
801 798 hmin_list = indexes[dataOut.heightList >= hmin]
802 799 hmax_list = indexes[dataOut.heightList <= hmax]
803 800
804 801 if hmin_list.any():
805 802 hmin_index = hmin_list[0]
806 803
807 804 if hmax_list.any():
808 805 hmax_index = hmax_list[-1]+1
809 806
810 807 x = dataOut.getTimeRange()
811 808
812 809 thisDatetime = dataOut.datatime
813 810
814 811 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
815 812 xlabel = "Local Time"
816 813 ylabel = "Phase (degrees)"
817 814
818 815 update_figfile = False
819 816
820 817 nplots = len(pairsIndexList)
821 818 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
822 819 phase_beacon = numpy.zeros(len(pairsIndexList))
823 820 for i in range(nplots):
824 821 pair = dataOut.pairsList[pairsIndexList[i]]
825 822 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
826 823 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
827 824 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
828 825 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
829 826 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
830 827
831 828 if dataOut.beacon_heiIndexList:
832 829 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
833 830 else:
834 831 phase_beacon[i] = numpy.average(phase)
835 832
836 833 if not self.isConfig:
837 834
838 835 nplots = len(pairsIndexList)
839 836
840 837 self.setup(id=id,
841 838 nplots=nplots,
842 839 wintitle=wintitle,
843 840 showprofile=showprofile,
844 841 show=show)
845 842
846 843 if timerange != None:
847 844 self.timerange = timerange
848 845
849 846 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
850 847
851 848 if ymin == None: ymin = 0
852 849 if ymax == None: ymax = 360
853 850
854 851 self.FTP_WEI = ftp_wei
855 852 self.EXP_CODE = exp_code
856 853 self.SUB_EXP_CODE = sub_exp_code
857 854 self.PLOT_POS = plot_pos
858 855
859 856 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
860 857 self.isConfig = True
861 858 self.figfile = figfile
862 859 self.xdata = numpy.array([])
863 860 self.ydata = numpy.array([])
864 861
865 862 update_figfile = True
866 863
867 864 #open file beacon phase
868 865 path = '%s%03d' %(self.PREFIX, self.id)
869 866 beacon_file = os.path.join(path,'%s.txt'%self.name)
870 867 self.filename_phase = os.path.join(figpath,beacon_file)
871 868 #self.save_phase(self.filename_phase)
872 869
873 870
874 871 #store data beacon phase
875 872 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
876 873
877 874 self.setWinTitle(title)
878 875
879 876
880 877 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
881 878
882 879 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
883 880
884 881 axes = self.axesList[0]
885 882
886 883 self.xdata = numpy.hstack((self.xdata, x[0:1]))
887 884
888 885 if len(self.ydata)==0:
889 886 self.ydata = phase_beacon.reshape(-1,1)
890 887 else:
891 888 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
892 889
893 890
894 891 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
895 892 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
896 893 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
897 894 XAxisAsTime=True, grid='both'
898 895 )
899 896
900 897 self.draw()
901 898
902 899 if dataOut.ltctime >= self.xmax:
903 900 self.counter_imagwr = wr_period
904 901 self.isConfig = False
905 902 update_figfile = True
906 903
907 904 self.save(figpath=figpath,
908 905 figfile=figfile,
909 906 save=save,
910 907 ftp=ftp,
911 908 wr_period=wr_period,
912 909 thisDatetime=thisDatetime,
913 910 update_figfile=update_figfile)
914 911
915 912 return dataOut
916 913
917 914 class NoiselessSpectraPlot(Plot):
918 915 '''
919 916 Plot for Spectra data, subtracting
920 917 the noise in all channels, using for
921 918 amisr-14 data
922 919 '''
923 920
924 921 CODE = 'noiseless_spc'
925 922 colormap = 'jet'
926 923 plot_type = 'pcolor'
927 924 buffering = False
928 925 channelList = []
929 926 last_noise = None
930 927
931 928 def setup(self):
932 929
933 930 self.nplots = len(self.data.channels)
934 931 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
935 932 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
936 933 self.height = 3.5 * self.nrows
937 934
938 935 self.cb_label = 'dB'
939 936 if self.showprofile:
940 937 self.width = 5.8 * self.ncols
941 938 else:
942 939 self.width = 4.8* self.ncols
943 940 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.92, 'bottom': 0.12})
944 941
945 942 self.ylabel = 'Range [km]'
946 943
947 944
948 945 def update_list(self,dataOut):
949 946 if len(self.channelList) == 0:
950 947 self.channelList = dataOut.channelList
951 948
952 949 def update(self, dataOut):
953 950
954 951 self.update_list(dataOut)
955 952 data = {}
956 953 meta = {}
957 954
958 955 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
959 956 n0 = (dataOut.getNoise()/norm)
960 957 noise = numpy.repeat(n0,(dataOut.nFFTPoints*dataOut.nHeights)).reshape(dataOut.nChannels,dataOut.nFFTPoints,dataOut.nHeights)
961 958 noise = 10*numpy.log10(noise)
962 959
963 960 z = numpy.zeros((dataOut.nChannels, dataOut.nFFTPoints, dataOut.nHeights))
964 961 for ch in range(dataOut.nChannels):
965 962 if hasattr(dataOut.normFactor,'ndim'):
966 963 if dataOut.normFactor.ndim > 1:
967 964 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
968 965 else:
969 966 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
970 967 else:
971 968 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
972 969
973 970 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
974 971 spc = 10*numpy.log10(z)
975 972
976 973
977 974 data['spc'] = spc - noise
978 975 #print(spc.shape)
979 976 data['rti'] = spc.mean(axis=1)
980 977 data['noise'] = noise
981 978
982 979
983 980
984 981 # data['noise'] = noise
985 982 meta['xrange'] = (dataOut.getFreqRange(EXTRA_POINTS)/1000., dataOut.getAcfRange(EXTRA_POINTS), dataOut.getVelRange(EXTRA_POINTS))
986 983
987 984 return data, meta
988 985
989 986 def plot(self):
990 987 if self.xaxis == "frequency":
991 988 x = self.data.xrange[0]
992 989 self.xlabel = "Frequency (kHz)"
993 990 elif self.xaxis == "time":
994 991 x = self.data.xrange[1]
995 992 self.xlabel = "Time (ms)"
996 993 else:
997 994 x = self.data.xrange[2]
998 995 self.xlabel = "Velocity (m/s)"
999 996
1000 997 self.titles = []
1001 998 y = self.data.yrange
1002 999 self.y = y
1003 1000
1004 1001 data = self.data[-1]
1005 1002 z = data['spc']
1006 1003
1007 1004 for n, ax in enumerate(self.axes):
1008 1005 #noise = data['noise'][n]
1009 1006
1010 1007 if ax.firsttime:
1011 1008 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
1012 1009 self.xmin = self.xmin if self.xmin else -self.xmax
1013 1010 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
1014 1011 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
1015 1012 ax.plt = ax.pcolormesh(x, y, z[n].T,
1016 1013 vmin=self.zmin,
1017 1014 vmax=self.zmax,
1018 1015 cmap=plt.get_cmap(self.colormap)
1019 1016 )
1020 1017
1021 1018 if self.showprofile:
1022 1019 ax.plt_profile = self.pf_axes[n].plot(
1023 1020 data['rti'][n], y)[0]
1024 1021
1025 1022
1026 1023 else:
1027 1024 ax.plt.set_array(z[n].T.ravel())
1028 1025 if self.showprofile:
1029 1026 ax.plt_profile.set_data(data['rti'][n], y)
1030 1027
1031 1028
1032 1029 self.titles.append('CH {}'.format(self.channelList[n]))
1033 1030
1034 1031
1035 1032 class NoiselessRTIPlot(RTIPlot):
1036 1033 '''
1037 1034 Plot for RTI data
1038 1035 '''
1039 1036
1040 1037 CODE = 'noiseless_rti'
1041 1038 colormap = 'jet'
1042 1039 plot_type = 'pcolorbuffer'
1043 1040 titles = None
1044 1041 channelList = []
1045 1042 elevationList = []
1046 1043 azimuthList = []
1047 1044 last_noise = None
1048 1045
1049 1046 def setup(self):
1050 1047 self.xaxis = 'time'
1051 1048 self.ncols = 1
1052 1049 #print("dataChannels ",self.data.channels)
1053 1050 self.nrows = len(self.data.channels)
1054 1051 self.nplots = len(self.data.channels)
1055 1052 self.ylabel = 'Range [km]'
1056 1053 #self.xlabel = 'Time'
1057 1054 self.cb_label = 'dB'
1058 1055 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1059 1056 self.titles = ['{} Channel {}'.format(
1060 1057 self.CODE.upper(), x) for x in range(self.nplots)]
1061 1058
1062 1059 def update_list(self,dataOut):
1063 1060 if len(self.channelList) == 0:
1064 1061 self.channelList = dataOut.channelList
1065 1062 if len(self.elevationList) == 0:
1066 1063 self.elevationList = dataOut.elevationList
1067 1064 if len(self.azimuthList) == 0:
1068 1065 self.azimuthList = dataOut.azimuthList
1069 1066
1070 1067 def update(self, dataOut):
1071 1068 if len(self.channelList) == 0:
1072 1069 self.update_list(dataOut)
1073 1070
1074 1071 data = {}
1075 1072 meta = {}
1076 1073 #print(dataOut.max_nIncohInt, dataOut.nIncohInt)
1077 1074 #print(dataOut.windowOfFilter,dataOut.nCohInt,dataOut.nProfiles,dataOut.max_nIncohInt,dataOut.nIncohInt
1078 1075 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
1079 1076 n0 = 10*numpy.log10(dataOut.getNoise()/norm)
1080 1077 data['noise'] = n0
1081 1078 noise = numpy.repeat(n0,dataOut.nHeights).reshape(dataOut.nChannels,dataOut.nHeights)
1082 1079 noiseless_data = dataOut.getPower() - noise
1083 1080
1084 1081 #print("power, noise:", dataOut.getPower(), n0)
1085 1082 #print(noise)
1086 1083 #print(noiseless_data)
1087 1084
1088 1085 data['noiseless_rti'] = noiseless_data
1089 1086
1090 1087 return data, meta
1091 1088
1092 1089 def plot(self):
1093 1090 from matplotlib import pyplot as plt
1094 1091 self.x = self.data.times
1095 1092 self.y = self.data.yrange
1096 1093 self.z = self.data['noiseless_rti']
1097 1094 self.z = numpy.array(self.z, dtype=float)
1098 1095 self.z = numpy.ma.masked_invalid(self.z)
1099 1096
1100 1097
1101 1098 try:
1102 1099 if self.channelList != None:
1103 1100 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
1104 1101 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
1105 1102 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
1106 1103 else:
1107 1104 self.titles = ['{} Channel {}'.format(
1108 1105 self.CODE.upper(), x) for x in self.channelList]
1109 1106 except:
1110 1107 if self.channelList.any() != None:
1111 1108 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
1112 1109 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
1113 1110 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
1114 1111 else:
1115 1112 self.titles = ['{} Channel {}'.format(
1116 1113 self.CODE.upper(), x) for x in self.channelList]
1117 1114
1118 1115
1119 1116 if self.decimation is None:
1120 1117 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1121 1118 else:
1122 1119 x, y, z = self.fill_gaps(*self.decimate())
1123 1120
1124 1121 dummy_var = self.axes #Extrañamente esto actualiza el valor axes
1125 1122 #print("plot shapes ", z.shape, x.shape, y.shape)
1126 1123 #print(self.axes)
1127 1124 for n, ax in enumerate(self.axes):
1128 1125
1129 1126
1130 1127 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
1131 1128 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
1132 1129 data = self.data[-1]
1133 1130 if ax.firsttime:
1134 1131 if (n+1) == len(self.channelList):
1135 1132 ax.set_xlabel('Time')
1136 1133 ax.plt = ax.pcolormesh(x, y, z[n].T,
1137 1134 vmin=self.zmin,
1138 1135 vmax=self.zmax,
1139 1136 cmap=plt.get_cmap(self.colormap)
1140 1137 )
1141 1138 if self.showprofile:
1142 1139 ax.plot_profile = self.pf_axes[n].plot(data['noiseless_rti'][n], self.y)[0]
1143 1140
1144 1141 else:
1145 1142 ax.collections.remove(ax.collections[0])
1146 1143 ax.plt = ax.pcolormesh(x, y, z[n].T,
1147 1144 vmin=self.zmin,
1148 1145 vmax=self.zmax,
1149 1146 cmap=plt.get_cmap(self.colormap)
1150 1147 )
1151 1148 if self.showprofile:
1152 1149 ax.plot_profile.set_data(data['noiseless_rti'][n], self.y)
1153 1150 # if "noise" in self.data:
1154 1151 # #ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
1155 1152 # ax.plot_noise.set_data(data['noise'][n], self.y)
1156 1153
1157 1154
1158 1155 class OutliersRTIPlot(Plot):
1159 1156 '''
1160 1157 Plot for data_xxxx object
1161 1158 '''
1162 1159
1163 1160 CODE = 'outlier_rtc' # Range Time Counts
1164 1161 colormap = 'cool'
1165 1162 plot_type = 'pcolorbuffer'
1166 1163
1167 1164 def setup(self):
1168 1165 self.xaxis = 'time'
1169 1166 self.ncols = 1
1170 1167 self.nrows = self.data.shape('outlier_rtc')[0]
1171 1168 self.nplots = self.nrows
1172 1169 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1173 1170
1174 1171
1175 1172 if not self.xlabel:
1176 1173 self.xlabel = 'Time'
1177 1174
1178 1175 self.ylabel = 'Height [km]'
1179 1176 if not self.titles:
1180 1177 self.titles = ['Outliers Ch:{}'.format(x) for x in range(self.nrows)]
1181 1178
1182 1179 def update(self, dataOut):
1183 1180
1184 1181 data = {}
1185 1182 data['outlier_rtc'] = dataOut.data_outlier
1186 1183
1187 1184 meta = {}
1188 1185
1189 1186 return data, meta
1190 1187
1191 1188 def plot(self):
1192 1189 # self.data.normalize_heights()
1193 1190 self.x = self.data.times
1194 1191 self.y = self.data.yrange
1195 1192 self.z = self.data['outlier_rtc']
1196 1193
1197 1194 #self.z = numpy.ma.masked_invalid(self.z)
1198 1195
1199 1196 if self.decimation is None:
1200 1197 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1201 1198 else:
1202 1199 x, y, z = self.fill_gaps(*self.decimate())
1203 1200
1204 1201 for n, ax in enumerate(self.axes):
1205 1202
1206 1203 self.zmax = self.zmax if self.zmax is not None else numpy.max(
1207 1204 self.z[n])
1208 1205 self.zmin = self.zmin if self.zmin is not None else numpy.min(
1209 1206 self.z[n])
1210 1207 data = self.data[-1]
1211 1208 if ax.firsttime:
1212 1209 if self.zlimits is not None:
1213 1210 self.zmin, self.zmax = self.zlimits[n]
1214 1211
1215 1212 ax.plt = ax.pcolormesh(x, y, z[n].T,
1216 1213 vmin=self.zmin,
1217 1214 vmax=self.zmax,
1218 1215 cmap=self.cmaps[n]
1219 1216 )
1220 1217 if self.showprofile:
1221 1218 ax.plot_profile = self.pf_axes[n].plot(data['outlier_rtc'][n], self.y)[0]
1222 1219 self.pf_axes[n].set_xlabel('')
1223 1220 else:
1224 1221 if self.zlimits is not None:
1225 1222 self.zmin, self.zmax = self.zlimits[n]
1226 1223 ax.collections.remove(ax.collections[0])
1227 1224 ax.plt = ax.pcolormesh(x, y, z[n].T ,
1228 1225 vmin=self.zmin,
1229 1226 vmax=self.zmax,
1230 1227 cmap=self.cmaps[n]
1231 1228 )
1232 1229 if self.showprofile:
1233 1230 ax.plot_profile.set_data(data['outlier_rtc'][n], self.y)
1234 1231 self.pf_axes[n].set_xlabel('')
1235 1232
1236 1233 class NIncohIntRTIPlot(Plot):
1237 1234 '''
1238 1235 Plot for data_xxxx object
1239 1236 '''
1240 1237
1241 1238 CODE = 'integrations_rtc' # Range Time Counts
1242 1239 colormap = 'BuGn'
1243 1240 plot_type = 'pcolorbuffer'
1244 1241
1245 1242 def setup(self):
1246 1243 self.xaxis = 'time'
1247 1244 self.ncols = 1
1248 1245 self.nrows = self.data.shape('integrations_rtc')[0]
1249 1246 self.nplots = self.nrows
1250 1247 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1251 1248
1252 1249
1253 1250 if not self.xlabel:
1254 1251 self.xlabel = 'Time'
1255 1252
1256 1253 self.ylabel = 'Height [km]'
1257 1254 if not self.titles:
1258 1255 self.titles = ['Integration Ch:{}'.format(x) for x in range(self.nrows)]
1259 1256
1260 1257 def update(self, dataOut):
1261 1258
1262 1259 data = {}
1263 1260 data['integrations_rtc'] = dataOut.nIncohInt
1264 1261
1265 1262 meta = {}
1266 1263
1267 1264 return data, meta
1268 1265
1269 1266 def plot(self):
1270 1267 # self.data.normalize_heights()
1271 1268 self.x = self.data.times
1272 1269 self.y = self.data.yrange
1273 1270 self.z = self.data['integrations_rtc']
1274 1271
1275 1272 #self.z = numpy.ma.masked_invalid(self.z)
1276 1273
1277 1274 if self.decimation is None:
1278 1275 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1279 1276 else:
1280 1277 x, y, z = self.fill_gaps(*self.decimate())
1281 1278
1282 1279 for n, ax in enumerate(self.axes):
1283 1280
1284 1281 self.zmax = self.zmax if self.zmax is not None else numpy.max(
1285 1282 self.z[n])
1286 1283 self.zmin = self.zmin if self.zmin is not None else numpy.min(
1287 1284 self.z[n])
1288 1285 data = self.data[-1]
1289 1286 if ax.firsttime:
1290 1287 if self.zlimits is not None:
1291 1288 self.zmin, self.zmax = self.zlimits[n]
1292 1289
1293 1290 ax.plt = ax.pcolormesh(x, y, z[n].T,
1294 1291 vmin=self.zmin,
1295 1292 vmax=self.zmax,
1296 1293 cmap=self.cmaps[n]
1297 1294 )
1298 1295 if self.showprofile:
1299 1296 ax.plot_profile = self.pf_axes[n].plot(data['integrations_rtc'][n], self.y)[0]
1300 1297 self.pf_axes[n].set_xlabel('')
1301 1298 else:
1302 1299 if self.zlimits is not None:
1303 1300 self.zmin, self.zmax = self.zlimits[n]
1304 1301 ax.collections.remove(ax.collections[0])
1305 1302 ax.plt = ax.pcolormesh(x, y, z[n].T ,
1306 1303 vmin=self.zmin,
1307 1304 vmax=self.zmax,
1308 1305 cmap=self.cmaps[n]
1309 1306 )
1310 1307 if self.showprofile:
1311 1308 ax.plot_profile.set_data(data['integrations_rtc'][n], self.y)
1312 1309 self.pf_axes[n].set_xlabel('')
1313 1310
1314 1311
1315 1312 import datetime
1316 1313 class NoiselessRTILinePlot(Plot):
1317 1314 '''
1318 1315 Plot for RTI data
1319 1316 '''
1320 1317
1321 1318 CODE = 'noiseless_rtiLine'
1322 1319
1323 1320 plot_type = 'scatter'
1324 1321 titles = None
1325 1322 channelList = []
1326 1323 elevationList = []
1327 1324 azimuthList = []
1328 1325 last_noise = None
1329 1326
1330 1327 def setup(self):
1331 1328 self.xaxis = 'Range (Km)'
1332 1329 self.nplots = len(self.data.channels)
1333 1330 self.nrows = int(numpy.ceil(self.nplots/2))
1334 1331 self.ncols = int(numpy.ceil(self.nplots/self.nrows))
1335 1332 self.ylabel = 'Intensity [dB]'
1336 1333 self.titles = ['Channel '+str(self.data.channels[i])+" " for i in self.data.channels]
1337 1334 self.colorbar = False
1338 1335 self.width = 6
1339 1336 self.height = 4
1340 1337
1341 1338 def update_list(self,dataOut):
1342 1339 if len(self.channelList) == 0:
1343 1340 self.channelList = dataOut.channelList
1344 1341 if len(self.elevationList) == 0:
1345 1342 self.elevationList = dataOut.elevationList
1346 1343 if len(self.azimuthList) == 0:
1347 1344 self.azimuthList = dataOut.azimuthList
1348 1345
1349 1346 def update(self, dataOut):
1350 1347 if len(self.channelList) == 0:
1351 1348 self.update_list(dataOut)
1352 1349
1353 1350 data = {}
1354 1351 meta = {}
1355 1352 #print(dataOut.max_nIncohInt, dataOut.nIncohInt)
1356 1353 #print(dataOut.windowOfFilter,dataOut.nCohInt,dataOut.nProfiles,dataOut.max_nIncohInt,dataOut.nIncohInt)
1357 1354 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
1358 1355
1359 1356 n0 = 10*numpy.log10(dataOut.getNoise()/norm)
1360 1357 data['noise'] = n0
1361 1358
1362 1359 noise = numpy.repeat(n0,dataOut.nHeights).reshape(dataOut.nChannels,dataOut.nHeights)
1363 1360 noiseless_data = dataOut.getPower() - noise
1364 1361
1365 1362 #print("power, noise:", dataOut.getPower(), n0)
1366 1363 #print(noise)
1367 1364 #print(noiseless_data)
1368 1365
1369 1366 data['noiseless_rtiLine'] = noiseless_data
1370 1367
1371 1368 #print(noiseless_data.shape, self.name)
1372 1369 data['time'] = dataOut.utctime
1373 1370
1374 1371 return data, meta
1375 1372
1376 1373 def plot(self):
1377 1374
1378 1375 self.x = self.data.times
1379 1376 self.y = self.data.yrange
1380 1377 #print(self.data['noiseless_rtiLine'].shape, self.y.shape, self.name)
1381 1378 #ts = self.data['time'][0].squeeze()
1382 1379 if len(self.data['noiseless_rtiLine'])>2 :
1383 1380 self.z = self.data['noiseless_rtiLine'][:, -1,:]
1384 1381 else:
1385 1382 self.z = self.data['noiseless_rtiLine']
1386 1383 #print(self.z.shape, self.y.shape, ts)
1387 1384 #thisDatetime = datetime.datetime.utcfromtimestamp(ts)
1388 1385
1389 1386 for i,ax in enumerate(self.axes):
1390 1387 #self.titles[i] = "Channel {} {}".format(i, thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1391 1388
1392 1389
1393 1390 if ax.firsttime:
1394 1391 #self.xmin = min(self.z)
1395 1392 #self.xmax = max(self.z)
1396 1393 ax.plt_r = ax.plot(self.z[i], self.y)[0]
1397 1394 else:
1398 1395 ax.plt_r.set_data(self.z[i], self.y)
1399 1396
1400 1397
1401 1398
1402 1399 class GeneralProfilePlot(Plot):
1403 1400 '''
1404 1401 Plot for RTI data
1405 1402 '''
1406 1403
1407 1404 CODE = 'general_profilePlot'
1408 1405
1409 1406 plot_type = 'scatter'
1410 1407 titles = None
1411 1408 channelList = []
1412 1409 elevationList = []
1413 1410 azimuthList = []
1414 1411 last_noise = None
1415 1412
1416 1413 def setup(self):
1417 1414 self.xaxis = 'Range (Km)'
1418 1415 self.nplots = len(self.data.channels)
1419 1416 self.nrows = int(numpy.ceil(self.nplots/2))
1420 1417 self.ncols = int(numpy.ceil(self.nplots/self.nrows))
1421 1418 self.ylabel = 'Intensity [dB]'
1422 1419 self.titles = ['Channel '+str(self.data.channels[i])+" " for i in self.data.channels]
1423 1420 self.colorbar = False
1424 1421 self.width = 6
1425 1422 self.height = 4
1426 1423
1427 1424 def update_list(self,dataOut):
1428 1425 if len(self.channelList) == 0:
1429 1426 self.channelList = dataOut.channelList
1430 1427 if len(self.elevationList) == 0:
1431 1428 self.elevationList = dataOut.elevationList
1432 1429 if len(self.azimuthList) == 0:
1433 1430 self.azimuthList = dataOut.azimuthList
1434 1431
1435 1432 def update(self, dataOut):
1436 1433 if len(self.channelList) == 0:
1437 1434 self.update_list(dataOut)
1438 1435
1439 1436 data = {}
1440 1437 meta = {}
1441 1438
1442 1439 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
1443 1440 n0 = 10*numpy.log10(dataOut.getNoise()/norm)
1444 1441 data['noise'] = n0
1445 1442
1446 1443 noise = numpy.repeat(n0,dataOut.nHeights).reshape(dataOut.nChannels,dataOut.nHeights)
1447 1444 noiseless_data = dataOut.getPower() - noise
1448 1445
1449 1446 data['noiseless_rtiLine'] = noiseless_data
1450 1447
1451 1448 #print(noiseless_data.shape, self.name)
1452 1449 data['time'] = dataOut.utctime
1453 1450
1454 1451 return data, meta
1455 1452
1456 1453 def plot(self):
1457 1454
1458 1455 self.x = self.data.times
1459 1456 self.y = self.data.yrange
1460 1457 #print(self.data['noiseless_rtiLine'].shape, self.y.shape, self.name)
1461 1458 #ts = self.data['time'][0].squeeze()
1462 1459 if len(self.data['noiseless_rtiLine'])>2 :
1463 1460 self.z = self.data['noiseless_rtiLine'][:, -1,:]
1464 1461 else:
1465 1462 self.z = self.data['noiseless_rtiLine']
1466 1463 #print(self.z.shape, self.y.shape, ts)
1467 1464 #thisDatetime = datetime.datetime.utcfromtimestamp(ts)
1468 1465
1469 1466 for i,ax in enumerate(self.axes):
1470 1467 #self.titles[i] = "Channel {} {}".format(i, thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1471 1468
1472 1469
1473 1470 if ax.firsttime:
1474 1471 #self.xmin = min(self.z)
1475 1472 #self.xmax = max(self.z)
1476 1473 ax.plt_r = ax.plot(self.z[i], self.y)[0]
1477 1474 else:
1478 1475 ax.plt_r.set_data(self.z[i], self.y)
1479 1476
1480 1477
1481 1478 ##########################################################################################################
1482 1479 ########################################## AMISR_V4 ######################################################
1483 1480
1484 1481 class RTIMapPlot(Plot):
1485 1482 '''
1486 1483 Plot for RTI data
1487 1484
1488 1485 Example:
1489 1486
1490 1487 controllerObj = Project()
1491 1488 controllerObj.setup(id = '11', name='eej_proc', description=desc)
1492 1489 ##.......................................................................................
1493 1490 ##.......................................................................................
1494 1491 readUnitConfObj = controllerObj.addReadUnit(datatype='AMISRReader', path=inPath, startDate='2023/05/24',endDate='2023/05/24',
1495 1492 startTime='12:00:00',endTime='12:45:59',walk=1,timezone='lt',margin_days=1,code = code,nCode = nCode,
1496 1493 nBaud = nBaud,nOsamp = nosamp,nChannels=nChannels,nFFT=NFFT,
1497 1494 syncronization=False,shiftChannels=0)
1498 1495
1499 1496 volts_proc = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
1500 1497
1501 1498 opObj01 = volts_proc.addOperation(name='Decoder', optype='other')
1502 1499 opObj01.addParameter(name='code', value=code, format='floatlist')
1503 1500 opObj01.addParameter(name='nCode', value=1, format='int')
1504 1501 opObj01.addParameter(name='nBaud', value=nBaud, format='int')
1505 1502 opObj01.addParameter(name='osamp', value=nosamp, format='int')
1506 1503
1507 1504 opObj12 = volts_proc.addOperation(name='selectHeights', optype='self')
1508 1505 opObj12.addParameter(name='minHei', value='90', format='float')
1509 1506 opObj12.addParameter(name='maxHei', value='150', format='float')
1510 1507
1511 1508 proc_spc = controllerObj.addProcUnit(datatype='SpectraProc', inputId=volts_proc.getId())
1512 1509 proc_spc.addParameter(name='nFFTPoints', value='8', format='int')
1513 1510
1514 1511 opObj11 = proc_spc.addOperation(name='IncohInt', optype='other')
1515 1512 opObj11.addParameter(name='n', value='1', format='int')
1516 1513
1517 1514 beamMapFile = "/home/japaza/Documents/AMISR_sky_mapper/UMET_beamcodes.csv"
1518 1515
1519 1516 opObj12 = proc_spc.addOperation(name='RTIMapPlot', optype='external')
1520 1517 opObj12.addParameter(name='selectedHeightsList', value='95, 100, 105, 110 ', format='int')
1521 1518 opObj12.addParameter(name='bField', value='100', format='int')
1522 1519 opObj12.addParameter(name='filename', value=beamMapFile, format='str')
1523 1520
1524 1521 '''
1525 1522
1526 1523 CODE = 'rti_skymap'
1527 1524
1528 1525 plot_type = 'scatter'
1529 1526 titles = None
1530 1527 colormap = 'jet'
1531 1528 channelList = []
1532 1529 elevationList = []
1533 1530 azimuthList = []
1534 1531 last_noise = None
1535 1532 flag_setIndex = False
1536 1533 heights = []
1537 1534 dcosx = []
1538 1535 dcosy = []
1539 1536 fullDcosy = None
1540 1537 fullDcosy = None
1541 1538 hindex = []
1542 1539 mapFile = False
1543 1540 ##### BField ####
1544 1541 flagBField = False
1545 1542 dcosxB = []
1546 1543 dcosyB = []
1547 1544 Bmarker = ['+','*','D','x','s','>','o','^']
1548 1545
1549 1546
1550 1547
1551 1548 def setup(self):
1552 1549
1553 1550 self.xaxis = 'Range (Km)'
1554 1551 if len(self.selectedHeightsList) > 0:
1555 1552 self.nplots = len(self.selectedHeightsList)
1556 1553 else:
1557 1554 self.nplots = 4
1558 1555 self.ncols = int(numpy.ceil(self.nplots/2))
1559 1556 self.nrows = int(numpy.ceil(self.nplots/self.ncols))
1560 1557 self.ylabel = 'dcosy'
1561 1558 self.xlabel = 'dcosx'
1562 1559 self.colorbar = True
1563 1560 self.width = 6 + 4.1*self.nrows
1564 1561 self.height = 3 + 3.5*self.ncols
1565 1562
1566 1563
1567 1564 if self.extFile!=None:
1568 1565 try:
1569 1566 pointings = numpy.genfromtxt(self.extFile, delimiter=',')
1570 1567 full_azi = pointings[:,1]
1571 1568 full_elev = pointings[:,2]
1572 1569 self.fullDcosx = numpy.cos(numpy.radians(full_elev))*numpy.sin(numpy.radians(full_azi))
1573 1570 self.fullDcosy = numpy.cos(numpy.radians(full_elev))*numpy.cos(numpy.radians(full_azi))
1574 1571 mapFile = True
1575 1572 except Exception as e:
1576 1573 self.extFile = None
1577 1574 print(e)
1578 1575
1579 1576
1580 1577
1581 1578
1582 1579 def update_list(self,dataOut):
1583 1580 if len(self.channelList) == 0:
1584 1581 self.channelList = dataOut.channelList
1585 1582 if len(self.elevationList) == 0:
1586 1583 self.elevationList = dataOut.elevationList
1587 1584 if len(self.azimuthList) == 0:
1588 1585 self.azimuthList = dataOut.azimuthList
1589 1586 a = numpy.radians(numpy.asarray(self.azimuthList))
1590 1587 e = numpy.radians(numpy.asarray(self.elevationList))
1591 1588 self.heights = dataOut.heightList
1592 1589 self.dcosx = numpy.cos(e)*numpy.sin(a)
1593 1590 self.dcosy = numpy.cos(e)*numpy.cos(a)
1594 1591
1595 1592 if len(self.bFieldList)>0:
1596 1593 datetObj = datetime.datetime.fromtimestamp(dataOut.utctime)
1597 1594 doy = datetObj.timetuple().tm_yday
1598 1595 year = datetObj.year
1599 1596 # self.dcosxB, self.dcosyB
1600 1597 ObjB = BField(year=year,doy=doy,site=2,heights=self.bFieldList)
1601 1598 [dcos, alpha, nlon, nlat] = ObjB.getBField()
1602 1599
1603 1600 alpha_location = numpy.zeros((nlon,2,len(self.bFieldList)))
1604 1601 for ih in range(len(self.bFieldList)):
1605 1602 alpha_location[:,0,ih] = dcos[:,0,ih,0]
1606 1603 for ilon in numpy.arange(nlon):
1607 1604 myx = (alpha[ilon,:,ih])[::-1]
1608 1605 myy = (dcos[ilon,:,ih,0])[::-1]
1609 1606 tck = splrep(myx,myy,s=0)
1610 1607 mydcosx = splev(ObjB.alpha_i,tck,der=0)
1611 1608
1612 1609 myx = (alpha[ilon,:,ih])[::-1]
1613 1610 myy = (dcos[ilon,:,ih,1])[::-1]
1614 1611 tck = splrep(myx,myy,s=0)
1615 1612 mydcosy = splev(ObjB.alpha_i,tck,der=0)
1616 1613 alpha_location[ilon,:,ih] = numpy.array([mydcosx, mydcosy])
1617 1614 self.dcosxB.append(alpha_location[:,0,ih])
1618 1615 self.dcosyB.append(alpha_location[:,1,ih])
1619 1616 self.flagBField = True
1620 1617
1621 1618 if len(self.celestialList)>0:
1622 1619 #getBField(self.bFieldList, date)
1623 1620 #pass = kwargs.get('celestial', [])
1624 1621 pass
1625 1622
1626 1623
1627 1624
1628 1625 def update(self, dataOut):
1629 1626
1630 1627 if len(self.channelList) == 0:
1631 1628 self.update_list(dataOut)
1632 1629
1633 1630 if not self.flag_setIndex:
1634 1631 if len(self.selectedHeightsList)>0:
1635 1632 for sel_height in self.selectedHeightsList:
1636 1633 index_list = numpy.where(self.heights >= sel_height)
1637 1634 index_list = index_list[0]
1638 1635 self.hindex.append(index_list[0])
1639 1636 # else:
1640 1637 # k = len(self.heights)
1641 1638 # self.hindex.append(int(k/2))
1642 1639 self.flag_setIndex = True
1643 1640
1644 1641 data = {}
1645 1642 meta = {}
1646 1643
1647 1644 data['rti_skymap'] = dataOut.getPower()
1648 1645 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
1649 1646 noise = 10*numpy.log10(dataOut.getNoise()/norm)
1650 1647 data['noise'] = noise
1651 1648
1652 1649 return data, meta
1653 1650
1654 1651 def plot(self):
1655 1652
1656 1653 ######
1657 1654 self.x = self.dcosx
1658 1655 self.y = self.dcosy
1659 1656 self.z = self.data[-1]['rti_skymap']
1660 1657 self.z = numpy.array(self.z, dtype=float)
1661 1658
1662 1659 #print("inde x1 ", self.height_index)
1663 1660 if len(self.hindex) > 0:
1664 1661 index = self.hindex
1665 1662 else:
1666 1663 index = numpy.arange(0, len(self.heights), int((len(self.heights))/4.2))
1667 1664
1668 1665 #print(index)
1669 1666 self.titles = ['Height {:.2f} km '.format(self.heights[i])+" " for i in index]
1670 1667 for n, ax in enumerate(self.axes):
1671 1668
1672 1669 if ax.firsttime:
1673 1670
1674 1671
1675 1672 self.xmax = self.xmax if self.xmax else numpy.nanmax(self.x)
1676 1673 self.xmin = self.xmin if self.xmin else numpy.nanmin(self.x)
1677 1674
1678 1675 self.ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
1679 1676 self.ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
1680 1677
1681 1678 self.zmax = self.zmax if self.zmax else numpy.nanmax(self.z)
1682 1679 self.zmin = self.zmin if self.zmin else numpy.nanmin(self.z)
1683 1680
1684 1681
1685 1682 if self.extFile!=None:
1686 1683 ax.scatter(self.fullDcosx, self.fullDcosy, marker="+", s=20)
1687 1684 #print(self.fullDcosx)
1688 1685 pass
1689 1686
1690 1687
1691 1688 ax.plt = ax.scatter(self.x, self.y, c=self.z[:,index[n]], cmap = 'jet',vmin = self.zmin,
1692 1689 s=60, marker="s", vmax = self.zmax)
1693 1690
1694 1691
1695 1692 ax.minorticks_on()
1696 1693 ax.grid(which='major', axis='both')
1697 1694 ax.grid(which='minor', axis='x')
1698 1695
1699 1696 if self.flagBField :
1700 1697
1701 1698 for ih in range(len(self.bFieldList)):
1702 1699 label = str(self.bFieldList[ih]) + ' km'
1703 1700 ax.plot(self.dcosxB[ih], self.dcosyB[ih], color='k', marker=self.Bmarker[ih % 8],
1704 1701 label=label, linestyle='--', ms=4.0,lw=0.5)
1705 1702 handles, labels = ax.get_legend_handles_labels()
1706 1703 a = -0.05
1707 1704 b = 1.15 - 1.19*(self.nrows)
1708 1705 self.axes[0].legend(handles,labels, bbox_to_anchor=(a,b), prop={'size': (5.8+ 1.1*self.nplots)}, title='B Field ⊥')
1709 1706
1710 1707 else:
1711 1708
1712 1709 ax.plt = ax.scatter(self.x, self.y, c=self.z[:,index[n]], cmap = 'jet',vmin = self.zmin,
1713 1710 s=80, marker="s", vmax = self.zmax)
1714 1711
1715 1712 if self.flagBField :
1716 1713 for ih in range(len(self.bFieldList)):
1717 1714 ax.plot (self.dcosxB[ih], self.dcosyB[ih], color='k', marker=self.Bmarker[ih % 8],
1718 1715 linestyle='--', ms=4.0,lw=0.5)
1719 1716
1720 1717 # handles, labels = ax.get_legend_handles_labels()
1721 1718 # a = -0.05
1722 1719 # b = 1.15 - 1.19*(self.nrows)
1723 1720 # self.axes[0].legend(handles,labels, bbox_to_anchor=(a,b), prop={'size': (5.8+ 1.1*self.nplots)}, title='B Field ⊥')
1724 1721
1725 1722
Show file before | Show file after | Hide comments
@@ -1,783 +1,779
1 1 ''''
2 2 Created on Set 9, 2015
3 3
4 4 @author: roj-idl71 Karim Kuyeng
5 5
6 6 @upgrade: 2021, Joab Apaza
7 7
8 8 '''
9 9
10 10 import os
11 11 import sys
12 12 import glob
13 13 import fnmatch
14 14 import datetime
15 15 import time
16 16 import re
17 17 import h5py
18 18 import numpy
19 19
20 20 try:
21 21 from gevent import sleep
22 22 except:
23 23 from time import sleep
24 24
25 25 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader,ProcessingHeader
26 26 from schainpy.model.data.jrodata import Voltage
27 27 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
28 28 from numpy import imag
29 29 from schainpy.utils import log
30 30
31 31
32 32 class AMISRReader(ProcessingUnit):
33 33 '''
34 34 classdocs
35 35 '''
36 36
37 37 def __init__(self):
38 38 '''
39 39 Constructor
40 40 '''
41 41
42 42 ProcessingUnit.__init__(self)
43 43
44 44 self.set = None
45 45 self.subset = None
46 46 self.extension_file = '.h5'
47 47 self.dtc_str = 'dtc'
48 48 self.dtc_id = 0
49 49 self.status = True
50 50 self.isConfig = False
51 51 self.dirnameList = []
52 52 self.filenameList = []
53 53 self.fileIndex = None
54 54 self.flagNoMoreFiles = False
55 55 self.flagIsNewFile = 0
56 56 self.filename = ''
57 57 self.amisrFilePointer = None
58 58
59 59 self.beamCodeMap = None
60 60 self.azimuthList = []
61 61 self.elevationList = []
62 62 self.dataShape = None
63 63 self.flag_old_beams = False
64 64
65 65 self.flagAsync = False #Use when the experiment has no syncronization
66 66 self.shiftChannels = 0
67 67 self.profileIndex = 0
68 68
69 69
70 70 self.beamCodeByFrame = None
71 71 self.radacTimeByFrame = None
72 72
73 73 self.dataset = None
74 74
75 75 self.__firstFile = True
76 76
77 77 self.buffer = None
78 78
79 79 self.timezone = 'ut'
80 80
81 81 self.__waitForNewFile = 20
82 82 self.__filename_online = None
83 83 #Is really necessary create the output object in the initializer
84 84 self.dataOut = Voltage()
85 85 self.dataOut.error=False
86 86 self.margin_days = 1
87 87 self.flag_ignoreFiles = False #to activate the ignoring Files flag
88 88 self.flag_standby = False # just keep waiting, use when ignoring files
89 89 self.ignStartDateTime=None
90 90 self.ignEndDateTime=None
91 91
92 92 def setup(self,path=None,
93 93 startDate=None,
94 94 endDate=None,
95 95 startTime=None,
96 96 endTime=None,
97 97 walk=True,
98 98 timezone='ut',
99 99 all=0,
100 100 code = 1,
101 101 nCode = 1,
102 102 nBaud = 0,
103 103 nOsamp = 0,
104 104 online=False,
105 105 old_beams=False,
106 106 margin_days=1,
107 107 nFFT = None,
108 108 nChannels = None,
109 109 ignStartDate=None,
110 110 ignEndDate=None,
111 111 ignStartTime=None,
112 112 ignEndTime=None,
113 113 syncronization=True,
114 114 shiftChannels=0
115 115 ):
116 116
117 117
118 118
119 119 self.timezone = timezone
120 120 self.all = all
121 121 self.online = online
122 122 self.flag_old_beams = old_beams
123 123 self.code = code
124 124 self.nCode = int(nCode)
125 125 self.nBaud = int(nBaud)
126 126 self.nOsamp = int(nOsamp)
127 127 self.margin_days = margin_days
128 128 self.__sampleRate = None
129 129 self.flagAsync = not syncronization
130 130 self.shiftChannels = shiftChannels
131 131 self.nFFT = nFFT
132 132 self.nChannels = nChannels
133 133 if ignStartTime!=None and ignEndTime!=None:
134 134 if ignStartDate!=None and ignEndDate!=None:
135 135 self.ignStartDateTime=datetime.datetime.combine(ignStartDate,ignStartTime)
136 136 self.ignEndDateTime=datetime.datetime.combine(ignEndDate,ignEndTime)
137 137 else:
138 138 self.ignStartDateTime=datetime.datetime.combine(startDate,ignStartTime)
139 139 self.ignEndDateTime=datetime.datetime.combine(endDate,ignEndTime)
140 140 self.flag_ignoreFiles = True
141 141
142 142 #self.findFiles()
143 143 if not(online):
144 144 #Busqueda de archivos offline
145 145 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk,)
146 146 else:
147 147 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
148 148
149 149 if not(self.filenameList):
150 150 raise schainpy.admin.SchainWarning("There is no files into the folder: %s"%(path))
151 151 #sys.exit(0)
152 152 self.dataOut.error = True
153 153
154 154 self.fileIndex = 0
155 155
156 156 self.readNextFile(online)
157 157
158 158 '''
159 159 Add code
160 160 '''
161 161 self.isConfig = True
162 162 # print("Setup Done")
163 163 pass
164 164
165 165
166 166 def readAMISRHeader(self,fp):
167 167
168 168 if self.isConfig and (not self.flagNoMoreFiles):
169 169 newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
170 170 if self.dataShape != newShape and newShape != None and not self.flag_standby:
171 171 raise schainpy.admin.SchainError("NEW FILE HAS A DIFFERENT SHAPE: ")
172 172 print(self.dataShape,newShape,"\n")
173 173 return 0
174 174 else:
175 175 self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
176 176
177 177
178 178 header = 'Raw11/Data/RadacHeader'
179 179 if self.nChannels == None:
180 180 expFile = fp['Setup/Experimentfile'][()].decode()
181 181 linesExp = expFile.split("\n")
182 182 a = [line for line in linesExp if "nbeamcodes" in line]
183 183 self.nChannels = int(a[0][11:])
184 184
185 185 if not self.flagAsync: #for experiments with no syncronization
186 186 self.shiftChannels = 0
187 187
188 188
189 189
190 190 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
191 191
192 192
193 193 if (self.startDate > datetime.date(2021, 7, 15)) or self.flag_old_beams: #Se cambió la forma de extracción de Apuntes el 17 o forzar con flag de reorganización
194 194 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
195 195 self.trueBeams = self.beamcodeFile.split("\n")
196 196 self.trueBeams.pop()#remove last
197 197 if self.nFFT == None:
198 198 log.error("FFT or number of repetitions per channels is needed",self.name)
199 199 beams_idx = [k*self.nFFT for k in range(self.nChannels)]
200 200 beams = [self.trueBeams[b] for b in beams_idx]
201 201 self.beamCode = [int(x, 16) for x in beams]
202 202
203 203 if(self.flagAsync and self.shiftChannels == 0):
204 204 initBeam = self.beamCodeByPulse[0, 0]
205 205 self.shiftChannels = numpy.argwhere(self.beamCode ==initBeam)[0,0]
206 206
207 207 else:
208 208 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
209 209 self.beamCode = _beamCode[0,:]
210 210
211 211
212 212
213 213
214 214 if self.beamCodeMap == None:
215 215 self.beamCodeMap = fp['Setup/BeamcodeMap']
216 216 for beam in self.beamCode:
217 217 beamAziElev = numpy.where(self.beamCodeMap[:,0]==beam)
218 218 beamAziElev = beamAziElev[0].squeeze()
219 219 self.azimuthList.append(self.beamCodeMap[beamAziElev,1])
220 220 self.elevationList.append(self.beamCodeMap[beamAziElev,2])
221 221 #print("Beamssss: ",self.beamCodeMap[beamAziElev,1],self.beamCodeMap[beamAziElev,2])
222 222 #print(self.beamCode)
223 223 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
224 224 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
225 225 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
226 226 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
227 227 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
228 228 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
229 229 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
230 230 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
231 231 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
232 232 self.frequency = fp.get('Rx/Frequency')
233 233 txAus = fp.get('Raw11/Data/Pulsewidth') #seconds
234 234 self.baud = fp.get('Raw11/Data/TxBaud')
235 235 sampleRate = fp.get('Rx/SampleRate')
236 236 self.__sampleRate = sampleRate[()]
237 237 self.nblocks = self.pulseCount.shape[0] #nblocks
238 238 self.profPerBlockRAW = self.pulseCount.shape[1] #profiles per block in raw data
239 239 self.nprofiles = self.pulseCount.shape[1] #nprofile
240 240 #self.nsa = self.nsamplesPulse[0,0] #ngates
241 241 self.nsa = len(self.rangeFromFile[0])
242 242 self.nchannels = len(self.beamCode)
243 243 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
244 244 #print("IPPS secs: ",self.ippSeconds)
245 245 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
246 246 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
247 247
248 248 #filling radar controller header parameters
249 249 self.__ippKm = self.ippSeconds *.15*1e6 # in km
250 250 #self.__txA = txAus[()]*.15 #(ipp[us]*.15km/1us) in km
251 251 self.__txA = txAus[()] #seconds
252 252 self.__txAKm = self.__txA*1e6*.15
253 253 self.__txB = 0
254 254 nWindows=1
255 255 self.__nSamples = self.nsa
256 256 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
257 257 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
258 258 #print("amisr-ipp:",self.ippSeconds, self.__ippKm)
259 259 #for now until understand why the code saved is different (code included even though code not in tuf file)
260 260 #self.__codeType = 0
261 261 # self.__nCode = None
262 262 # self.__nBaud = None
263 263 self.__code = self.code
264 264 self.__codeType = 0
265 265 if self.code != None:
266 266 self.__codeType = 1
267 267 self.__nCode = self.nCode
268 268 self.__nBaud = self.nBaud
269 269 self.__baudTX = self.__txA/(self.nBaud)
270 270 #self.__code = 0
271 271
272 272 #filling system header parameters
273 273 self.__nSamples = self.nsa
274 274 self.newProfiles = self.nprofiles/self.nchannels
275 275 self.__channelList = [n for n in range(self.nchannels)]
276 276
277 277 self.__frequency = self.frequency[0][0]
278 278
279 279
280 280 return 1
281 281
282 282
283 283 def createBuffers(self):
284 284
285 285 pass
286 286
287 287 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
288 288 self.path = path
289 289 self.startDate = startDate
290 290 self.endDate = endDate
291 291 self.startTime = startTime
292 292 self.endTime = endTime
293 293 self.walk = walk
294 294
295 295
296 296 def __checkPath(self):
297 297 if os.path.exists(self.path):
298 298 self.status = 1
299 299 else:
300 300 self.status = 0
301 301 print('Path:%s does not exists'%self.path)
302 302
303 303 return
304 304
305 305
306 306 def __selDates(self, amisr_dirname_format):
307 307 try:
308 308 year = int(amisr_dirname_format[0:4])
309 309 month = int(amisr_dirname_format[4:6])
310 310 dom = int(amisr_dirname_format[6:8])
311 311 thisDate = datetime.date(year,month,dom)
312 312 #margen de un día extra, igual luego se filtra for fecha y hora
313 313 if (thisDate>=(self.startDate - datetime.timedelta(days=self.margin_days)) and thisDate <= (self.endDate)+ datetime.timedelta(days=1)):
314 314 return amisr_dirname_format
315 315 except:
316 316 return None
317 317
318 318
319 319 def __findDataForDates(self,online=False):
320 320
321 321 if not(self.status):
322 322 return None
323 323
324 324 pat = '\d+.\d+'
325 325 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
326 326 dirnameList = [x for x in dirnameList if x!=None]
327 327 dirnameList = [x.string for x in dirnameList]
328 328 if not(online):
329 329 dirnameList = [self.__selDates(x) for x in dirnameList]
330 330 dirnameList = [x for x in dirnameList if x!=None]
331 331 if len(dirnameList)>0:
332 332 self.status = 1
333 333 self.dirnameList = dirnameList
334 334 self.dirnameList.sort()
335 335 else:
336 336 self.status = 0
337 337 return None
338 338
339 339 def __getTimeFromData(self):
340 340 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
341 341 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
342 342
343 343 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
344 344 print('........................................')
345 345 filter_filenameList = []
346 346 self.filenameList.sort()
347 347 total_files = len(self.filenameList)
348 348 #for i in range(len(self.filenameList)-1):
349 349 for i in range(total_files):
350 350 filename = self.filenameList[i]
351 351 #print("file-> ",filename)
352 352 try:
353 353 fp = h5py.File(filename,'r')
354 354 time_str = fp.get('Time/RadacTimeString')
355 355
356 356 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
357 357 #startDateTimeStr_File = "2019-12-16 09:21:11"
358 358 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
359 359 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
360 360
361 361 #endDateTimeStr_File = "2019-12-16 11:10:11"
362 362 endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
363 363 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
364 364 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
365 365
366 366 fp.close()
367 367
368 368 #print("check time", startDateTime_File)
369 369 if self.timezone == 'lt':
370 370 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
371 371 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
372 372 if (startDateTime_File >=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
373 373 filter_filenameList.append(filename)
374 374
375 375 if (startDateTime_File>endDateTime_Reader):
376 376 break
377 377 except Exception as e:
378 378 log.warning("Error opening file {} -> {}".format(os.path.split(filename)[1],e))
379 379
380 380 filter_filenameList.sort()
381 381 self.filenameList = filter_filenameList
382 382
383 383 return 1
384 384
385 385 def __filterByGlob1(self, dirName):
386 386 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
387 387 filter_files.sort()
388 388 filterDict = {}
389 389 filterDict.setdefault(dirName)
390 390 filterDict[dirName] = filter_files
391 391 return filterDict
392 392
393 393 def __getFilenameList(self, fileListInKeys, dirList):
394 394 for value in fileListInKeys:
395 395 dirName = list(value.keys())[0]
396 396 for file in value[dirName]:
397 397 filename = os.path.join(dirName, file)
398 398 self.filenameList.append(filename)
399 399
400 400
401 401 def __selectDataForTimes(self, online=False):
402 402 #aun no esta implementado el filtro for tiempo-> implementado en readNextFile
403 403 if not(self.status):
404 404 return None
405 405
406 406 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
407 407 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
408 408 self.__getFilenameList(fileListInKeys, dirList)
409 409 if not(online):
410 410 #filtro por tiempo
411 411 if not(self.all):
412 412 self.__getTimeFromData()
413 413
414 414 if len(self.filenameList)>0:
415 415 self.status = 1
416 416 self.filenameList.sort()
417 417 else:
418 418 self.status = 0
419 419 return None
420 420
421 421 else:
422 422 #get the last file - 1
423 423 self.filenameList = [self.filenameList[-2]]
424 424 new_dirnameList = []
425 425 for dirname in self.dirnameList:
426 426 junk = numpy.array([dirname in x for x in self.filenameList])
427 427 junk_sum = junk.sum()
428 428 if junk_sum > 0:
429 429 new_dirnameList.append(dirname)
430 430 self.dirnameList = new_dirnameList
431 431 return 1
432 432
433 433 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
434 434 endTime=datetime.time(23,59,59),walk=True):
435 435
436 436 if endDate ==None:
437 437 startDate = datetime.datetime.utcnow().date()
438 438 endDate = datetime.datetime.utcnow().date()
439 439
440 440 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
441 441
442 442 self.__checkPath()
443 443
444 444 self.__findDataForDates(online=True)
445 445
446 446 self.dirnameList = [self.dirnameList[-1]]
447 447
448 448 self.__selectDataForTimes(online=True)
449 449
450 450 return
451 451
452 452
453 453 def searchFilesOffLine(self,
454 454 path,
455 455 startDate,
456 456 endDate,
457 457 startTime=datetime.time(0,0,0),
458 458 endTime=datetime.time(23,59,59),
459 459 walk=True):
460 460
461 461 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
462 462
463 463 self.__checkPath()
464 464
465 465 self.__findDataForDates()
466 466
467 467 self.__selectDataForTimes()
468 468
469 469 for i in range(len(self.filenameList)):
470 470 print("%s" %(self.filenameList[i]))
471 471
472 472 return
473 473
474 474 def __setNextFileOffline(self):
475 475
476 476 try:
477 477 self.filename = self.filenameList[self.fileIndex]
478 478 self.amisrFilePointer = h5py.File(self.filename,'r')
479 479 self.fileIndex += 1
480 480 except:
481 481 self.flagNoMoreFiles = 1
482 482 raise schainpy.admin.SchainError('No more files to read')
483 483 return 0
484 484
485 485 self.flagIsNewFile = 1
486 486 print("Setting the file: %s"%self.filename)
487 487
488 488 return 1
489 489
490 490
491 491 def __setNextFileOnline(self):
492 492 filename = self.filenameList[0]
493 493 if self.__filename_online != None:
494 494 self.__selectDataForTimes(online=True)
495 495 filename = self.filenameList[0]
496 496 wait = 0
497 497 self.__waitForNewFile=300 ## DEBUG:
498 498 while self.__filename_online == filename:
499 499 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
500 500 if wait == 5:
501 501 self.flagNoMoreFiles = 1
502 502 return 0
503 503 sleep(self.__waitForNewFile)
504 504 self.__selectDataForTimes(online=True)
505 505 filename = self.filenameList[0]
506 506 wait += 1
507 507
508 508 self.__filename_online = filename
509 509
510 510 self.amisrFilePointer = h5py.File(filename,'r')
511 511 self.flagIsNewFile = 1
512 512 self.filename = filename
513 513 print("Setting the file: %s"%self.filename)
514 514 return 1
515 515
516 516
517 517 def readData(self):
518 518 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
519 519 re = buffer[:,:,:,0]
520 520 im = buffer[:,:,:,1]
521 521 dataset = re + im*1j
522 522
523 523 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
524 524 timeset = self.radacTime[:,0]
525 525
526 526 return dataset,timeset
527 527
528 528 def reshapeData(self):
529 529 #print(self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa)
530 530 channels = self.beamCodeByPulse[0,:]
531 531 nchan = self.nchannels
532 532 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
533 533 nblocks = self.nblocks
534 534 nsamples = self.nsa
535 535 #print("Channels: ",self.nChannels)
536 536 #Dimensions : nChannels, nProfiles, nSamples
537 537 new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
538 538 ############################################
539 539 profPerCH = int(self.profPerBlockRAW / (self.nFFT* self.nChannels))
540 540 #profPerCH = int(self.profPerBlockRAW / self.nChannels)
541 541 for thisChannel in range(nchan):
542 542
543 543 ich = thisChannel
544 544
545 545 idx_ch = [self.nFFT*(ich + nchan*k) for k in range(profPerCH)]
546 546 #print(idx_ch)
547 547 if self.nFFT > 1:
548 548 aux = [numpy.arange(i, i+self.nFFT) for i in idx_ch]
549 549 idx_ch = None
550 550 idx_ch =aux
551 551 idx_ch = numpy.array(idx_ch, dtype=int).flatten()
552 552 else:
553 553 idx_ch = numpy.array(idx_ch, dtype=int)
554 554
555 555 #print(ich,profPerCH,idx_ch)
556 556 #print(numpy.where(channels==self.beamCode[ich])[0])
557 557 #new_block[:,ich,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[ich])[0],:]
558 558 new_block[:,ich,:,:] = self.dataset[:,idx_ch,:]
559 559
560 560 new_block = numpy.transpose(new_block, (1,0,2,3))
561 561 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
562 562 if self.flagAsync:
563 563 new_block = numpy.roll(new_block, self.shiftChannels, axis=0)
564 564 return new_block
565 565
566 566 def updateIndexes(self):
567 567
568 568 pass
569 569
570 570 def fillJROHeader(self):
571 571
572 572 #fill radar controller header
573 573
574 574 #fill system header
575 575 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
576 576 nProfiles=self.newProfiles,
577 577 nChannels=len(self.__channelList),
578 578 adcResolution=14,
579 579 pciDioBusWidth=32)
580 580
581 581 self.dataOut.type = "Voltage"
582 582 self.dataOut.data = None
583 583 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
584 584 # self.dataOut.nChannels = 0
585 585
586 586 # self.dataOut.nHeights = 0
587 587
588 588 self.dataOut.nProfiles = self.newProfiles*self.nblocks
589 589 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
590 590 ranges = numpy.reshape(self.rangeFromFile[()],(-1))
591 591 self.dataOut.heightList = ranges/1000.0 #km
592 592 self.dataOut.channelList = self.__channelList
593 593
594 594 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
595 595
596 596 # self.dataOut.channelIndexList = None
597 597
598 598
599 #self.dataOut.azimuthList = numpy.roll( numpy.array(self.azimuthList) ,self.shiftChannels)
600 #self.dataOut.elevationList = numpy.roll(numpy.array(self.elevationList) ,self.shiftChannels)
601 #self.dataOut.codeList = numpy.roll(numpy.array(self.beamCode), self.shiftChannels)
599 # #self.dataOut.azimuthList = numpy.roll( numpy.array(self.azimuthList) ,self.shiftChannels)
600 # #self.dataOut.elevationList = numpy.roll(numpy.array(self.elevationList) ,self.shiftChannels)
601 # #self.dataOut.codeList = numpy.roll(numpy.array(self.beamCode), self.shiftChannels)
602 602
603 603 self.dataOut.azimuthList = self.azimuthList
604 604 self.dataOut.elevationList = self.elevationList
605 605 self.dataOut.codeList = self.beamCode
606 606
607 607
608 608
609 609 #print(self.dataOut.elevationList)
610 610 self.dataOut.flagNoData = True
611 611
612 612 #Set to TRUE if the data is discontinuous
613 613 self.dataOut.flagDiscontinuousBlock = False
614 614
615 615 self.dataOut.utctime = None
616 616
617 617 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
618 618 if self.timezone == 'lt':
619 619 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
620 620 else:
621 621 self.dataOut.timeZone = 0 #by default time is UTC
622 622
623 623 self.dataOut.dstFlag = 0
624 624 self.dataOut.errorCount = 0
625 625 self.dataOut.nCohInt = 1
626 626 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
627 627 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
628 628 self.dataOut.flagShiftFFT = False
629 629 self.dataOut.ippSeconds = self.ippSeconds
630 630 self.dataOut.ipp = self.__ippKm
631 631 self.dataOut.nCode = self.__nCode
632 632 self.dataOut.code = self.__code
633 633 self.dataOut.nBaud = self.__nBaud
634 634
635 635
636 636 self.dataOut.frequency = self.__frequency
637 637 self.dataOut.realtime = self.online
638 638
639 639 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
640 640 txA=self.__txAKm,
641 641 txB=0,
642 642 nWindows=1,
643 643 nHeights=self.__nSamples,
644 644 firstHeight=self.__firstHeight,
645 645 codeType=self.__codeType,
646 646 nCode=self.__nCode, nBaud=self.__nBaud,
647 647 code = self.__code,
648 648 nOsamp=self.nOsamp,
649 649 frequency = self.__frequency,
650 650 sampleRate= self.__sampleRate,
651 651 fClock=self.__sampleRate)
652 652
653 653
654 654 self.dataOut.radarControllerHeaderObj.heightList = ranges/1000.0 #km
655 655 self.dataOut.radarControllerHeaderObj.heightResolution = self.__deltaHeight
656 656 self.dataOut.radarControllerHeaderObj.rangeIpp = self.__ippKm #km
657 657 self.dataOut.radarControllerHeaderObj.rangeTxA = self.__txA*1e6*.15 #km
658 658 self.dataOut.radarControllerHeaderObj.nChannels = self.nchannels
659 659 self.dataOut.radarControllerHeaderObj.channelList = self.__channelList
660 660 self.dataOut.radarControllerHeaderObj.azimuthList = self.azimuthList
661 661 self.dataOut.radarControllerHeaderObj.elevationList = self.elevationList
662 662 self.dataOut.radarControllerHeaderObj.dtype = "Voltage"
663 663 self.dataOut.ippSeconds = self.ippSeconds
664 <<<<<<< HEAD
665 self.dataOut.ippFactor = self.nchannels*self.nFFT
666 =======
667 664 self.dataOut.ippFactor = self.nFFT
668 >>>>>>> 37cccf17c7b80521b59b978cb30e4ab2e6f37fce
669 665 pass
670 666
671 667 def readNextFile(self,online=False):
672 668
673 669 if not(online):
674 670 newFile = self.__setNextFileOffline()
675 671 else:
676 672 newFile = self.__setNextFileOnline()
677 673
678 674 if not(newFile):
679 675 self.dataOut.error = True
680 676 return 0
681 677
682 678 if not self.readAMISRHeader(self.amisrFilePointer):
683 679 self.dataOut.error = True
684 680 return 0
685 681
686 682 #self.createBuffers()
687 683 self.fillJROHeader()
688 684
689 685 #self.__firstFile = False
690 686
691 687 self.dataset,self.timeset = self.readData()
692 688
693 689 if self.endDate!=None:
694 690 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
695 691 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
696 692 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
697 693 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
698 694 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
699 695 if self.timezone == 'lt':
700 696 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
701 697 if (startDateTime_File>endDateTime_Reader):
702 698 self.flag_standby = False
703 699 return 0
704 700 if self.flag_ignoreFiles and (startDateTime_File >= self.ignStartDateTime and startDateTime_File <= self.ignEndDateTime):
705 701 print("Ignoring...")
706 702 self.flag_standby = True
707 703 return 1
708 704 self.flag_standby = False
709 705
710 706 self.jrodataset = self.reshapeData()
711 707 #----self.updateIndexes()
712 708 self.profileIndex = 0
713 709
714 710 return 1
715 711
716 712
717 713 def __hasNotDataInBuffer(self):
718 714 if self.profileIndex >= (self.newProfiles*self.nblocks):
719 715 return 1
720 716 return 0
721 717
722 718
723 719 def getData(self):
724 720
725 721 if self.flagNoMoreFiles:
726 722 self.dataOut.flagNoData = True
727 723 return 0
728 724
729 725 if self.profileIndex >= (self.newProfiles*self.nblocks): #
730 726 #if self.__hasNotDataInBuffer():
731 727 if not (self.readNextFile(self.online)):
732 728 print("Profile Index break...")
733 729 return 0
734 730
735 731 if self.flag_standby: #Standby mode, if files are being ignoring, just return with no error flag
736 732 return 0
737 733
738 734 if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
739 735 self.dataOut.flagNoData = True
740 736 print("No more data break...")
741 737 return 0
742 738
743 739 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
744 740
745 741 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
746 742
747 743 #print("R_t",self.timeset)
748 744
749 745 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
750 746 #verificar basic header de jro data y ver si es compatible con este valor
751 747 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
752 748 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
753 749 indexblock = self.profileIndex/self.newProfiles
754 750 #print (indexblock, indexprof)
755 751 diffUTC = 0
756 752 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
757 753
758 754 #print("utc :",indexblock," __ ",t_comp)
759 755 #print(numpy.shape(self.timeset))
760 756 self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
761 757 #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
762 758
763 759 self.dataOut.profileIndex = self.profileIndex
764 760 #print("N profile:",self.profileIndex,self.newProfiles,self.nblocks,self.dataOut.utctime)
765 761 self.dataOut.flagNoData = False
766 762 # if indexprof == 0:
767 763 # print("kamisr: ",self.dataOut.utctime)
768 764
769 765 self.profileIndex += 1
770 766
771 767 return self.dataOut.data #retorno necesario??
772 768
773 769
774 770 def run(self, **kwargs):
775 771 '''
776 772 This method will be called many times so here you should put all your code
777 773 '''
778 774 #print("running kamisr")
779 775 if not self.isConfig:
780 776 self.setup(**kwargs)
781 777 self.isConfig = True
782 778
783 779 self.getData()
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,2296 +1,2305
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Spectra processing Unit and operations
6 6
7 7 Here you will find the processing unit `SpectraProc` and several operations
8 8 to work with Spectra data type
9 9 """
10 10
11 11 import time
12 12 import itertools
13 13
14 14 import numpy
15 15 import math
16 16
17 17 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
18 18 from schainpy.model.data.jrodata import Spectra
19 19 from schainpy.model.data.jrodata import hildebrand_sekhon
20 20 from schainpy.model.data import _noise
21 21
22 22 from schainpy.utils import log
23 23 import matplotlib.pyplot as plt
24 24 #from scipy.optimize import curve_fit
25 25 from schainpy.model.io.utilsIO import getHei_index
26 26 import datetime
27 27
28 28 class SpectraProc(ProcessingUnit):
29 29
30 30 def __init__(self):
31 31
32 32 ProcessingUnit.__init__(self)
33 33
34 34 self.buffer = None
35 35 self.firstdatatime = None
36 36 self.profIndex = 0
37 37 self.dataOut = Spectra()
38 38 self.id_min = None
39 39 self.id_max = None
40 40 self.setupReq = False #Agregar a todas las unidades de proc
41 41 self.nsamplesFFT = 0
42 42
43 43 def __updateSpecFromVoltage(self):
44 44
45 45
46 46
47 47 self.dataOut.timeZone = self.dataIn.timeZone
48 48 self.dataOut.dstFlag = self.dataIn.dstFlag
49 49 self.dataOut.errorCount = self.dataIn.errorCount
50 50 self.dataOut.useLocalTime = self.dataIn.useLocalTime
51 51
52 52 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
53 53 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
54 54 self.dataOut.ippSeconds = self.dataIn.ippSeconds
55 55 self.dataOut.ipp = self.dataIn.ipp
56 56 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
57 57 self.dataOut.channelList = self.dataIn.channelList
58 58 self.dataOut.heightList = self.dataIn.heightList
59 59 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
60 60 self.dataOut.nProfiles = self.dataOut.nFFTPoints
61 61 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
62 62 self.dataOut.utctime = self.firstdatatime
63 63 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
64 64 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
65 65 self.dataOut.flagShiftFFT = False
66 66 self.dataOut.nCohInt = self.dataIn.nCohInt
67 67 self.dataOut.nIncohInt = 1
68 68 self.dataOut.deltaHeight = self.dataIn.deltaHeight
69 69 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
70 70 self.dataOut.frequency = self.dataIn.frequency
71 71 self.dataOut.realtime = self.dataIn.realtime
72 72 self.dataOut.azimuth = self.dataIn.azimuth
73 73 self.dataOut.zenith = self.dataIn.zenith
74 74 self.dataOut.codeList = self.dataIn.codeList
75 75 self.dataOut.azimuthList = self.dataIn.azimuthList
76 76 self.dataOut.elevationList = self.dataIn.elevationList
77 77 self.dataOut.code = self.dataIn.code
78 78 self.dataOut.nCode = self.dataIn.nCode
79 79 self.dataOut.flagProfilesByRange = self.dataIn.flagProfilesByRange
80 80 self.dataOut.nProfilesByRange = self.dataIn.nProfilesByRange
81 81
82 82
83 83 def __getFft(self):
84 84 # print("fft donw")
85 85 """
86 86 Convierte valores de Voltaje a Spectra
87 87
88 88 Affected:
89 89 self.dataOut.data_spc
90 90 self.dataOut.data_cspc
91 91 self.dataOut.data_dc
92 92 self.dataOut.heightList
93 93 self.profIndex
94 94 self.buffer
95 95 self.dataOut.flagNoData
96 96 """
97 97 fft_volt = numpy.fft.fft(
98 98 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
99 99 fft_volt = fft_volt.astype(numpy.dtype('complex'))
100 100 dc = fft_volt[:, 0, :]
101 101
102 102 # calculo de self-spectra
103 103 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
104 104 spc = fft_volt * numpy.conjugate(fft_volt)
105 105 spc = spc.real
106 106
107 107 blocksize = 0
108 108 blocksize += dc.size
109 109 blocksize += spc.size
110 110
111 111 cspc = None
112 112 pairIndex = 0
113 113 if self.dataOut.pairsList != None:
114 114 # calculo de cross-spectra
115 115 cspc = numpy.zeros(
116 116 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
117 117 for pair in self.dataOut.pairsList:
118 118 if pair[0] not in self.dataOut.channelList:
119 119 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
120 120 str(pair), str(self.dataOut.channelList)))
121 121 if pair[1] not in self.dataOut.channelList:
122 122 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
123 123 str(pair), str(self.dataOut.channelList)))
124 124
125 125 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
126 126 numpy.conjugate(fft_volt[pair[1], :, :])
127 127 pairIndex += 1
128 128 blocksize += cspc.size
129 129
130 130 self.dataOut.data_spc = spc
131 131 self.dataOut.data_cspc = cspc
132 132 self.dataOut.data_dc = dc
133 133 self.dataOut.blockSize = blocksize
134 134 self.dataOut.flagShiftFFT = False
135 135
136 136 def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False, zeroPad=False, zeroPoints=0):
137 137
138 138
139 139 try:
140 140 type = self.dataIn.type.decode("utf-8")
141 141 self.dataIn.type = type
142 142 except:
143 143 pass
144 144 if self.dataIn.type == "Spectra":
145 145
146 146 try:
147 147 self.dataOut.copy(self.dataIn)
148 148 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
149 149 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
150 150 self.dataOut.nProfiles = self.dataOut.nFFTPoints
151 151 #self.dataOut.nHeights = len(self.dataOut.heightList)
152 152 except Exception as e:
153 153 print("Error dataIn ",e)
154 154
155 155
156 156
157 157 if shift_fft:
158 158 #desplaza a la derecha en el eje 2 determinadas posiciones
159 159 shift = int(self.dataOut.nFFTPoints/2)
160 160 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
161 161
162 162 if self.dataOut.data_cspc is not None:
163 163 #desplaza a la derecha en el eje 2 determinadas posiciones
164 164 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
165 165 if pairsList:
166 166 self.__selectPairs(pairsList)
167 167
168 168
169 169 elif self.dataIn.type == "Voltage":
170 170
171 171 self.dataOut.flagNoData = True
172 172 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
173 173 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
174 174 if nFFTPoints == None:
175 175 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
176 176
177 177 if nProfiles == None:
178 178 nProfiles = nFFTPoints
179 179
180 180 #if ippFactor == None:
181 181 # self.dataOut.ippFactor = 1
182 182 if ippFactor == None:
183 183 self.dataOut.ippFactor = self.dataIn.ippFactor
184 184 else:
185 185 self.dataOut.ippFactor = ippFactor
186 186
187 187 #print(" volts ch,prof, h: ", self.dataIn.data.shape)
188 188 if self.buffer is None:
189 189 if not zeroPad:
190 190 self.buffer = numpy.zeros((self.dataIn.nChannels,
191 191 nProfiles,
192 192 self.dataIn.nHeights),
193 193 dtype='complex')
194 194 zeroPoints = 0
195 195 else:
196 196 self.buffer = numpy.zeros((self.dataIn.nChannels,
197 197 nFFTPoints+int(zeroPoints),
198 198 self.dataIn.nHeights),
199 199 dtype='complex')
200 200
201 201 self.dataOut.nFFTPoints = nFFTPoints + int(zeroPoints)
202 202
203 203 if self.dataIn.flagDataAsBlock:
204 204 nVoltProfiles = self.dataIn.data.shape[1]
205 205 zeroPoints = 0
206 206 if nVoltProfiles == nProfiles or zeroPad:
207 207 self.buffer = self.dataIn.data.copy()
208 208 self.profIndex = nVoltProfiles
209 209
210 210 elif nVoltProfiles < nProfiles:
211 211
212 212 if self.profIndex == 0:
213 213 self.id_min = 0
214 214 self.id_max = nVoltProfiles
215 215
216 216 self.buffer[:, self.id_min:self.id_max,
217 217 :] = self.dataIn.data
218 218 self.profIndex += nVoltProfiles
219 219 self.id_min += nVoltProfiles
220 220 self.id_max += nVoltProfiles
221 221 else:
222 222 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
223 223 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
224 224 self.dataOut.flagNoData = True
225 225 else:
226 226 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
227 227 self.profIndex += 1
228 228
229 229 if self.firstdatatime == None:
230 230 self.firstdatatime = self.dataIn.utctime
231 231
232 232 if self.profIndex == nProfiles or (zeroPad and zeroPoints==0):
233 233
234 234 self.__updateSpecFromVoltage()
235 235
236 236 if pairsList == None:
237 237 self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)]
238 238 else:
239 239 self.dataOut.pairsList = pairsList
240 240 self.__getFft()
241 241 self.dataOut.flagNoData = False
242 242 self.firstdatatime = None
243 243 self.nsamplesFFT = self.profIndex
244 244 self.profIndex = 0
245 245
246 246 #update Processing Header:
247 247 self.dataOut.processingHeaderObj.dtype = "Spectra"
248 248 self.dataOut.processingHeaderObj.nFFTPoints = self.dataOut.nFFTPoints
249 249 self.dataOut.processingHeaderObj.nSamplesFFT = self.nsamplesFFT
250 250 self.dataOut.processingHeaderObj.nIncohInt = 1
251 251
252 252
253 253 elif self.dataIn.type == "Parameters":
254 254
255 255 self.dataOut.data_spc = self.dataIn.data_spc
256 256 self.dataOut.data_cspc = self.dataIn.data_cspc
257 257 self.dataOut.data_outlier = self.dataIn.data_outlier
258 258 self.dataOut.nProfiles = self.dataIn.nProfiles
259 259 self.dataOut.nIncohInt = self.dataIn.nIncohInt
260 260 self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
261 261 self.dataOut.ippFactor = self.dataIn.ippFactor
262 262 self.dataOut.max_nIncohInt = self.dataIn.max_nIncohInt
263 263 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
264 264 self.dataOut.ProcessingHeader = self.dataIn.ProcessingHeader.copy()
265 265 self.dataOut.ippSeconds = self.dataIn.ippSeconds
266 266 self.dataOut.ipp = self.dataIn.ipp
267 267 #self.dataOut.abscissaList = self.dataIn.getVelRange(1)
268 268 #self.dataOut.spc_noise = self.dataIn.getNoise()
269 269 #self.dataOut.spc_range = (self.dataIn.getFreqRange(1) , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1))
270 270 # self.dataOut.normFactor = self.dataIn.normFactor
271 271 if hasattr(self.dataIn, 'channelList'):
272 272 self.dataOut.channelList = self.dataIn.channelList
273 273 if hasattr(self.dataIn, 'pairsList'):
274 274 self.dataOut.pairsList = self.dataIn.pairsList
275 275 self.dataOut.groupList = self.dataIn.pairsList
276 276
277 277 self.dataOut.flagNoData = False
278 278
279 279 if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels
280 280 self.dataOut.ChanDist = self.dataIn.ChanDist
281 281 else: self.dataOut.ChanDist = None
282 282
283 283 #if hasattr(self.dataIn, 'VelRange'): #Velocities range
284 284 # self.dataOut.VelRange = self.dataIn.VelRange
285 285 #else: self.dataOut.VelRange = None
286 286
287 287
288 288
289 289 else:
290 290 raise ValueError("The type of input object {} is not valid".format(
291 291 self.dataIn.type))
292 292
293 293
294 294
295 295
296 296 #print("spc proc Done", self.dataOut.data_spc.shape)
297 297 #print(self.dataOut.data_spc)
298 298 return
299 299
300 300 def __selectPairs(self, pairsList):
301 301
302 302 if not pairsList:
303 303 return
304 304
305 305 pairs = []
306 306 pairsIndex = []
307 307
308 308 for pair in pairsList:
309 309 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
310 310 continue
311 311 pairs.append(pair)
312 312 pairsIndex.append(pairs.index(pair))
313 313
314 314 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
315 315 self.dataOut.pairsList = pairs
316 316
317 317 return
318 318
319 319 def selectFFTs(self, minFFT, maxFFT ):
320 320 """
321 321 Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
322 322 minFFT<= FFT <= maxFFT
323 323 """
324 324
325 325 if (minFFT > maxFFT):
326 326 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
327 327
328 328 if (minFFT < self.dataOut.getFreqRange()[0]):
329 329 minFFT = self.dataOut.getFreqRange()[0]
330 330
331 331 if (maxFFT > self.dataOut.getFreqRange()[-1]):
332 332 maxFFT = self.dataOut.getFreqRange()[-1]
333 333
334 334 minIndex = 0
335 335 maxIndex = 0
336 336 FFTs = self.dataOut.getFreqRange()
337 337
338 338 inda = numpy.where(FFTs >= minFFT)
339 339 indb = numpy.where(FFTs <= maxFFT)
340 340
341 341 try:
342 342 minIndex = inda[0][0]
343 343 except:
344 344 minIndex = 0
345 345
346 346 try:
347 347 maxIndex = indb[0][-1]
348 348 except:
349 349 maxIndex = len(FFTs)
350 350
351 351 self.selectFFTsByIndex(minIndex, maxIndex)
352 352
353 353 return 1
354 354
355 355 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
356 356 newheis = numpy.where(
357 357 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
358 358
359 359 if hei_ref != None:
360 360 newheis = numpy.where(self.dataOut.heightList > hei_ref)
361 361
362 362 minIndex = min(newheis[0])
363 363 maxIndex = max(newheis[0])
364 364 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
365 365 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
366 366
367 367 # determina indices
368 368 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
369 369 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
370 370 avg_dB = 10 * \
371 371 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
372 372 beacon_dB = numpy.sort(avg_dB)[-nheis:]
373 373 beacon_heiIndexList = []
374 374 for val in avg_dB.tolist():
375 375 if val >= beacon_dB[0]:
376 376 beacon_heiIndexList.append(avg_dB.tolist().index(val))
377 377
378 378 #data_spc = data_spc[:,:,beacon_heiIndexList]
379 379 data_cspc = None
380 380 if self.dataOut.data_cspc is not None:
381 381 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
382 382 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
383 383
384 384 data_dc = None
385 385 if self.dataOut.data_dc is not None:
386 386 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
387 387 #data_dc = data_dc[:,beacon_heiIndexList]
388 388
389 389 self.dataOut.data_spc = data_spc
390 390 self.dataOut.data_cspc = data_cspc
391 391 self.dataOut.data_dc = data_dc
392 392 self.dataOut.heightList = heightList
393 393 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
394 394
395 395 return 1
396 396
397 397 def selectFFTsByIndex(self, minIndex, maxIndex):
398 398 """
399 399
400 400 """
401 401
402 402 if (minIndex < 0) or (minIndex > maxIndex):
403 403 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
404 404
405 405 if (maxIndex >= self.dataOut.nProfiles):
406 406 maxIndex = self.dataOut.nProfiles-1
407 407
408 408 #Spectra
409 409 data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
410 410
411 411 data_cspc = None
412 412 if self.dataOut.data_cspc is not None:
413 413 data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
414 414
415 415 data_dc = None
416 416 if self.dataOut.data_dc is not None:
417 417 data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
418 418
419 419 self.dataOut.data_spc = data_spc
420 420 self.dataOut.data_cspc = data_cspc
421 421 self.dataOut.data_dc = data_dc
422 422
423 423 self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
424 424 self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
425 425 self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
426 426
427 427 return 1
428 428
429 429 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
430 430 # validacion de rango
431 431 if minHei == None:
432 432 minHei = self.dataOut.heightList[0]
433 433
434 434 if maxHei == None:
435 435 maxHei = self.dataOut.heightList[-1]
436 436
437 437 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
438 438 print('minHei: %.2f is out of the heights range' % (minHei))
439 439 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
440 440 minHei = self.dataOut.heightList[0]
441 441
442 442 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
443 443 print('maxHei: %.2f is out of the heights range' % (maxHei))
444 444 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
445 445 maxHei = self.dataOut.heightList[-1]
446 446
447 447 # validacion de velocidades
448 448 velrange = self.dataOut.getVelRange(1)
449 449
450 450 if minVel == None:
451 451 minVel = velrange[0]
452 452
453 453 if maxVel == None:
454 454 maxVel = velrange[-1]
455 455
456 456 if (minVel < velrange[0]) or (minVel > maxVel):
457 457 print('minVel: %.2f is out of the velocity range' % (minVel))
458 458 print('minVel is setting to %.2f' % (velrange[0]))
459 459 minVel = velrange[0]
460 460
461 461 if (maxVel > velrange[-1]) or (maxVel < minVel):
462 462 print('maxVel: %.2f is out of the velocity range' % (maxVel))
463 463 print('maxVel is setting to %.2f' % (velrange[-1]))
464 464 maxVel = velrange[-1]
465 465
466 466 # seleccion de indices para rango
467 467 minIndex = 0
468 468 maxIndex = 0
469 469 heights = self.dataOut.heightList
470 470
471 471 inda = numpy.where(heights >= minHei)
472 472 indb = numpy.where(heights <= maxHei)
473 473
474 474 try:
475 475 minIndex = inda[0][0]
476 476 except:
477 477 minIndex = 0
478 478
479 479 try:
480 480 maxIndex = indb[0][-1]
481 481 except:
482 482 maxIndex = len(heights)
483 483
484 484 if (minIndex < 0) or (minIndex > maxIndex):
485 485 raise ValueError("some value in (%d,%d) is not valid" % (
486 486 minIndex, maxIndex))
487 487
488 488 if (maxIndex >= self.dataOut.nHeights):
489 489 maxIndex = self.dataOut.nHeights - 1
490 490
491 491 # seleccion de indices para velocidades
492 492 indminvel = numpy.where(velrange >= minVel)
493 493 indmaxvel = numpy.where(velrange <= maxVel)
494 494 try:
495 495 minIndexVel = indminvel[0][0]
496 496 except:
497 497 minIndexVel = 0
498 498
499 499 try:
500 500 maxIndexVel = indmaxvel[0][-1]
501 501 except:
502 502 maxIndexVel = len(velrange)
503 503
504 504 # seleccion del espectro
505 505 data_spc = self.dataOut.data_spc[:,
506 506 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
507 507 # estimacion de ruido
508 508 noise = numpy.zeros(self.dataOut.nChannels)
509 509
510 510 for channel in range(self.dataOut.nChannels):
511 511 daux = data_spc[channel, :, :]
512 512 sortdata = numpy.sort(daux, axis=None)
513 513 noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt)
514 514
515 515 self.dataOut.noise_estimation = noise.copy()
516 516
517 517 return 1
518 518
519 519 class removeDC(Operation):
520 520
521 521 def run(self, dataOut, mode=2):
522 522 self.dataOut = dataOut
523 523 jspectra = self.dataOut.data_spc
524 524 jcspectra = self.dataOut.data_cspc
525 525
526 526 num_chan = jspectra.shape[0]
527 527 num_hei = jspectra.shape[2]
528 528
529 529 if jcspectra is not None:
530 530 jcspectraExist = True
531 531 num_pairs = jcspectra.shape[0]
532 532 else:
533 533 jcspectraExist = False
534 534
535 535 freq_dc = int(jspectra.shape[1] / 2)
536 536 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
537 537 ind_vel = ind_vel.astype(int)
538 538
539 539 if ind_vel[0] < 0:
540 540 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
541 541
542 542 if mode == 1:
543 543 jspectra[:, freq_dc, :] = (
544 544 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
545 545
546 546 if jcspectraExist:
547 547 jcspectra[:, freq_dc, :] = (
548 548 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
549 549
550 550 if mode == 2:
551 551
552 552 vel = numpy.array([-2, -1, 1, 2])
553 553 xx = numpy.zeros([4, 4])
554 554
555 555 for fil in range(4):
556 556 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
557 557
558 558 xx_inv = numpy.linalg.inv(xx)
559 559 xx_aux = xx_inv[0, :]
560 560
561 561 for ich in range(num_chan):
562 562 yy = jspectra[ich, ind_vel, :]
563 563 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
564 564
565 565 junkid = jspectra[ich, freq_dc, :] <= 0
566 566 cjunkid = sum(junkid)
567 567
568 568 if cjunkid.any():
569 569 jspectra[ich, freq_dc, junkid.nonzero()] = (
570 570 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
571 571
572 572 if jcspectraExist:
573 573 for ip in range(num_pairs):
574 574 yy = jcspectra[ip, ind_vel, :]
575 575 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
576 576
577 577 self.dataOut.data_spc = jspectra
578 578 self.dataOut.data_cspc = jcspectra
579 579
580 580 return self.dataOut
581 581
582 582 class getNoiseB(Operation):
583 583
584 584 __slots__ =('offset','warnings', 'isConfig', 'minIndex','maxIndex','minIndexFFT','maxIndexFFT')
585 585 def __init__(self):
586 586
587 587 Operation.__init__(self)
588 588 self.isConfig = False
589 589
590 590 def setup(self, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
591 591
592 592 self.warnings = warnings
593 593 if minHei == None:
594 594 minHei = self.dataOut.heightList[0]
595 595
596 596 if maxHei == None:
597 597 maxHei = self.dataOut.heightList[-1]
598 598
599 599 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
600 600 if self.warnings:
601 601 print('minHei: %.2f is out of the heights range' % (minHei))
602 602 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
603 603 minHei = self.dataOut.heightList[0]
604 604
605 605 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
606 606 if self.warnings:
607 607 print('maxHei: %.2f is out of the heights range' % (maxHei))
608 608 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
609 609 maxHei = self.dataOut.heightList[-1]
610 610
611 611
612 612 #indices relativos a los puntos de fft, puede ser de acuerdo a velocidad o frecuencia
613 613 minIndexFFT = 0
614 614 maxIndexFFT = 0
615 615 # validacion de velocidades
616 616 indminPoint = None
617 617 indmaxPoint = None
618 618 if self.dataOut.type == 'Spectra':
619 619 if minVel == None and maxVel == None :
620 620
621 621 freqrange = self.dataOut.getFreqRange(1)
622 622
623 623 if minFreq == None:
624 624 minFreq = freqrange[0]
625 625
626 626 if maxFreq == None:
627 627 maxFreq = freqrange[-1]
628 628
629 629 if (minFreq < freqrange[0]) or (minFreq > maxFreq):
630 630 if self.warnings:
631 631 print('minFreq: %.2f is out of the frequency range' % (minFreq))
632 632 print('minFreq is setting to %.2f' % (freqrange[0]))
633 633 minFreq = freqrange[0]
634 634
635 635 if (maxFreq > freqrange[-1]) or (maxFreq < minFreq):
636 636 if self.warnings:
637 637 print('maxFreq: %.2f is out of the frequency range' % (maxFreq))
638 638 print('maxFreq is setting to %.2f' % (freqrange[-1]))
639 639 maxFreq = freqrange[-1]
640 640
641 641 indminPoint = numpy.where(freqrange >= minFreq)
642 642 indmaxPoint = numpy.where(freqrange <= maxFreq)
643 643
644 644 else:
645 645
646 646 velrange = self.dataOut.getVelRange(1)
647 647
648 648 if minVel == None:
649 649 minVel = velrange[0]
650 650
651 651 if maxVel == None:
652 652 maxVel = velrange[-1]
653 653
654 654 if (minVel < velrange[0]) or (minVel > maxVel):
655 655 if self.warnings:
656 656 print('minVel: %.2f is out of the velocity range' % (minVel))
657 657 print('minVel is setting to %.2f' % (velrange[0]))
658 658 minVel = velrange[0]
659 659
660 660 if (maxVel > velrange[-1]) or (maxVel < minVel):
661 661 if self.warnings:
662 662 print('maxVel: %.2f is out of the velocity range' % (maxVel))
663 663 print('maxVel is setting to %.2f' % (velrange[-1]))
664 664 maxVel = velrange[-1]
665 665
666 666 indminPoint = numpy.where(velrange >= minVel)
667 667 indmaxPoint = numpy.where(velrange <= maxVel)
668 668
669 669
670 670 # seleccion de indices para rango
671 671 minIndex = 0
672 672 maxIndex = 0
673 673 heights = self.dataOut.heightList
674 674
675 675 inda = numpy.where(heights >= minHei)
676 676 indb = numpy.where(heights <= maxHei)
677 677
678 678 try:
679 679 minIndex = inda[0][0]
680 680 except:
681 681 minIndex = 0
682 682
683 683 try:
684 684 maxIndex = indb[0][-1]
685 685 except:
686 686 maxIndex = len(heights)
687 687
688 688 if (minIndex < 0) or (minIndex > maxIndex):
689 689 raise ValueError("some value in (%d,%d) is not valid" % (
690 690 minIndex, maxIndex))
691 691
692 692 if (maxIndex >= self.dataOut.nHeights):
693 693 maxIndex = self.dataOut.nHeights - 1
694 694 #############################################################3
695 695 # seleccion de indices para velocidades
696 696 if self.dataOut.type == 'Spectra':
697 697 try:
698 698 minIndexFFT = indminPoint[0][0]
699 699 except:
700 700 minIndexFFT = 0
701 701
702 702 try:
703 703 maxIndexFFT = indmaxPoint[0][-1]
704 704 except:
705 705 maxIndexFFT = len( self.dataOut.getFreqRange(1))
706 706
707 707 self.minIndex, self.maxIndex, self.minIndexFFT, self.maxIndexFFT = minIndex, maxIndex, minIndexFFT, maxIndexFFT
708 708 self.isConfig = True
709 709 self.offset = 1
710 710 if offset!=None:
711 711 self.offset = 10**(offset/10)
712 712 #print("config getNoiseB Done")
713 713
714 714 def run(self, dataOut, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
715 715 self.dataOut = dataOut
716 716
717 717 if not self.isConfig:
718 718 self.setup(offset, minHei, maxHei,minVel, maxVel, minFreq, maxFreq, warnings)
719 719
720 720 self.dataOut.noise_estimation = None
721 721 noise = None
722 722 #print("data type: ",self.dataOut.type, self.dataOut.nIncohInt, self.dataOut.max_nIncohInt)
723 723 if self.dataOut.type == 'Voltage':
724 724 noise = self.dataOut.getNoise(ymin_index=self.minIndex, ymax_index=self.maxIndex)
725 725 #print(minIndex, maxIndex,minIndexVel, maxIndexVel)
726 726 elif self.dataOut.type == 'Spectra':
727 727 #print(self.dataOut.nChannels, self.minIndex, self.maxIndex,self.minIndexFFT, self.maxIndexFFT, self.dataOut.max_nIncohInt, self.dataOut.nIncohInt)
728 728 noise = numpy.zeros( self.dataOut.nChannels)
729 729 norm = 1
730 730
731 731 for channel in range( self.dataOut.nChannels):
732 732 if not hasattr(self.dataOut.nIncohInt,'__len__'):
733 733 norm = 1
734 734 else:
735 735 norm = self.dataOut.max_nIncohInt[channel]/self.dataOut.nIncohInt[channel, self.minIndex:self.maxIndex]
736 736
737 737 #print("norm nIncoh: ", norm ,self.dataOut.data_spc.shape, self.dataOut.max_nIncohInt)
738 738 daux = self.dataOut.data_spc[channel,self.minIndexFFT:self.maxIndexFFT, self.minIndex:self.maxIndex]
739 739 daux = numpy.multiply(daux, norm)
740 740 #print("offset: ", self.offset, 10*numpy.log10(self.offset))
741 741 # noise[channel] = self.getNoiseByMean(daux)/self.offset
742 742 #print(daux.shape, daux)
743 743 #noise[channel] = self.getNoiseByHS(daux, self.dataOut.max_nIncohInt)/self.offset
744 744 sortdata = numpy.sort(daux, axis=None)
745 745
746 746 noise[channel] = _noise.hildebrand_sekhon(sortdata, self.dataOut.max_nIncohInt[channel])/self.offset
747 747 #print("noise shape", noise[channel], self.name)
748 748
749 749 #noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex)
750 750 else:
751 751 noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex)
752 752
753 753 self.dataOut.noise_estimation = noise.copy() # dataOut.noise
754 754 #print("2: ",10*numpy.log10(self.dataOut.noise_estimation/64))
755 755 #print("2: ",self.dataOut.noise_estimation)
756 756 #print(self.dataOut.flagNoData)
757 757 #print("getNoise Done", 10*numpy.log10(noise))
758 758 return self.dataOut
759 759
760 760 def getNoiseByMean(self,data):
761 761 #data debe estar ordenado
762 762 data = numpy.mean(data,axis=1)
763 763 sortdata = numpy.sort(data, axis=None)
764 764 #sortID=data.argsort()
765 765 #print(data.shape)
766 766
767 767 pnoise = None
768 768 j = 0
769 769
770 770 mean = numpy.mean(sortdata)
771 771 min = numpy.min(sortdata)
772 772 delta = mean - min
773 773 indexes = numpy.where(sortdata > (mean+delta))[0] #only array of indexes
774 774 #print(len(indexes))
775 775 if len(indexes)==0:
776 776 pnoise = numpy.mean(sortdata)
777 777 else:
778 778 j = indexes[0]
779 779 pnoise = numpy.mean(sortdata[0:j])
780 780
781 781 # from matplotlib import pyplot as plt
782 782 # plt.plot(sortdata)
783 783 # plt.vlines(j,(pnoise-delta),(pnoise+delta), color='r')
784 784 # plt.show()
785 785 #print("noise: ", 10*numpy.log10(pnoise))
786 786 return pnoise
787 787
788 788 def getNoiseByHS(self,data, navg):
789 789 #data debe estar ordenado
790 790 #data = numpy.mean(data,axis=1)
791 791 sortdata = numpy.sort(data, axis=None)
792 792
793 793 lenOfData = len(sortdata)
794 794 nums_min = lenOfData*0.2
795 795
796 796 if nums_min <= 5:
797 797
798 798 nums_min = 5
799 799
800 800 sump = 0.
801 801 sumq = 0.
802 802
803 803 j = 0
804 804 cont = 1
805 805
806 806 while((cont == 1)and(j < lenOfData)):
807 807
808 808 sump += sortdata[j]
809 809 sumq += sortdata[j]**2
810 810 #sumq -= sump**2
811 811 if j > nums_min:
812 812 rtest = float(j)/(j-1) + 1.0/navg
813 813 #if ((sumq*j) > (sump**2)):
814 814 if ((sumq*j) > (rtest*sump**2)):
815 815 j = j - 1
816 816 sump = sump - sortdata[j]
817 817 sumq = sumq - sortdata[j]**2
818 818 cont = 0
819 819
820 820 j += 1
821 821
822 822 lnoise = sump / j
823 823
824 824 return lnoise
825 825
826 826
827 827
828 828 def fit_func( x, a0, a1, a2): #, a3, a4, a5):
829 829 z = (x - a1) / a2
830 830 y = a0 * numpy.exp(-z**2 / a2) #+ a3 + a4 * x + a5 * x**2
831 831 return y
832 832
833 833
834 834 # class CleanRayleigh(Operation):
835 835 #
836 836 # def __init__(self):
837 837 #
838 838 # Operation.__init__(self)
839 839 # self.i=0
840 840 # self.isConfig = False
841 841 # self.__dataReady = False
842 842 # self.__profIndex = 0
843 843 # self.byTime = False
844 844 # self.byProfiles = False
845 845 #
846 846 # self.bloques = None
847 847 # self.bloque0 = None
848 848 #
849 849 # self.index = 0
850 850 #
851 851 # self.buffer = 0
852 852 # self.buffer2 = 0
853 853 # self.buffer3 = 0
854 854 #
855 855 #
856 856 # def setup(self,dataOut,min_hei,max_hei,n, timeInterval,factor_stdv):
857 857 #
858 858 # self.nChannels = dataOut.nChannels
859 859 # self.nProf = dataOut.nProfiles
860 860 # self.nPairs = dataOut.data_cspc.shape[0]
861 861 # self.pairsArray = numpy.array(dataOut.pairsList)
862 862 # self.spectra = dataOut.data_spc
863 863 # self.cspectra = dataOut.data_cspc
864 864 # self.heights = dataOut.heightList #alturas totales
865 865 # self.nHeights = len(self.heights)
866 866 # self.min_hei = min_hei
867 867 # self.max_hei = max_hei
868 868 # if (self.min_hei == None):
869 869 # self.min_hei = 0
870 870 # if (self.max_hei == None):
871 871 # self.max_hei = dataOut.heightList[-1]
872 872 # self.hval = ((self.max_hei>=self.heights) & (self.heights >= self.min_hei)).nonzero()
873 873 # self.heightsClean = self.heights[self.hval] #alturas filtradas
874 874 # self.hval = self.hval[0] # forma (N,), an solo N elementos -> Indices de alturas
875 875 # self.nHeightsClean = len(self.heightsClean)
876 876 # self.channels = dataOut.channelList
877 877 # self.nChan = len(self.channels)
878 878 # self.nIncohInt = dataOut.nIncohInt
879 879 # self.__initime = dataOut.utctime
880 880 # self.maxAltInd = self.hval[-1]+1
881 881 # self.minAltInd = self.hval[0]
882 882 #
883 883 # self.crosspairs = dataOut.pairsList
884 884 # self.nPairs = len(self.crosspairs)
885 885 # self.normFactor = dataOut.normFactor
886 886 # self.nFFTPoints = dataOut.nFFTPoints
887 887 # self.ippSeconds = dataOut.ippSeconds
888 888 # self.currentTime = self.__initime
889 889 # self.pairsArray = numpy.array(dataOut.pairsList)
890 890 # self.factor_stdv = factor_stdv
891 891 #
892 892 # if n != None :
893 893 # self.byProfiles = True
894 894 # self.nIntProfiles = n
895 895 # else:
896 896 # self.__integrationtime = timeInterval
897 897 #
898 898 # self.__dataReady = False
899 899 # self.isConfig = True
900 900 #
901 901 #
902 902 #
903 903 # def run(self, dataOut,min_hei=None,max_hei=None, n=None, timeInterval=10,factor_stdv=2.5):
904 904 # #print("runing cleanRayleigh")
905 905 # if not self.isConfig :
906 906 #
907 907 # self.setup(dataOut, min_hei,max_hei,n,timeInterval,factor_stdv)
908 908 #
909 909 # tini=dataOut.utctime
910 910 #
911 911 # if self.byProfiles:
912 912 # if self.__profIndex == self.nIntProfiles:
913 913 # self.__dataReady = True
914 914 # else:
915 915 # if (tini - self.__initime) >= self.__integrationtime:
916 916 #
917 917 # self.__dataReady = True
918 918 # self.__initime = tini
919 919 #
920 920 # #if (tini.tm_min % 2) == 0 and (tini.tm_sec < 5 and self.fint==0):
921 921 #
922 922 # if self.__dataReady:
923 923 #
924 924 # self.__profIndex = 0
925 925 # jspc = self.buffer
926 926 # jcspc = self.buffer2
927 927 # #jnoise = self.buffer3
928 928 # self.buffer = dataOut.data_spc
929 929 # self.buffer2 = dataOut.data_cspc
930 930 # #self.buffer3 = dataOut.noise
931 931 # self.currentTime = dataOut.utctime
932 932 # if numpy.any(jspc) :
933 933 # #print( jspc.shape, jcspc.shape)
934 934 # jspc = numpy.reshape(jspc,(int(len(jspc)/self.nChannels),self.nChannels,self.nFFTPoints,self.nHeights))
935 935 # try:
936 936 # jcspc= numpy.reshape(jcspc,(int(len(jcspc)/self.nPairs),self.nPairs,self.nFFTPoints,self.nHeights))
937 937 # except:
938 938 # print("no cspc")
939 939 # self.__dataReady = False
940 940 # #print( jspc.shape, jcspc.shape)
941 941 # dataOut.flagNoData = False
942 942 # else:
943 943 # dataOut.flagNoData = True
944 944 # self.__dataReady = False
945 945 # return dataOut
946 946 # else:
947 947 # #print( len(self.buffer))
948 948 # if numpy.any(self.buffer):
949 949 # self.buffer = numpy.concatenate((self.buffer,dataOut.data_spc), axis=0)
950 950 # try:
951 951 # self.buffer2 = numpy.concatenate((self.buffer2,dataOut.data_cspc), axis=0)
952 952 # self.buffer3 += dataOut.data_dc
953 953 # except:
954 954 # pass
955 955 # else:
956 956 # self.buffer = dataOut.data_spc
957 957 # self.buffer2 = dataOut.data_cspc
958 958 # self.buffer3 = dataOut.data_dc
959 959 # #print self.index, self.fint
960 960 # #print self.buffer2.shape
961 961 # dataOut.flagNoData = True ## NOTE: ?? revisar LUEGO
962 962 # self.__profIndex += 1
963 963 # return dataOut ## NOTE: REV
964 964 #
965 965 #
966 966 # #index = tini.tm_hour*12+tini.tm_min/5
967 967 # '''
968 968 # #REVISAR
969 969 # '''
970 970 # # jspc = jspc/self.nFFTPoints/self.normFactor
971 971 # # jcspc = jcspc/self.nFFTPoints/self.normFactor
972 972 #
973 973 #
974 974 #
975 975 # tmp_spectra,tmp_cspectra = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
976 976 # dataOut.data_spc = tmp_spectra
977 977 # dataOut.data_cspc = tmp_cspectra
978 978 #
979 979 # #dataOut.data_spc,dataOut.data_cspc = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
980 980 #
981 981 # dataOut.data_dc = self.buffer3
982 982 # dataOut.nIncohInt *= self.nIntProfiles
983 983 # dataOut.max_nIncohInt = self.nIntProfiles
984 984 # dataOut.utctime = self.currentTime #tiempo promediado
985 985 # #print("Time: ",time.localtime(dataOut.utctime))
986 986 # # dataOut.data_spc = sat_spectra
987 987 # # dataOut.data_cspc = sat_cspectra
988 988 # self.buffer = 0
989 989 # self.buffer2 = 0
990 990 # self.buffer3 = 0
991 991 #
992 992 # return dataOut
993 993 #
994 994 # def cleanRayleigh(self,dataOut,spectra,cspectra,factor_stdv):
995 995 # print("OP cleanRayleigh")
996 996 # #import matplotlib.pyplot as plt
997 997 # #for k in range(149):
998 998 # #channelsProcssd = []
999 999 # #channelA_ok = False
1000 1000 # #rfunc = cspectra.copy() #self.bloques
1001 1001 # rfunc = spectra.copy()
1002 1002 # #rfunc = cspectra
1003 1003 # #val_spc = spectra*0.0 #self.bloque0*0.0
1004 1004 # #val_cspc = cspectra*0.0 #self.bloques*0.0
1005 1005 # #in_sat_spectra = spectra.copy() #self.bloque0
1006 1006 # #in_sat_cspectra = cspectra.copy() #self.bloques
1007 1007 #
1008 1008 #
1009 1009 # ###ONLY FOR TEST:
1010 1010 # raxs = math.ceil(math.sqrt(self.nPairs))
1011 1011 # if raxs == 0:
1012 1012 # raxs = 1
1013 1013 # caxs = math.ceil(self.nPairs/raxs)
1014 1014 # if self.nPairs <4:
1015 1015 # raxs = 2
1016 1016 # caxs = 2
1017 1017 # #print(raxs, caxs)
1018 1018 # fft_rev = 14 #nFFT to plot
1019 1019 # hei_rev = ((self.heights >= 550) & (self.heights <= 551)).nonzero() #hei to plot
1020 1020 # hei_rev = hei_rev[0]
1021 1021 # #print(hei_rev)
1022 1022 #
1023 1023 # #print numpy.absolute(rfunc[:,0,0,14])
1024 1024 #
1025 1025 # gauss_fit, covariance = None, None
1026 1026 # for ih in range(self.minAltInd,self.maxAltInd):
1027 1027 # for ifreq in range(self.nFFTPoints):
1028 1028 # '''
1029 1029 # ###ONLY FOR TEST:
1030 1030 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1031 1031 # fig, axs = plt.subplots(raxs, caxs)
1032 1032 # fig2, axs2 = plt.subplots(raxs, caxs)
1033 1033 # col_ax = 0
1034 1034 # row_ax = 0
1035 1035 # '''
1036 1036 # #print(self.nPairs)
1037 1037 # for ii in range(self.nChan): #PARES DE CANALES SELF y CROSS
1038 1038 # # if self.crosspairs[ii][1]-self.crosspairs[ii][0] > 1: # APLICAR SOLO EN PARES CONTIGUOS
1039 1039 # # continue
1040 1040 # # if not self.crosspairs[ii][0] in channelsProcssd:
1041 1041 # # channelA_ok = True
1042 1042 # #print("pair: ",self.crosspairs[ii])
1043 1043 # '''
1044 1044 # ###ONLY FOR TEST:
1045 1045 # if (col_ax%caxs==0 and col_ax!=0 and self.nPairs !=1):
1046 1046 # col_ax = 0
1047 1047 # row_ax += 1
1048 1048 # '''
1049 1049 # func2clean = 10*numpy.log10(numpy.absolute(rfunc[:,ii,ifreq,ih])) #Potencia?
1050 1050 # #print(func2clean.shape)
1051 1051 # val = (numpy.isfinite(func2clean)==True).nonzero()
1052 1052 #
1053 1053 # if len(val)>0: #limitador
1054 1054 # min_val = numpy.around(numpy.amin(func2clean)-2) #> (-40)
1055 1055 # if min_val <= -40 :
1056 1056 # min_val = -40
1057 1057 # max_val = numpy.around(numpy.amax(func2clean)+2) #< 200
1058 1058 # if max_val >= 200 :
1059 1059 # max_val = 200
1060 1060 # #print min_val, max_val
1061 1061 # step = 1
1062 1062 # #print("Getting bins and the histogram")
1063 1063 # x_dist = min_val + numpy.arange(1 + ((max_val-(min_val))/step))*step
1064 1064 # y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
1065 1065 # #print(len(y_dist),len(binstep[:-1]))
1066 1066 # #print(row_ax,col_ax, " ..")
1067 1067 # #print(self.pairsArray[ii][0],self.pairsArray[ii][1])
1068 1068 # mean = numpy.sum(x_dist * y_dist) / numpy.sum(y_dist)
1069 1069 # sigma = numpy.sqrt(numpy.sum(y_dist * (x_dist - mean)**2) / numpy.sum(y_dist))
1070 1070 # parg = [numpy.amax(y_dist),mean,sigma]
1071 1071 #
1072 1072 # newY = None
1073 1073 #
1074 1074 # try :
1075 1075 # gauss_fit, covariance = curve_fit(fit_func, x_dist, y_dist,p0=parg)
1076 1076 # mode = gauss_fit[1]
1077 1077 # stdv = gauss_fit[2]
1078 1078 # #print(" FIT OK",gauss_fit)
1079 1079 # '''
1080 1080 # ###ONLY FOR TEST:
1081 1081 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1082 1082 # newY = fit_func(x_dist,gauss_fit[0],gauss_fit[1],gauss_fit[2])
1083 1083 # axs[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
1084 1084 # axs[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
1085 1085 # axs[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
1086 1086 # '''
1087 1087 # except:
1088 1088 # mode = mean
1089 1089 # stdv = sigma
1090 1090 # #print("FIT FAIL")
1091 1091 # #continue
1092 1092 #
1093 1093 #
1094 1094 # #print(mode,stdv)
1095 1095 # #Removing echoes greater than mode + std_factor*stdv
1096 1096 # noval = (abs(func2clean - mode)>=(factor_stdv*stdv)).nonzero()
1097 1097 # #noval tiene los indices que se van a remover
1098 1098 # #print("Chan ",ii," novals: ",len(noval[0]))
1099 1099 # if len(noval[0]) > 0: #forma de array (N,) es igual a longitud (N)
1100 1100 # novall = ((func2clean - mode) >= (factor_stdv*stdv)).nonzero()
1101 1101 # #print(novall)
1102 1102 # #print(" ",self.pairsArray[ii])
1103 1103 # #cross_pairs = self.pairsArray[ii]
1104 1104 # #Getting coherent echoes which are removed.
1105 1105 # # if len(novall[0]) > 0:
1106 1106 # #
1107 1107 # # val_spc[novall[0],cross_pairs[0],ifreq,ih] = 1
1108 1108 # # val_spc[novall[0],cross_pairs[1],ifreq,ih] = 1
1109 1109 # # val_cspc[novall[0],ii,ifreq,ih] = 1
1110 1110 # #print("OUT NOVALL 1")
1111 1111 # try:
1112 1112 # pair = (self.channels[ii],self.channels[ii + 1])
1113 1113 # except:
1114 1114 # pair = (99,99)
1115 1115 # #print("par ", pair)
1116 1116 # if ( pair in self.crosspairs):
1117 1117 # q = self.crosspairs.index(pair)
1118 1118 # #print("está aqui: ", q, (ii,ii + 1))
1119 1119 # new_a = numpy.delete(cspectra[:,q,ifreq,ih], noval[0])
1120 1120 # cspectra[noval,q,ifreq,ih] = numpy.mean(new_a) #mean CrossSpectra
1121 1121 #
1122 1122 # #if channelA_ok:
1123 1123 # #chA = self.channels.index(cross_pairs[0])
1124 1124 # new_b = numpy.delete(spectra[:,ii,ifreq,ih], noval[0])
1125 1125 # spectra[noval,ii,ifreq,ih] = numpy.mean(new_b) #mean Spectra Pair A
1126 1126 # #channelA_ok = False
1127 1127 #
1128 1128 # # chB = self.channels.index(cross_pairs[1])
1129 1129 # # new_c = numpy.delete(spectra[:,chB,ifreq,ih], noval[0])
1130 1130 # # spectra[noval,chB,ifreq,ih] = numpy.mean(new_c) #mean Spectra Pair B
1131 1131 # #
1132 1132 # # channelsProcssd.append(self.crosspairs[ii][0]) # save channel A
1133 1133 # # channelsProcssd.append(self.crosspairs[ii][1]) # save channel B
1134 1134 # '''
1135 1135 # ###ONLY FOR TEST:
1136 1136 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1137 1137 # func2clean = 10*numpy.log10(numpy.absolute(spectra[:,ii,ifreq,ih]))
1138 1138 # y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
1139 1139 # axs2[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
1140 1140 # axs2[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
1141 1141 # axs2[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
1142 1142 # '''
1143 1143 # '''
1144 1144 # ###ONLY FOR TEST:
1145 1145 # col_ax += 1 #contador de ploteo columnas
1146 1146 # ##print(col_ax)
1147 1147 # ###ONLY FOR TEST:
1148 1148 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1149 1149 # title = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km"
1150 1150 # title2 = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km CLEANED"
1151 1151 # fig.suptitle(title)
1152 1152 # fig2.suptitle(title2)
1153 1153 # plt.show()
1154 1154 # '''
1155 1155 # ##################################################################################################
1156 1156 #
1157 1157 # #print("Getting average of the spectra and cross-spectra from incoherent echoes.")
1158 1158 # out_spectra = numpy.zeros([self.nChan,self.nFFTPoints,self.nHeights], dtype=float) #+numpy.nan
1159 1159 # out_cspectra = numpy.zeros([self.nPairs,self.nFFTPoints,self.nHeights], dtype=complex) #+numpy.nan
1160 1160 # for ih in range(self.nHeights):
1161 1161 # for ifreq in range(self.nFFTPoints):
1162 1162 # for ich in range(self.nChan):
1163 1163 # tmp = spectra[:,ich,ifreq,ih]
1164 1164 # valid = (numpy.isfinite(tmp[:])==True).nonzero()
1165 1165 #
1166 1166 # if len(valid[0]) >0 :
1167 1167 # out_spectra[ich,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
1168 1168 #
1169 1169 # for icr in range(self.nPairs):
1170 1170 # tmp = numpy.squeeze(cspectra[:,icr,ifreq,ih])
1171 1171 # valid = (numpy.isfinite(tmp)==True).nonzero()
1172 1172 # if len(valid[0]) > 0:
1173 1173 # out_cspectra[icr,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
1174 1174 #
1175 1175 # return out_spectra, out_cspectra
1176 1176 #
1177 1177 # def REM_ISOLATED_POINTS(self,array,rth):
1178 1178 # # import matplotlib.pyplot as plt
1179 1179 # if rth == None :
1180 1180 # rth = 4
1181 1181 # #print("REM ISO")
1182 1182 # num_prof = len(array[0,:,0])
1183 1183 # num_hei = len(array[0,0,:])
1184 1184 # n2d = len(array[:,0,0])
1185 1185 #
1186 1186 # for ii in range(n2d) :
1187 1187 # #print ii,n2d
1188 1188 # tmp = array[ii,:,:]
1189 1189 # #print tmp.shape, array[ii,101,:],array[ii,102,:]
1190 1190 #
1191 1191 # # fig = plt.figure(figsize=(6,5))
1192 1192 # # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1193 1193 # # ax = fig.add_axes([left, bottom, width, height])
1194 1194 # # x = range(num_prof)
1195 1195 # # y = range(num_hei)
1196 1196 # # cp = ax.contour(y,x,tmp)
1197 1197 # # ax.clabel(cp, inline=True,fontsize=10)
1198 1198 # # plt.show()
1199 1199 #
1200 1200 # #indxs = WHERE(FINITE(tmp) AND tmp GT 0,cindxs)
1201 1201 # tmp = numpy.reshape(tmp,num_prof*num_hei)
1202 1202 # indxs1 = (numpy.isfinite(tmp)==True).nonzero()
1203 1203 # indxs2 = (tmp > 0).nonzero()
1204 1204 #
1205 1205 # indxs1 = (indxs1[0])
1206 1206 # indxs2 = indxs2[0]
1207 1207 # #indxs1 = numpy.array(indxs1[0])
1208 1208 # #indxs2 = numpy.array(indxs2[0])
1209 1209 # indxs = None
1210 1210 # #print indxs1 , indxs2
1211 1211 # for iv in range(len(indxs2)):
1212 1212 # indv = numpy.array((indxs1 == indxs2[iv]).nonzero())
1213 1213 # #print len(indxs2), indv
1214 1214 # if len(indv[0]) > 0 :
1215 1215 # indxs = numpy.concatenate((indxs,indxs2[iv]), axis=None)
1216 1216 # # print indxs
1217 1217 # indxs = indxs[1:]
1218 1218 # #print(indxs, len(indxs))
1219 1219 # if len(indxs) < 4 :
1220 1220 # array[ii,:,:] = 0.
1221 1221 # return
1222 1222 #
1223 1223 # xpos = numpy.mod(indxs ,num_hei)
1224 1224 # ypos = (indxs / num_hei)
1225 1225 # sx = numpy.argsort(xpos) # Ordering respect to "x" (time)
1226 1226 # #print sx
1227 1227 # xpos = xpos[sx]
1228 1228 # ypos = ypos[sx]
1229 1229 #
1230 1230 # # *********************************** Cleaning isolated points **********************************
1231 1231 # ic = 0
1232 1232 # while True :
1233 1233 # r = numpy.sqrt(list(numpy.power((xpos[ic]-xpos),2)+ numpy.power((ypos[ic]-ypos),2)))
1234 1234 # #no_coh = WHERE(FINITE(r) AND (r LE rth),cno_coh)
1235 1235 # #plt.plot(r)
1236 1236 # #plt.show()
1237 1237 # no_coh1 = (numpy.isfinite(r)==True).nonzero()
1238 1238 # no_coh2 = (r <= rth).nonzero()
1239 1239 # #print r, no_coh1, no_coh2
1240 1240 # no_coh1 = numpy.array(no_coh1[0])
1241 1241 # no_coh2 = numpy.array(no_coh2[0])
1242 1242 # no_coh = None
1243 1243 # #print valid1 , valid2
1244 1244 # for iv in range(len(no_coh2)):
1245 1245 # indv = numpy.array((no_coh1 == no_coh2[iv]).nonzero())
1246 1246 # if len(indv[0]) > 0 :
1247 1247 # no_coh = numpy.concatenate((no_coh,no_coh2[iv]), axis=None)
1248 1248 # no_coh = no_coh[1:]
1249 1249 # #print len(no_coh), no_coh
1250 1250 # if len(no_coh) < 4 :
1251 1251 # #print xpos[ic], ypos[ic], ic
1252 1252 # # plt.plot(r)
1253 1253 # # plt.show()
1254 1254 # xpos[ic] = numpy.nan
1255 1255 # ypos[ic] = numpy.nan
1256 1256 #
1257 1257 # ic = ic + 1
1258 1258 # if (ic == len(indxs)) :
1259 1259 # break
1260 1260 # #print( xpos, ypos)
1261 1261 #
1262 1262 # indxs = (numpy.isfinite(list(xpos))==True).nonzero()
1263 1263 # #print indxs[0]
1264 1264 # if len(indxs[0]) < 4 :
1265 1265 # array[ii,:,:] = 0.
1266 1266 # return
1267 1267 #
1268 1268 # xpos = xpos[indxs[0]]
1269 1269 # ypos = ypos[indxs[0]]
1270 1270 # for i in range(0,len(ypos)):
1271 1271 # ypos[i]=int(ypos[i])
1272 1272 # junk = tmp
1273 1273 # tmp = junk*0.0
1274 1274 #
1275 1275 # tmp[list(xpos + (ypos*num_hei))] = junk[list(xpos + (ypos*num_hei))]
1276 1276 # array[ii,:,:] = numpy.reshape(tmp,(num_prof,num_hei))
1277 1277 #
1278 1278 # #print array.shape
1279 1279 # #tmp = numpy.reshape(tmp,(num_prof,num_hei))
1280 1280 # #print tmp.shape
1281 1281 #
1282 1282 # # fig = plt.figure(figsize=(6,5))
1283 1283 # # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1284 1284 # # ax = fig.add_axes([left, bottom, width, height])
1285 1285 # # x = range(num_prof)
1286 1286 # # y = range(num_hei)
1287 1287 # # cp = ax.contour(y,x,array[ii,:,:])
1288 1288 # # ax.clabel(cp, inline=True,fontsize=10)
1289 1289 # # plt.show()
1290 1290 # return array
1291 1291 #
1292 1292
1293 1293 class IntegrationFaradaySpectra(Operation):
1294 1294
1295 1295 __profIndex = 0
1296 1296 __withOverapping = False
1297 1297
1298 1298 __byTime = False
1299 1299 __initime = None
1300 1300 __lastdatatime = None
1301 1301 __integrationtime = None
1302 1302
1303 1303 __buffer_spc = None
1304 1304 __buffer_cspc = None
1305 1305 __buffer_dc = None
1306 1306
1307 1307 __dataReady = False
1308 1308
1309 1309 __timeInterval = None
1310 1310 n_ints = None #matriz de numero de integracions (CH,HEI)
1311 1311 n = None
1312 1312 minHei_ind = None
1313 1313 maxHei_ind = None
1314 1314 navg = 1.0
1315 1315 factor = 0.0
1316 1316 dataoutliers = None # (CHANNELS, HEIGHTS)
1317 1317
1318 1318 _flagProfilesByRange = False
1319 1319 _nProfilesByRange = 0
1320 1320
1321 1321 def __init__(self):
1322 1322
1323 1323 Operation.__init__(self)
1324 1324
1325 1325 def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None, minHei=None, maxHei=None, avg=1,factor=0.75):
1326 1326 """
1327 1327 Set the parameters of the integration class.
1328 1328
1329 1329 Inputs:
1330 1330
1331 1331 n : Number of coherent integrations
1332 1332 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1333 1333 overlapping :
1334 1334
1335 1335 """
1336 1336
1337 1337 self.__initime = None
1338 1338 self.__lastdatatime = 0
1339 1339
1340 1340 self.__buffer_spc = []
1341 1341 self.__buffer_cspc = []
1342 1342 self.__buffer_dc = 0
1343 1343
1344 1344 self.__profIndex = 0
1345 1345 self.__dataReady = False
1346 1346 self.__byTime = False
1347 1347
1348 1348 self.factor = factor
1349 1349 self.navg = avg
1350 1350 #self.ByLags = dataOut.ByLags ###REDEFINIR
1351 1351 self.ByLags = False
1352 1352 self.maxProfilesInt = 0
1353 1353 self.__nChannels = dataOut.nChannels
1354 1354 if DPL != None:
1355 1355 self.DPL=DPL
1356 1356 else:
1357 1357 #self.DPL=dataOut.DPL ###REDEFINIR
1358 1358 self.DPL=0
1359 1359
1360 1360 if n is None and timeInterval is None:
1361 1361 raise ValueError("n or timeInterval should be specified ...")
1362 1362
1363 1363 if n is not None:
1364 1364 self.n = int(n)
1365 1365 else:
1366 1366 self.__integrationtime = int(timeInterval)
1367 1367 self.n = None
1368 1368 self.__byTime = True
1369 1369
1370 1370
1371 1371 if minHei == None:
1372 1372 minHei = self.dataOut.heightList[0]
1373 1373
1374 1374 if maxHei == None:
1375 1375 maxHei = self.dataOut.heightList[-1]
1376 1376
1377 1377 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
1378 1378 print('minHei: %.2f is out of the heights range' % (minHei))
1379 1379 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
1380 1380 minHei = self.dataOut.heightList[0]
1381 1381
1382 1382 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
1383 1383 print('maxHei: %.2f is out of the heights range' % (maxHei))
1384 1384 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
1385 1385 maxHei = self.dataOut.heightList[-1]
1386 1386
1387 1387 ind_list1 = numpy.where(self.dataOut.heightList >= minHei)
1388 1388 ind_list2 = numpy.where(self.dataOut.heightList <= maxHei)
1389 1389 self.minHei_ind = ind_list1[0][0]
1390 1390 self.maxHei_ind = ind_list2[0][-1]
1391 1391
1392 1392 def putData(self, data_spc, data_cspc, data_dc):
1393 1393 """
1394 1394 Add a profile to the __buffer_spc and increase in one the __profileIndex
1395 1395
1396 1396 """
1397 1397
1398 1398 self.__buffer_spc.append(data_spc)
1399 1399
1400 1400 if self.__nChannels < 2:
1401 1401 self.__buffer_cspc = None
1402 1402 else:
1403 1403 self.__buffer_cspc.append(data_cspc)
1404 1404
1405 1405 if data_dc is None:
1406 1406 self.__buffer_dc = None
1407 1407 else:
1408 1408 self.__buffer_dc += data_dc
1409 1409
1410 1410 self.__profIndex += 1
1411 1411
1412 1412 return
1413 1413
1414 1414 def hildebrand_sekhon_Integration(self,sortdata,navg, factor):
1415 1415 #data debe estar ordenado
1416 1416 #sortdata = numpy.sort(data, axis=None)
1417 1417 #sortID=data.argsort()
1418 1418 lenOfData = len(sortdata)
1419 1419 nums_min = lenOfData*factor
1420 1420 if nums_min <= 5:
1421 1421 nums_min = 5
1422 1422 sump = 0.
1423 1423 sumq = 0.
1424 1424 j = 0
1425 1425 cont = 1
1426 1426 while((cont == 1)and(j < lenOfData)):
1427 1427 sump += sortdata[j]
1428 1428 sumq += sortdata[j]**2
1429 1429 if j > nums_min:
1430 1430 rtest = float(j)/(j-1) + 1.0/navg
1431 1431 if ((sumq*j) > (rtest*sump**2)):
1432 1432 j = j - 1
1433 1433 sump = sump - sortdata[j]
1434 1434 sumq = sumq - sortdata[j]**2
1435 1435 cont = 0
1436 1436 j += 1
1437 1437 #lnoise = sump / j
1438 1438 #print("H S done")
1439 1439 #return j,sortID
1440 1440 return j
1441 1441
1442 1442
1443 1443 def pushData(self):
1444 1444 """
1445 1445 Return the sum of the last profiles and the profiles used in the sum.
1446 1446
1447 1447 Affected:
1448 1448
1449 1449 self.__profileIndex
1450 1450
1451 1451 """
1452 1452 bufferH=None
1453 1453 buffer=None
1454 1454 buffer1=None
1455 1455 buffer_cspc=None
1456 1456 #print("aes: ", self.__buffer_cspc)
1457 1457 self.__buffer_spc=numpy.array(self.__buffer_spc)
1458 1458 if self.__nChannels > 1 :
1459 1459 self.__buffer_cspc=numpy.array(self.__buffer_cspc)
1460 1460
1461 1461 #print("FREQ_DC",self.__buffer_spc.shape,self.__buffer_cspc.shape)
1462 1462
1463 1463 freq_dc = int(self.__buffer_spc.shape[2] / 2)
1464 1464 #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights)
1465 1465
1466 1466 self.dataOutliers = numpy.zeros((self.nChannels,self.nHeights)) # --> almacen de outliers
1467 1467
1468 1468 for k in range(self.minHei_ind,self.maxHei_ind):
1469 1469 if self.__nChannels > 1:
1470 1470 buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k])
1471 1471
1472 1472 outliers_IDs_cspc=[]
1473 1473 cspc_outliers_exist=False
1474 1474 for i in range(self.nChannels):#dataOut.nChannels):
1475 1475
1476 1476 buffer1=numpy.copy(self.__buffer_spc[:,i,:,k])
1477 1477 indexes=[]
1478 1478 #sortIDs=[]
1479 1479 outliers_IDs=[]
1480 1480
1481 1481 for j in range(self.nProfiles): #frecuencias en el tiempo
1482 1482 # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0
1483 1483 # continue
1484 1484 # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1
1485 1485 # continue
1486 1486 buffer=buffer1[:,j]
1487 1487 sortdata = numpy.sort(buffer, axis=None)
1488 1488
1489 1489 sortID=buffer.argsort()
1490 1490 index = _noise.hildebrand_sekhon2(sortdata,self.navg)
1491 1491
1492 1492 #index,sortID=self.hildebrand_sekhon_Integration(buffer,1,self.factor)
1493 1493
1494 1494 # fig,ax = plt.subplots()
1495 1495 # ax.set_title(str(k)+" "+str(j))
1496 1496 # x=range(len(sortdata))
1497 1497 # ax.scatter(x,sortdata)
1498 1498 # ax.axvline(index)
1499 1499 # plt.show()
1500 1500
1501 1501 indexes.append(index)
1502 1502 #sortIDs.append(sortID)
1503 1503 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
1504 1504
1505 1505 #print("Outliers: ",outliers_IDs)
1506 1506 outliers_IDs=numpy.array(outliers_IDs)
1507 1507 outliers_IDs=outliers_IDs.ravel()
1508 1508 outliers_IDs=numpy.unique(outliers_IDs)
1509 1509 outliers_IDs=outliers_IDs.astype(numpy.dtype('int64'))
1510 1510 indexes=numpy.array(indexes)
1511 1511 indexmin=numpy.min(indexes)
1512 1512
1513 1513
1514 1514 #print(indexmin,buffer1.shape[0], k)
1515 1515
1516 1516 # fig,ax = plt.subplots()
1517 1517 # ax.plot(sortdata)
1518 1518 # ax2 = ax.twinx()
1519 1519 # x=range(len(indexes))
1520 1520 # #plt.scatter(x,indexes)
1521 1521 # ax2.scatter(x,indexes)
1522 1522 # plt.show()
1523 1523
1524 1524 if indexmin != buffer1.shape[0]:
1525 1525 if self.__nChannels > 1:
1526 1526 cspc_outliers_exist= True
1527 1527
1528 1528 lt=outliers_IDs
1529 1529 #avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0)
1530 1530
1531 1531 for p in list(outliers_IDs):
1532 1532 #buffer1[p,:]=avg
1533 1533 buffer1[p,:] = numpy.NaN
1534 1534
1535 1535 self.dataOutliers[i,k] = len(outliers_IDs)
1536 1536
1537 1537
1538 1538 self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1)
1539 1539
1540 1540
1541 1541 if self.__nChannels > 1:
1542 1542 outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs)
1543 1543
1544 1544
1545 1545 if self.__nChannels > 1:
1546 1546 outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64'))
1547 1547 if cspc_outliers_exist:
1548 1548
1549 1549 lt=outliers_IDs_cspc
1550 1550
1551 1551 #avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0)
1552 1552 for p in list(outliers_IDs_cspc):
1553 1553 #buffer_cspc[p,:]=avg
1554 1554 buffer_cspc[p,:] = numpy.NaN
1555 1555
1556 1556 if self.__nChannels > 1:
1557 1557 self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc)
1558 1558
1559 1559
1560 1560
1561 1561
1562 1562 nOutliers = len(outliers_IDs)
1563 1563 #print("Outliers n: ",self.dataOutliers,nOutliers)
1564 1564 buffer=None
1565 1565 bufferH=None
1566 1566 buffer1=None
1567 1567 buffer_cspc=None
1568 1568
1569 1569
1570 1570 buffer=None
1571 1571
1572 1572 #data_spc = numpy.sum(self.__buffer_spc,axis=0)
1573 1573 data_spc = numpy.nansum(self.__buffer_spc,axis=0)
1574 1574 if self.__nChannels > 1:
1575 1575 #data_cspc = numpy.sum(self.__buffer_cspc,axis=0)
1576 1576 data_cspc = numpy.nansum(self.__buffer_cspc,axis=0)
1577 1577 else:
1578 1578 data_cspc = None
1579 1579 data_dc = self.__buffer_dc
1580 1580 #(CH, HEIGH)
1581 1581 self.maxProfilesInt = self.__profIndex - 1
1582 1582 n = self.__profIndex - self.dataOutliers # n becomes a matrix
1583 1583
1584 1584 self.__buffer_spc = []
1585 1585 self.__buffer_cspc = []
1586 1586 self.__buffer_dc = 0
1587 1587 self.__profIndex = 0
1588 1588 #print("cleaned ",data_cspc)
1589 1589 return data_spc, data_cspc, data_dc, n
1590 1590
1591 1591 def byProfiles(self, *args):
1592 1592
1593 1593 self.__dataReady = False
1594 1594 avgdata_spc = None
1595 1595 avgdata_cspc = None
1596 1596 avgdata_dc = None
1597 1597
1598 1598 self.putData(*args)
1599 1599
1600 if self.__profIndex == self.n:
1600 if self.__profIndex >= self.n:
1601 1601
1602 1602 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1603 1603 self.n_ints = n
1604 1604 self.__dataReady = True
1605 1605
1606 1606 return avgdata_spc, avgdata_cspc, avgdata_dc
1607 1607
1608 1608 def byTime(self, datatime, *args):
1609 1609
1610 1610 self.__dataReady = False
1611 1611 avgdata_spc = None
1612 1612 avgdata_cspc = None
1613 1613 avgdata_dc = None
1614 1614
1615 1615 self.putData(*args)
1616 1616
1617 1617 if (datatime - self.__initime) >= self.__integrationtime:
1618 1618 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1619 1619 self.n_ints = n
1620 1620 self.__dataReady = True
1621 1621
1622 1622 return avgdata_spc, avgdata_cspc, avgdata_dc
1623 1623
1624 1624 def integrate(self, datatime, *args):
1625 1625
1626 1626 if self.__profIndex == 0:
1627 1627 self.__initime = datatime
1628 1628
1629 1629 if self.__byTime:
1630 1630 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1631 1631 datatime, *args)
1632 1632 else:
1633 1633 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1634 1634
1635 1635 if not self.__dataReady:
1636 1636 return None, None, None, None
1637 1637
1638 1638 #print("integrate", avgdata_cspc)
1639 1639 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1640 1640
1641 1641 def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False, minHei=None, maxHei=None, avg=1, factor=0.75):
1642 1642 self.dataOut = dataOut
1643 1643 if n == 1:
1644 1644 return self.dataOut
1645 1645 self.dataOut.processingHeaderObj.timeIncohInt = timeInterval
1646 1646
1647 1647 if dataOut.flagProfilesByRange:
1648 1648 self._flagProfilesByRange = True
1649 1649
1650 1650 if self.dataOut.nChannels == 1:
1651 1651 self.dataOut.data_cspc = None #si es un solo canal no vale la pena acumular DATOS
1652 1652 #print("IN spc:", self.dataOut.data_spc.shape, self.dataOut.data_cspc)
1653 1653 if not self.isConfig:
1654 1654 self.setup(self.dataOut, n, timeInterval, overlapping,DPL ,minHei, maxHei, avg, factor)
1655 1655 self.isConfig = True
1656 1656
1657 1657 if not self.ByLags:
1658 1658 self.nProfiles=self.dataOut.nProfiles
1659 1659 self.nChannels=self.dataOut.nChannels
1660 1660 self.nHeights=self.dataOut.nHeights
1661 1661 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1662 1662 self.dataOut.data_spc,
1663 1663 self.dataOut.data_cspc,
1664 1664 self.dataOut.data_dc)
1665 1665 else:
1666 1666 self.nProfiles=self.dataOut.nProfiles
1667 1667 self.nChannels=self.dataOut.nChannels
1668 1668 self.nHeights=self.dataOut.nHeights
1669 1669 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1670 1670 self.dataOut.dataLag_spc,
1671 1671 self.dataOut.dataLag_cspc,
1672 1672 self.dataOut.dataLag_dc)
1673 1673 self.dataOut.flagNoData = True
1674 1674
1675 1675 if self._flagProfilesByRange:
1676 1676 dataOut.flagProfilesByRange = True
1677 1677 self._nProfilesByRange += dataOut.nProfilesByRange
1678 1678
1679 1679 if self.__dataReady:
1680 1680
1681 1681 if not self.ByLags:
1682 1682 if self.nChannels == 1:
1683 1683 #print("f int", avgdata_spc.shape)
1684 1684 self.dataOut.data_spc = avgdata_spc
1685 1685 self.dataOut.data_cspc = None
1686 1686 else:
1687 1687 self.dataOut.data_spc = numpy.squeeze(avgdata_spc)
1688 1688 self.dataOut.data_cspc = numpy.squeeze(avgdata_cspc)
1689 1689 self.dataOut.data_dc = avgdata_dc
1690 1690 self.dataOut.data_outlier = self.dataOutliers
1691 1691
1692 1692
1693 1693 else:
1694 1694 self.dataOut.dataLag_spc = avgdata_spc
1695 1695 self.dataOut.dataLag_cspc = avgdata_cspc
1696 1696 self.dataOut.dataLag_dc = avgdata_dc
1697 1697
1698 1698 self.dataOut.data_spc=self.dataOut.dataLag_spc[:,:,:,self.dataOut.LagPlot]
1699 1699 self.dataOut.data_cspc=self.dataOut.dataLag_cspc[:,:,:,self.dataOut.LagPlot]
1700 1700 self.dataOut.data_dc=self.dataOut.dataLag_dc[:,:,self.dataOut.LagPlot]
1701 1701
1702
1703 1702 self.dataOut.nIncohInt *= self.n_ints
1704 1703 #print("maxProfilesInt: ",self.maxProfilesInt)
1705 1704
1706 1705 self.dataOut.utctime = avgdatatime
1707 1706 self.dataOut.flagNoData = False
1708 1707
1709 1708 dataOut.nProfilesByRange = self._nProfilesByRange
1710 1709 self._nProfilesByRange = numpy.zeros((1,len(dataOut.heightList)))
1711 1710 self._flagProfilesByRange = False
1712 1711
1713 1712 # #update Processing Header:
1714 1713 # self.dataOut.processingHeaderObj.nIncohInt =
1715 1714 # self.dataOut.processingHeaderObj.nFFTPoints = self.dataOut.nFFTPoints
1716 1715
1717 1716 #print("Faraday Integration DONE...", self.dataOut.data_cspc)
1718 1717 #print(self.dataOut.flagNoData)
1719 1718 return self.dataOut
1720 1719
1721 1720
1722 1721
1723 1722 class removeInterference(Operation):
1724 1723
1725 1724 def removeInterference3(self, min_hei = None, max_hei = None):
1726 1725
1727 1726 jspectra = self.dataOut.data_spc
1728 1727 #jcspectra = self.dataOut.data_cspc
1729 1728 jnoise = self.dataOut.getNoise()
1730 1729 num_incoh = self.dataOut.max_nIncohInt
1731 1730 #print(jspectra.shape)
1732 1731 num_channel, num_prof, num_hei = jspectra.shape
1733 1732 minHei = min_hei
1734 1733 maxHei = max_hei
1735 1734 ########################################################################
1736 1735 if minHei == None or (minHei < self.dataOut.heightList[0]):
1737 1736 minHei = self.dataOut.heightList[0]
1738 1737
1739 1738 if maxHei == None or (maxHei > self.dataOut.heightList[-1]):
1740 1739 maxHei = self.dataOut.heightList[-1]
1741 1740 minIndex = 0
1742 1741 maxIndex = 0
1743 1742 heights = self.dataOut.heightList
1744 1743
1745 1744 inda = numpy.where(heights >= minHei)
1746 1745 indb = numpy.where(heights <= maxHei)
1747 1746
1748 1747 try:
1749 1748 minIndex = inda[0][0]
1750 1749 except:
1751 1750 minIndex = 0
1752 1751 try:
1753 1752 maxIndex = indb[0][-1]
1754 1753 except:
1755 1754 maxIndex = len(heights)
1756 1755
1757 1756 if (minIndex < 0) or (minIndex > maxIndex):
1758 1757 raise ValueError("some value in (%d,%d) is not valid" % (
1759 1758 minIndex, maxIndex))
1760 1759 if (maxIndex >= self.dataOut.nHeights):
1761 1760 maxIndex = self.dataOut.nHeights - 1
1762 1761
1763 1762 ########################################################################
1764 1763
1765 1764
1766 1765 #dataOut.max_nIncohInt * dataOut.nCohInt
1767 norm = self.dataOut.nIncohInt /self.dataOut.max_nIncohInt
1768 #print(norm.shape)
1766 if hasattr(self.dataOut.nIncohInt, 'shape'):
1767 norm = self.dataOut.nIncohInt.T /self.dataOut.max_nIncohInt
1768 norm = norm.T
1769 else:
1770 norm = self.dataOut.nIncohInt /self.dataOut.max_nIncohInt
1771 norm = norm
1772
1769 1773 # Subrutina de Remocion de la Interferencia
1770 1774 for ich in range(num_channel):
1771 1775 # Se ordena los espectros segun su potencia (menor a mayor)
1772 1776 #power = jspectra[ich, mask_prof, :]
1773 interf = jspectra[ich, :, minIndex:maxIndex]
1774 #print(interf.shape)
1777 if hasattr(self.dataOut.nIncohInt, 'shape'):
1778 interf = jspectra[ich, :, minIndex:maxIndex]/norm[ich,minIndex:maxIndex]
1779 else:
1780 interf = jspectra[ich, :, minIndex:maxIndex]/norm
1781 # print(interf.shape)
1775 1782 inttef = interf.mean(axis=1)
1776 1783
1777 1784 for hei in range(num_hei):
1778 temp = jspectra[ich,:, hei]
1785 temp = jspectra[ich,:, hei]#/norm[ich,hei]
1779 1786 temp -= inttef
1780 temp += jnoise[ich]*norm[ich,hei]
1781 jspectra[ich,:, hei] = temp
1787 temp += jnoise[ich]
1788 # print(jspectra.shape, temp.shape)
1789 jspectra[ich,:, hei] = temp
1782 1790
1783 1791 # Guardar Resultados
1784 1792 self.dataOut.data_spc = jspectra
1785 1793 #self.dataOut.data_cspc = jcspectra
1786 1794
1787 1795 return 1
1788 1796
1789 1797 def removeInterference2(self):
1790 1798
1791 1799 cspc = self.dataOut.data_cspc
1792 1800 spc = self.dataOut.data_spc
1793 1801 Heights = numpy.arange(cspc.shape[2])
1794 1802 realCspc = numpy.abs(cspc)
1795 1803
1796 1804 for i in range(cspc.shape[0]):
1797 1805 LinePower= numpy.sum(realCspc[i], axis=0)
1798 1806 Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
1799 1807 SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
1800 1808 InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
1801 1809 InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
1802 1810 InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
1803 1811
1804 1812
1805 1813 InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
1806 1814 #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
1807 1815 if len(InterferenceRange)<int(cspc.shape[1]*0.3):
1808 1816 cspc[i,InterferenceRange,:] = numpy.NaN
1809 1817
1810 1818 self.dataOut.data_cspc = cspc
1811 1819
1812 1820 def removeInterference(self, interf = 2, hei_interf = None, nhei_interf = None, offhei_interf = None):
1813 1821
1814 1822 jspectra = self.dataOut.data_spc
1815 1823 jcspectra = self.dataOut.data_cspc
1816 1824 jnoise = self.dataOut.getNoise()
1817 1825 #num_incoh = self.dataOut.nIncohInt
1818 1826 num_incoh = self.dataOut.max_nIncohInt
1819 1827 #print("spc: ", jspectra.shape, jcspectra)
1820 1828 num_channel = jspectra.shape[0]
1821 1829 num_prof = jspectra.shape[1]
1822 1830 num_hei = jspectra.shape[2]
1823 1831
1832 count_hei = nhei_interf
1824 1833 # hei_interf
1825 1834 if hei_interf is None:
1826 1835 count_hei = int(num_hei / 2) # a half of total ranges
1827 1836 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
1828 1837 hei_interf = numpy.asarray(hei_interf)[0]
1829 1838 #print(hei_interf)
1830 1839 # nhei_interf
1831 1840 if (nhei_interf == None):
1832 1841 nhei_interf = 5
1833 1842 if (nhei_interf < 1):
1834 1843 nhei_interf = 1
1835 1844 if (nhei_interf > count_hei):
1836 1845 nhei_interf = count_hei
1837 1846 if (offhei_interf == None):
1838 1847 offhei_interf = 0
1839 1848
1840 1849 ind_hei = list(range(num_hei))
1841 1850 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
1842 1851 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
1843 1852 mask_prof = numpy.asarray(list(range(num_prof)))
1844 1853 num_mask_prof = mask_prof.size
1845 1854 comp_mask_prof = [0, num_prof / 2]
1846 1855
1847 1856 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
1848 1857 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
1849 1858 jnoise = numpy.nan
1850 1859 noise_exist = jnoise[0] < numpy.Inf
1851 1860
1852 1861 # Subrutina de Remocion de la Interferencia
1853 1862 for ich in range(num_channel):
1854 1863 # Se ordena los espectros segun su potencia (menor a mayor)
1855 1864 power = jspectra[ich, mask_prof, :]
1856 1865 power = power[:, hei_interf]
1857 1866 power = power.sum(axis=0)
1858 1867 psort = power.ravel().argsort()
1859 1868 #print(hei_interf[psort[list(range(offhei_interf, nhei_interf + offhei_interf))]])
1860 1869 # Se estima la interferencia promedio en los Espectros de Potencia empleando
1861 1870 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
1862 1871 offhei_interf, nhei_interf + offhei_interf))]]]
1863 1872
1864 1873 if noise_exist:
1865 1874 # tmp_noise = jnoise[ich] / num_prof
1866 1875 tmp_noise = jnoise[ich]
1867 1876 junkspc_interf = junkspc_interf - tmp_noise
1868 1877 #junkspc_interf[:,comp_mask_prof] = 0
1869 1878 #print(junkspc_interf.shape)
1870 1879 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
1871 1880 jspc_interf = jspc_interf.transpose()
1872 1881 # Calculando el espectro de interferencia promedio
1873 1882 noiseid = numpy.where(jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
1874 1883 noiseid = noiseid[0]
1875 1884 cnoiseid = noiseid.size
1876 1885 interfid = numpy.where(jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
1877 1886 interfid = interfid[0]
1878 1887 cinterfid = interfid.size
1879 1888
1880 1889 if (cnoiseid > 0):
1881 1890 jspc_interf[noiseid] = 0
1882 1891 # Expandiendo los perfiles a limpiar
1883 1892 if (cinterfid > 0):
1884 1893 new_interfid = (
1885 1894 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
1886 1895 new_interfid = numpy.asarray(new_interfid)
1887 1896 new_interfid = {x for x in new_interfid}
1888 1897 new_interfid = numpy.array(list(new_interfid))
1889 1898 new_cinterfid = new_interfid.size
1890 1899 else:
1891 1900 new_cinterfid = 0
1892 1901
1893 1902 for ip in range(new_cinterfid):
1894 1903 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
1895 1904 jspc_interf[new_interfid[ip]] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
1896 1905
1897 1906 jspectra[ich, :, ind_hei] = jspectra[ich, :,ind_hei] - jspc_interf # Corregir indices
1898 1907
1899 1908 # Removiendo la interferencia del punto de mayor interferencia
1900 1909 ListAux = jspc_interf[mask_prof].tolist()
1901 1910 maxid = ListAux.index(max(ListAux))
1902 1911 #print(cinterfid)
1903 1912 if cinterfid > 0:
1904 1913 for ip in range(cinterfid * (interf == 2) - 1):
1905 1914 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
1906 1915 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
1907 1916 cind = len(ind)
1908 1917
1909 1918 if (cind > 0):
1910 1919 jspectra[ich, interfid[ip], ind] = tmp_noise * \
1911 1920 (1 + (numpy.random.uniform(cind) - 0.5) /
1912 1921 numpy.sqrt(num_incoh))
1913 1922
1914 1923 ind = numpy.array([-2, -1, 1, 2])
1915 1924 xx = numpy.zeros([4, 4])
1916 1925
1917 1926 for id1 in range(4):
1918 1927 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1919 1928 xx_inv = numpy.linalg.inv(xx)
1920 1929 xx = xx_inv[:, 0]
1921 1930 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1922 1931 yy = jspectra[ich, mask_prof[ind], :]
1923 1932 jspectra[ich, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1924 1933
1925 1934 indAux = (jspectra[ich, :, :] < tmp_noise *
1926 1935 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
1927 1936 #print(indAux)
1928 1937 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
1929 1938 (1 - 1 / numpy.sqrt(num_incoh))
1930 1939
1931 1940 # Remocion de Interferencia en el Cross Spectra
1932 1941 if jcspectra is None:
1933 1942 return jspectra, jcspectra
1934 1943 num_pairs = int(jcspectra.size / (num_prof * num_hei))
1935 1944 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
1936 1945
1937 1946 for ip in range(num_pairs):
1938 1947
1939 1948 #-------------------------------------------
1940 1949
1941 1950 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
1942 1951 cspower = cspower[:, hei_interf]
1943 1952 cspower = cspower.sum(axis=0)
1944 1953
1945 1954 cspsort = cspower.ravel().argsort()
1946 1955 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
1947 1956 offhei_interf, nhei_interf + offhei_interf))]]]
1948 1957 junkcspc_interf = junkcspc_interf.transpose()
1949 1958 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
1950 1959
1951 1960 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
1952 1961
1953 1962 median_real = int(numpy.median(numpy.real(
1954 1963 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1955 1964 median_imag = int(numpy.median(numpy.imag(
1956 1965 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1957 1966 comp_mask_prof = [int(e) for e in comp_mask_prof]
1958 1967 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
1959 1968 median_real, median_imag)
1960 1969
1961 1970 for iprof in range(num_prof):
1962 1971 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
1963 1972 jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
1964 1973
1965 1974 # Removiendo la Interferencia
1966 1975 jcspectra[ip, :, ind_hei] = jcspectra[ip,
1967 1976 :, ind_hei] - jcspc_interf
1968 1977
1969 1978 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
1970 1979 maxid = ListAux.index(max(ListAux))
1971 1980
1972 1981 ind = numpy.array([-2, -1, 1, 2])
1973 1982 xx = numpy.zeros([4, 4])
1974 1983
1975 1984 for id1 in range(4):
1976 1985 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1977 1986
1978 1987 xx_inv = numpy.linalg.inv(xx)
1979 1988 xx = xx_inv[:, 0]
1980 1989
1981 1990 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1982 1991 yy = jcspectra[ip, mask_prof[ind], :]
1983 1992 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1984 1993
1985 1994 # Guardar Resultados
1986 1995 self.dataOut.data_spc = jspectra
1987 1996 self.dataOut.data_cspc = jcspectra
1988 1997
1989 1998 return 1
1990 1999
1991 2000 def run(self, dataOut, interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None, mode=1, minHei=None, maxHei=None):
1992 2001
1993 2002 self.dataOut = dataOut
1994 2003
1995 2004 if mode == 1:
1996 self.removeInterference(interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None)
2005 self.removeInterference(interf = 2,hei_interf = hei_interf, nhei_interf = nhei_interf, offhei_interf = offhei_interf)
1997 2006 elif mode == 2:
1998 2007 self.removeInterference2()
1999 2008 elif mode == 3:
2000 2009 self.removeInterference3(min_hei=minHei, max_hei=maxHei)
2001 2010 return self.dataOut
2002 2011
2003 2012
2004 2013 class IncohInt(Operation):
2005 2014
2006 2015 __profIndex = 0
2007 2016 __withOverapping = False
2008 2017
2009 2018 __byTime = False
2010 2019 __initime = None
2011 2020 __lastdatatime = None
2012 2021 __integrationtime = None
2013 2022
2014 2023 __buffer_spc = None
2015 2024 __buffer_cspc = None
2016 2025 __buffer_dc = None
2017 2026
2018 2027 __dataReady = False
2019 2028
2020 2029 __timeInterval = None
2021 2030 incohInt = 0
2022 2031 nOutliers = 0
2023 2032 n = None
2024 2033
2025 2034 _flagProfilesByRange = False
2026 2035 _nProfilesByRange = 0
2027 2036 def __init__(self):
2028 2037
2029 2038 Operation.__init__(self)
2030 2039
2031 2040 def setup(self, n=None, timeInterval=None, overlapping=False):
2032 2041 """
2033 2042 Set the parameters of the integration class.
2034 2043
2035 2044 Inputs:
2036 2045
2037 2046 n : Number of coherent integrations
2038 2047 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
2039 2048 overlapping :
2040 2049
2041 2050 """
2042 2051
2043 2052 self.__initime = None
2044 2053 self.__lastdatatime = 0
2045 2054
2046 2055 self.__buffer_spc = 0
2047 2056 self.__buffer_cspc = 0
2048 2057 self.__buffer_dc = 0
2049 2058
2050 2059 self.__profIndex = 0
2051 2060 self.__dataReady = False
2052 2061 self.__byTime = False
2053 2062 self.incohInt = 0
2054 2063 self.nOutliers = 0
2055 2064 if n is None and timeInterval is None:
2056 2065 raise ValueError("n or timeInterval should be specified ...")
2057 2066
2058 2067 if n is not None:
2059 2068 self.n = int(n)
2060 2069 else:
2061 2070
2062 2071 self.__integrationtime = int(timeInterval)
2063 2072 self.n = None
2064 2073 self.__byTime = True
2065 2074
2066 2075
2067 2076
2068 2077 def putData(self, data_spc, data_cspc, data_dc):
2069 2078 """
2070 2079 Add a profile to the __buffer_spc and increase in one the __profileIndex
2071 2080
2072 2081 """
2073 2082 if data_spc.all() == numpy.nan :
2074 2083 print("nan ")
2075 2084 return
2076 2085 self.__buffer_spc += data_spc
2077 2086
2078 2087 if data_cspc is None:
2079 2088 self.__buffer_cspc = None
2080 2089 else:
2081 2090 self.__buffer_cspc += data_cspc
2082 2091
2083 2092 if data_dc is None:
2084 2093 self.__buffer_dc = None
2085 2094 else:
2086 2095 self.__buffer_dc += data_dc
2087 2096
2088 2097 self.__profIndex += 1
2089 2098
2090 2099 return
2091 2100
2092 2101 def pushData(self):
2093 2102 """
2094 2103 Return the sum of the last profiles and the profiles used in the sum.
2095 2104
2096 2105 Affected:
2097 2106
2098 2107 self.__profileIndex
2099 2108
2100 2109 """
2101 2110
2102 2111 data_spc = self.__buffer_spc
2103 2112 data_cspc = self.__buffer_cspc
2104 2113 data_dc = self.__buffer_dc
2105 2114 n = self.__profIndex
2106 2115
2107 2116 self.__buffer_spc = 0
2108 2117 self.__buffer_cspc = 0
2109 2118 self.__buffer_dc = 0
2110 2119
2111 2120
2112 2121 return data_spc, data_cspc, data_dc, n
2113 2122
2114 2123 def byProfiles(self, *args):
2115 2124
2116 2125 self.__dataReady = False
2117 2126 avgdata_spc = None
2118 2127 avgdata_cspc = None
2119 2128 avgdata_dc = None
2120 2129
2121 2130 self.putData(*args)
2122 2131
2123 2132 if self.__profIndex == self.n:
2124 2133
2125 2134 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
2126 2135 self.n = n
2127 2136 self.__dataReady = True
2128 2137
2129 2138 return avgdata_spc, avgdata_cspc, avgdata_dc
2130 2139
2131 2140 def byTime(self, datatime, *args):
2132 2141
2133 2142 self.__dataReady = False
2134 2143 avgdata_spc = None
2135 2144 avgdata_cspc = None
2136 2145 avgdata_dc = None
2137 2146
2138 2147 self.putData(*args)
2139 2148
2140 2149 if (datatime - self.__initime) >= self.__integrationtime:
2141 2150 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
2142 2151 self.n = n
2143 2152 self.__dataReady = True
2144 2153
2145 2154 return avgdata_spc, avgdata_cspc, avgdata_dc
2146 2155
2147 2156 def integrate(self, datatime, *args):
2148 2157
2149 2158 if self.__profIndex == 0:
2150 2159 self.__initime = datatime
2151 2160
2152 2161 if self.__byTime:
2153 2162 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
2154 2163 datatime, *args)
2155 2164 else:
2156 2165 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
2157 2166
2158 2167 if not self.__dataReady:
2159 2168 return None, None, None, None
2160 2169
2161 2170 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
2162 2171
2163 2172 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
2164 2173 if n == 1:
2165 2174 return dataOut
2166 2175
2167 2176 if dataOut.flagNoData == True:
2168 2177 return dataOut
2169 2178
2170 2179 if dataOut.flagProfilesByRange == True:
2171 2180 self._flagProfilesByRange = True
2172 2181
2173 2182 dataOut.flagNoData = True
2174 2183 dataOut.processingHeaderObj.timeIncohInt = timeInterval
2175 2184 if not self.isConfig:
2176 2185 self._nProfilesByRange = numpy.zeros((1,len(dataOut.heightList)))
2177 2186 self.setup(n, timeInterval, overlapping)
2178 2187 self.isConfig = True
2179 2188
2180 2189
2181 2190 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
2182 2191 dataOut.data_spc,
2183 2192 dataOut.data_cspc,
2184 2193 dataOut.data_dc)
2185 2194
2186 2195 self.incohInt += dataOut.nIncohInt
2187 2196
2188 2197
2189 2198 if isinstance(dataOut.data_outlier,numpy.ndarray) or isinstance(dataOut.data_outlier,int) or isinstance(dataOut.data_outlier, float):
2190 2199 self.nOutliers += dataOut.data_outlier
2191 2200
2192 2201 if self._flagProfilesByRange:
2193 2202 dataOut.flagProfilesByRange = True
2194 2203 self._nProfilesByRange += dataOut.nProfilesByRange
2195 2204
2196 2205 if self.__dataReady:
2197 2206 #print("prof: ",dataOut.max_nIncohInt,self.__profIndex)
2198 2207 dataOut.data_spc = avgdata_spc
2199 2208 dataOut.data_cspc = avgdata_cspc
2200 2209 dataOut.data_dc = avgdata_dc
2201 2210 dataOut.nIncohInt = self.incohInt
2202 2211 dataOut.data_outlier = self.nOutliers
2203 2212 dataOut.utctime = avgdatatime
2204 2213 dataOut.flagNoData = False
2205 2214 self.incohInt = 0
2206 2215 self.nOutliers = 0
2207 2216 self.__profIndex = 0
2208 2217 dataOut.nProfilesByRange = self._nProfilesByRange
2209 2218 self._nProfilesByRange = numpy.zeros((1,len(dataOut.heightList)))
2210 2219 self._flagProfilesByRange = False
2211 2220 #print("IncohInt Done")
2212 2221 return dataOut
2213 2222
2214 2223 class dopplerFlip(Operation):
2215 2224
2216 2225 def run(self, dataOut):
2217 2226 # arreglo 1: (num_chan, num_profiles, num_heights)
2218 2227 self.dataOut = dataOut
2219 2228 # JULIA-oblicua, indice 2
2220 2229 # arreglo 2: (num_profiles, num_heights)
2221 2230 jspectra = self.dataOut.data_spc[2]
2222 2231 jspectra_tmp = numpy.zeros(jspectra.shape)
2223 2232 num_profiles = jspectra.shape[0]
2224 2233 freq_dc = int(num_profiles / 2)
2225 2234 # Flip con for
2226 2235 for j in range(num_profiles):
2227 2236 jspectra_tmp[num_profiles-j-1]= jspectra[j]
2228 2237 # Intercambio perfil de DC con perfil inmediato anterior
2229 2238 jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1]
2230 2239 jspectra_tmp[freq_dc]= jspectra[freq_dc]
2231 2240 # canal modificado es re-escrito en el arreglo de canales
2232 2241 self.dataOut.data_spc[2] = jspectra_tmp
2233 2242
2234 2243 return self.dataOut
2235 2244
2236 2245
2237 2246
2238 2247
2239 2248
2240 2249
2241 2250 class cleanJULIAInterf(Operation):
2242 2251 """
2243 2252 Operación de prueba
2244 2253 """
2245 2254 __slots__ =('heights_indx', 'repeats','span' ,'step', 'factor', 'idate', 'idxs','isConfig','minHrefN', 'maxHrefN')
2246 2255 def __init__(self):
2247 2256 self.repeats = 0
2248 2257 self.factor=1
2249 2258 self.isConfig = False
2250 2259 self.idxs = None
2251 2260 self.heights_indx = None
2252 2261
2253 2262 def setup(self, dataOutHeightsList, heightsList, span=10, repeats=0, step=0, idate=None, startH=None, endH=None, minHref=None, maxHref=None):
2254 2263 totalHeihtList = dataOutHeightsList
2255 2264 heights = [float(hei) for hei in heightsList]
2256 2265 for r in range(repeats):
2257 2266 heights += [ (h+(step*(r+1))) for h in heights]
2258 2267 #print(heights)
2259 2268 self.heights_indx = [getHei_index(h,h,totalHeihtList)[0] for h in heights]
2260 2269
2261 2270 self.minHrefN, self.maxHrefN = getHei_index(minHref,maxHref,totalHeihtList)
2262 2271
2263 2272
2264 2273 self.config = True
2265 2274 self.span = span
2266 2275
2267 2276 def run(self, dataOut, heightsList, span=10, repeats=0, step=0, idate=None, startH=None, endH=None, minHref=None, maxHref=None):
2268 2277
2269 2278
2270 2279 self.dataOut = dataOut
2271 2280 startTime = datetime.datetime.combine(idate,startH)
2272 2281 endTime = datetime.datetime.combine(idate,endH)
2273 2282 currentTime = datetime.datetime.fromtimestamp(self.dataOut.utctime)
2274 2283
2275 2284 if currentTime < startTime or currentTime > endTime:
2276 2285 return self.dataOut
2277 2286
2278 2287 if not self.isConfig:
2279 2288 self.setup(self.dataOut.heightList,heightsList, span=span, repeats=repeats, step=step, idate=idate, startH=startH, endH=endH, minHref=minHref, maxHref=maxHref )
2280 2289
2281 2290 for ch in range(self.dataOut.data_spc.shape[0]):
2282 2291 i = 0
2283 2292 N_ref = self.dataOut.data_spc[ch, :, self.minHrefN: self.maxHrefN].mean()
2284 2293 mn = self.heights_indx[-1] - self.span/2
2285 2294 mx = self.heights_indx[-1] + self.span/2
2286 2295 J_lev = self.dataOut.data_spc[ch, :, mn: mx].mean() - N_ref
2287 2296
2288 2297 for hei in self.heights_indx:
2289 2298 h = hei - 1
2290 2299 mn_i = hei - self.span/2
2291 2300 mx_i = hei + self.span/2
2292 2301 self.dataOut.data_spc[ch, :,mn_i:mx_i ] -= J_lev
2293 2302 i += 1
2294 2303
2295 2304
2296 2305 return self.dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,3090 +1,3101
1 1 import sys
2 2 import numpy,math
3 3 from scipy import interpolate
4 4 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
5 5 from schainpy.model.data.jrodata import Voltage,hildebrand_sekhon
6 6 from schainpy.utils import log
7 7 from schainpy.model.io.utilsIO import getHei_index
8 8 from time import time
9 9 import datetime
10 10 import numpy
11 11 #import copy
12 12 from schainpy.model.data import _noise
13 13
14 14 from matplotlib import pyplot as plt
15 15
16 16 class VoltageProc(ProcessingUnit):
17 17
18 18 def __init__(self):
19 19
20 20 ProcessingUnit.__init__(self)
21 21
22 22 self.dataOut = Voltage()
23 23 self.flip = 1
24 24 self.setupReq = False
25 25
26 26 def run(self):
27 27 #print("running volt proc")
28 28
29 29 if self.dataIn.type == 'AMISR':
30 30 self.__updateObjFromAmisrInput()
31 31
32 32 if self.dataOut.buffer_empty:
33 33 if self.dataIn.type == 'Voltage':
34 34 self.dataOut.copy(self.dataIn)
35 35 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
36 36 self.dataOut.ippSeconds = self.dataIn.ippSeconds
37 37 self.dataOut.ipp = self.dataIn.ipp
38 38
39 39 #update Processing Header:
40 40 self.dataOut.processingHeaderObj.heightList = self.dataOut.heightList
41 41 self.dataOut.processingHeaderObj.ipp = self.dataOut.ipp
42 42 self.dataOut.processingHeaderObj.nCohInt = self.dataOut.nCohInt
43 43 self.dataOut.processingHeaderObj.dtype = self.dataOut.type
44 44 self.dataOut.processingHeaderObj.channelList = self.dataOut.channelList
45 45 self.dataOut.processingHeaderObj.azimuthList = self.dataOut.azimuthList
46 46 self.dataOut.processingHeaderObj.elevationList = self.dataOut.elevationList
47 47 self.dataOut.processingHeaderObj.codeList = self.dataOut.nChannels
48 48 self.dataOut.processingHeaderObj.heightList = self.dataOut.heightList
49 49 self.dataOut.processingHeaderObj.heightResolution = self.dataOut.heightList[1] - self.dataOut.heightList[0]
50 50
51 51
52 52
53 53 def __updateObjFromAmisrInput(self):
54 54
55 55 self.dataOut.timeZone = self.dataIn.timeZone
56 56 self.dataOut.dstFlag = self.dataIn.dstFlag
57 57 self.dataOut.errorCount = self.dataIn.errorCount
58 58 self.dataOut.useLocalTime = self.dataIn.useLocalTime
59 59
60 60 self.dataOut.flagNoData = self.dataIn.flagNoData
61 61 self.dataOut.data = self.dataIn.data
62 62 self.dataOut.utctime = self.dataIn.utctime
63 63 self.dataOut.channelList = self.dataIn.channelList
64 64 #self.dataOut.timeInterval = self.dataIn.timeInterval
65 65 self.dataOut.heightList = self.dataIn.heightList
66 66 self.dataOut.nProfiles = self.dataIn.nProfiles
67 67
68 68 self.dataOut.nCohInt = self.dataIn.nCohInt
69 69 self.dataOut.ippSeconds = self.dataIn.ippSeconds
70 70 self.dataOut.frequency = self.dataIn.frequency
71 71
72 72 self.dataOut.azimuth = self.dataIn.azimuth
73 73 self.dataOut.zenith = self.dataIn.zenith
74 74
75 75 self.dataOut.beam.codeList = self.dataIn.beam.codeList
76 76 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
77 77 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
78 78
79 79
80 80 class selectChannels(Operation):
81 81
82 def run(self, dataOut, channelList=None):
82 def run(self, dataOut, channelList=[]):
83
84 if isinstance(channelList, int):
85 channelList = [channelList]
86
83 87 self.channelList = channelList
84 if self.channelList == None:
88 if len(self.channelList) == 0:
85 89 print("Missing channelList")
86 90 return dataOut
87 91 channelIndexList = []
88 92 if not dataOut.buffer_empty: # cuando se usa proc volts como buffer de datos
89 93 return dataOut
90 94 #print("channel List: ", dataOut.channelList)
91 95 if type(dataOut.channelList) is not list: #leer array desde HDF5
92 96 try:
93 97 dataOut.channelList = dataOut.channelList.tolist()
94 98 except Exception as e:
95 99 print("Select Channels: ",e)
96 100 for channel in self.channelList:
97 101 if channel not in dataOut.channelList:
98 102 raise ValueError("Channel %d is not in %s" %(channel, str(dataOut.channelList)))
99 103
100 104 index = dataOut.channelList.index(channel)
101 105 channelIndexList.append(index)
106
102 107 dataOut = self.selectChannelsByIndex(dataOut,channelIndexList)
103
108
104 109 #update Processing Header:
105 110 dataOut.processingHeaderObj.channelList = dataOut.channelList
106 111 dataOut.processingHeaderObj.elevationList = dataOut.elevationList
107 112 dataOut.processingHeaderObj.azimuthList = dataOut.azimuthList
108 113 dataOut.processingHeaderObj.codeList = dataOut.codeList
109 114 dataOut.processingHeaderObj.nChannels = len(dataOut.channelList)
110
115
111 116 return dataOut
112 117
113 118 def selectChannelsByIndex(self, dataOut, channelIndexList):
114 119 """
115 120 Selecciona un bloque de datos en base a canales segun el channelIndexList
116 121
117 122 Input:
118 123 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
119 124
120 125 Affected:
121 126 dataOut.data
122 127 dataOut.channelIndexList
123 128 dataOut.nChannels
124 129 dataOut.m_ProcessingHeader.totalSpectra
125 130 dataOut.systemHeaderObj.numChannels
126 131 dataOut.m_ProcessingHeader.blockSize
127 132
128 133 Return:
129 134 None
130 135 """
131 136 #print("selectChannelsByIndex")
132 137 # for channelIndex in channelIndexList:
133 138 # if channelIndex not in dataOut.channelIndexList:
134 139 # raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
135 140
136 141 if dataOut.type == 'Voltage':
137 142 if dataOut.flagDataAsBlock:
138 143 """
139 144 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
140 145 """
141 146 data = dataOut.data[channelIndexList,:,:]
142 147 else:
143 148 data = dataOut.data[channelIndexList,:]
144 149
145 150 dataOut.data = data
146 151 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
147 152 dataOut.channelList = [n for n in range(len(channelIndexList))]
148
153
149 154 elif dataOut.type == 'Spectra':
150 155 if hasattr(dataOut, 'data_spc'):
151 156 if dataOut.data_spc is None:
152 157 raise ValueError("data_spc is None")
153 158 return dataOut
154 159 else:
155 160 data_spc = dataOut.data_spc[channelIndexList, :]
156 161 dataOut.data_spc = data_spc
157 162
158 163 # if hasattr(dataOut, 'data_dc') :# and
159 164 # if dataOut.data_dc is None:
160 165 # raise ValueError("data_dc is None")
161 166 # return dataOut
162 167 # else:
163 168 # data_dc = dataOut.data_dc[channelIndexList, :]
164 169 # dataOut.data_dc = data_dc
165 170 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
166 171 dataOut.channelList = channelIndexList
167 172 dataOut = self.__selectPairsByChannel(dataOut,channelIndexList)
168 if len(dataOut.elevationList>0):
173
174 # channelIndexList = numpy.asarray(channelIndexList)
175 dataOut.elevationList = numpy.asarray(dataOut.elevationList)
176 dataOut.azimuthList = numpy.asarray(dataOut.azimuthList)
177 dataOut.codeList = numpy.asarray(dataOut.codeList)
178 if (len(dataOut.elevationList) > 0):
169 179 dataOut.elevationList = dataOut.elevationList[channelIndexList]
170 180 dataOut.azimuthList = dataOut.azimuthList[channelIndexList]
171 181 dataOut.codeList = dataOut.codeList[channelIndexList]
182
172 183 return dataOut
173 184
174 185 def __selectPairsByChannel(self, dataOut, channelList=None):
175 186 #print("__selectPairsByChannel")
176 187 if channelList == None:
177 188 return
178 189
179 190 pairsIndexListSelected = []
180 191 for pairIndex in dataOut.pairsIndexList:
181 192 # First pair
182 193 if dataOut.pairsList[pairIndex][0] not in channelList:
183 194 continue
184 195 # Second pair
185 196 if dataOut.pairsList[pairIndex][1] not in channelList:
186 197 continue
187 198
188 199 pairsIndexListSelected.append(pairIndex)
189 200 if not pairsIndexListSelected:
190 201 dataOut.data_cspc = None
191 202 dataOut.pairsList = []
192 203 return
193 204
194 205 dataOut.data_cspc = dataOut.data_cspc[pairsIndexListSelected]
195 206 dataOut.pairsList = [dataOut.pairsList[i]
196 207 for i in pairsIndexListSelected]
197 208
198 209 return dataOut
199 210
200 211 class selectHeights(Operation):
201 212
202 213 def run(self, dataOut, minHei=None, maxHei=None, minIndex=None, maxIndex=None):
203 214 """
204 215 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
205 216 minHei <= height <= maxHei
206 217
207 218 Input:
208 219 minHei : valor minimo de altura a considerar
209 220 maxHei : valor maximo de altura a considerar
210 221
211 222 Affected:
212 223 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
213 224
214 225 Return:
215 226 1 si el metodo se ejecuto con exito caso contrario devuelve 0
216 227 """
217 228
218 229 self.dataOut = dataOut
219 230
220 231 if minHei and maxHei:
221 232
222 233 if (minHei < dataOut.heightList[0]):
223 234 minHei = dataOut.heightList[0]
224 235
225 236 if (maxHei > dataOut.heightList[-1]):
226 237 maxHei = dataOut.heightList[-1]
227 238
228 239 minIndex = 0
229 240 maxIndex = 0
230 241 heights = dataOut.heightList
231 242
232 243 inda = numpy.where(heights >= minHei)
233 244 indb = numpy.where(heights <= maxHei)
234 245
235 246 try:
236 247 minIndex = inda[0][0]
237 248 except:
238 249 minIndex = 0
239 250
240 251 try:
241 252 maxIndex = indb[0][-1]
242 253 except:
243 254 maxIndex = len(heights)
244 255
245 256 self.selectHeightsByIndex(minIndex, maxIndex)
246 257
247 258 #update Processing Header:
248 259 dataOut.processingHeaderObj.heightList = dataOut.heightList
249 260
250 261
251 262
252 263 return dataOut
253 264
254 265 def selectHeightsByIndex(self, minIndex, maxIndex):
255 266 """
256 267 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
257 268 minIndex <= index <= maxIndex
258 269
259 270 Input:
260 271 minIndex : valor de indice minimo de altura a considerar
261 272 maxIndex : valor de indice maximo de altura a considerar
262 273
263 274 Affected:
264 275 self.dataOut.data
265 276 self.dataOut.heightList
266 277
267 278 Return:
268 279 1 si el metodo se ejecuto con exito caso contrario devuelve 0
269 280 """
270 281
271 282 if self.dataOut.type == 'Voltage':
272 283 if (minIndex < 0) or (minIndex > maxIndex):
273 284 raise ValueError("Height index range (%d,%d) is not valid" % (minIndex, maxIndex))
274 285
275 286 if (maxIndex >= self.dataOut.nHeights):
276 287 maxIndex = self.dataOut.nHeights
277 288
278 289 #voltage
279 290 if self.dataOut.flagDataAsBlock:
280 291 """
281 292 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
282 293 """
283 294 data = self.dataOut.data[:,:, minIndex:maxIndex]
284 295 else:
285 296 data = self.dataOut.data[:, minIndex:maxIndex]
286 297
287 298 # firstHeight = self.dataOut.heightList[minIndex]
288 299
289 300 self.dataOut.data = data
290 301 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
291 302
292 303 if self.dataOut.nHeights <= 1:
293 304 raise ValueError("selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights))
294 305 elif self.dataOut.type == 'Spectra':
295 306 if (minIndex < 0) or (minIndex > maxIndex):
296 307 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (
297 308 minIndex, maxIndex))
298 309
299 310 if (maxIndex >= self.dataOut.nHeights):
300 311 maxIndex = self.dataOut.nHeights - 1
301 312
302 313 # Spectra
303 314 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
304 315
305 316 data_cspc = None
306 317 if self.dataOut.data_cspc is not None:
307 318 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
308 319
309 320 data_dc = None
310 321 if self.dataOut.data_dc is not None:
311 322 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
312 323
313 324 self.dataOut.data_spc = data_spc
314 325 self.dataOut.data_cspc = data_cspc
315 326 self.dataOut.data_dc = data_dc
316 327
317 328 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
318 329
319 330 return 1
320 331
321 332
322 333 class filterByHeights(Operation):
323 334 ifConfig=False
324 335 deltaHeight = None
325 336 newdelta=None
326 337 newheights=None
327 338 r=None
328 339 h0=None
329 340 nHeights=None
330 341 def run(self, dataOut, window):
331 342
332 343
333 344 # print("1",dataOut.data.shape)
334 345 # print(dataOut.nHeights)
335 346 if window == None:
336 347 window = (dataOut.radarControllerHeaderObj.txA/dataOut.radarControllerHeaderObj.nBaud) / self.deltaHeight
337 348
338 349 if not self.ifConfig: #and dataOut.useInputBuffer:
339 350 self.deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
340 351 self.ifConfig = True
341 352 self.newdelta = self.deltaHeight * window
342 353 self.r = dataOut.nHeights % window
343 354 self.newheights = (dataOut.nHeights-self.r)/window
344 355 self.h0 = dataOut.heightList[0]
345 356 self.nHeights = dataOut.nHeights
346 357 if self.newheights <= 1:
347 358 raise ValueError("filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(dataOut.nHeights, window))
348 359
349 360 if dataOut.flagDataAsBlock:
350 361 """
351 362 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
352 363 """
353 364 buffer = dataOut.data[:, :, 0:int(self.nHeights-self.r)]
354 365 buffer = buffer.reshape(dataOut.nChannels, dataOut.nProfiles, int(self.nHeights/window), window)
355 366 buffer = numpy.sum(buffer,3)
356 367
357 368 else:
358 369 buffer = dataOut.data[:,0:int(self.nHeights-self.r)]
359 370 buffer = buffer.reshape(dataOut.nChannels,int(self.nHeights/window),int(window))
360 371 buffer = numpy.sum(buffer,2)
361 372
362 373 dataOut.data = buffer
363 374 dataOut.heightList = self.h0 + numpy.arange( self.newheights )*self.newdelta
364 375 dataOut.windowOfFilter = window
365 376
366 377 #update Processing Header:
367 378 dataOut.processingHeaderObj.heightList = dataOut.heightList
368 379 dataOut.processingHeaderObj.nWindows = window
369 380
370 381 return dataOut
371 382
372 383
373 384
374 385 class setH0(Operation):
375 386
376 387 def run(self, dataOut, h0, deltaHeight = None):
377 388
378 389 if not deltaHeight:
379 390 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
380 391
381 392 nHeights = dataOut.nHeights
382 393
383 394 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
384 395
385 396 dataOut.heightList = newHeiRange
386 397
387 398 #update Processing Header:
388 399 dataOut.processingHeaderObj.heightList = dataOut.heightList
389 400
390 401 return dataOut
391 402
392 403
393 404 class deFlip(Operation):
394 405
395 406 def run(self, dataOut, channelList = []):
396 407
397 408 data = dataOut.data.copy()
398 409
399 410 if dataOut.flagDataAsBlock:
400 411 flip = self.flip
401 412 profileList = list(range(dataOut.nProfiles))
402 413
403 414 if not channelList:
404 415 for thisProfile in profileList:
405 416 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
406 417 flip *= -1.0
407 418 else:
408 419 for thisChannel in channelList:
409 420 if thisChannel not in dataOut.channelList:
410 421 continue
411 422
412 423 for thisProfile in profileList:
413 424 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
414 425 flip *= -1.0
415 426
416 427 self.flip = flip
417 428
418 429 else:
419 430 if not channelList:
420 431 data[:,:] = data[:,:]*self.flip
421 432 else:
422 433 for thisChannel in channelList:
423 434 if thisChannel not in dataOut.channelList:
424 435 continue
425 436
426 437 data[thisChannel,:] = data[thisChannel,:]*self.flip
427 438
428 439 self.flip *= -1.
429 440
430 441 dataOut.data = data
431 442
432 443 return dataOut
433 444
434 445
435 446 class setAttribute(Operation):
436 447 '''
437 448 Set an arbitrary attribute(s) to dataOut
438 449 '''
439 450
440 451 def __init__(self):
441 452
442 453 Operation.__init__(self)
443 454 self._ready = False
444 455
445 456 def run(self, dataOut, **kwargs):
446 457
447 458 for key, value in kwargs.items():
448 459 setattr(dataOut, key, value)
449 460
450 461 return dataOut
451 462
452 463
453 464 @MPDecorator
454 465 class printAttribute(Operation):
455 466 '''
456 467 Print an arbitrary attribute of dataOut
457 468 '''
458 469
459 470 def __init__(self):
460 471
461 472 Operation.__init__(self)
462 473
463 474 def run(self, dataOut, attributes):
464 475
465 476 if isinstance(attributes, str):
466 477 attributes = [attributes]
467 478 for attr in attributes:
468 479 if hasattr(dataOut, attr):
469 480 log.log(getattr(dataOut, attr), attr)
470 481
471 482 class cleanHeightsInterf(Operation):
472 483 __slots__ =('heights_indx', 'repeats', 'step', 'factor', 'idate', 'idxs','config','wMask')
473 484 def __init__(self):
474 485 self.repeats = 0
475 486 self.factor=1
476 487 self.wMask = None
477 488 self.config = False
478 489 self.idxs = None
479 490 self.heights_indx = None
480 491
481 492 def run(self, dataOut, heightsList, repeats=0, step=0, factor=1, idate=None, startH=None, endH=None):
482 493
483 494 #print(dataOut.data.shape)
484 495
485 496 startTime = datetime.datetime.combine(idate,startH)
486 497 endTime = datetime.datetime.combine(idate,endH)
487 498 currentTime = datetime.datetime.fromtimestamp(dataOut.utctime)
488 499
489 500 if currentTime < startTime or currentTime > endTime:
490 501 return dataOut
491 502 if not self.config:
492 503
493 504 #print(wMask)
494 505 heights = [float(hei) for hei in heightsList]
495 506 for r in range(repeats):
496 507 heights += [ (h+(step*(r+1))) for h in heights]
497 508 #print(heights)
498 509 heiList = dataOut.heightList
499 510 self.heights_indx = [getHei_index(h,h,heiList)[0] for h in heights]
500 511
501 512 self.wMask = numpy.asarray(factor)
502 513 self.wMask = numpy.tile(self.wMask,(repeats+2))
503 514 self.config = True
504 515
505 516 """
506 517 getNoisebyHildebrand(self, channel=None, ymin_index=None, ymax_index=None)
507 518 """
508 519 #print(self.noise =10*numpy.log10(dataOut.getNoisebyHildebrand(ymin_index=self.min_ref, ymax_index=self.max_ref)))
509 520
510 521
511 522 for ch in range(dataOut.data.shape[0]):
512 523 i = 0
513 524
514 525
515 526 for hei in self.heights_indx:
516 527 h = hei - 1
517 528
518 529
519 530 if dataOut.data.ndim < 3:
520 531 module = numpy.absolute(dataOut.data[ch,h])
521 532 prev_h1 = numpy.absolute(dataOut.data[ch,h-1])
522 533 dataOut.data[ch,h] = (dataOut.data[ch,h])/module * prev_h1
523 534
524 535 #dataOut.data[ch,hei-1] = (dataOut.data[ch,hei-1])*self.wMask[i]
525 536 else:
526 537 module = numpy.absolute(dataOut.data[ch,:,h])
527 538 prev_h1 = numpy.absolute(dataOut.data[ch,:,h-1])
528 539 dataOut.data[ch,:,h] = (dataOut.data[ch,:,h])/module * prev_h1
529 540 #dataOut.data[ch,:,hei-1] = (dataOut.data[ch,:,hei-1])*self.wMask[i]
530 541 #print("done")
531 542 i += 1
532 543
533 544
534 545 return dataOut
535 546
536 547
537 548
538 549 class interpolateHeights(Operation):
539 550
540 551 def run(self, dataOut, topLim, botLim):
541 552 #69 al 72 para julia
542 553 #82-84 para meteoros
543 554 if len(numpy.shape(dataOut.data))==2:
544 555 sampInterp = (dataOut.data[:,botLim-1] + dataOut.data[:,topLim+1])/2
545 556 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
546 557 #dataOut.data[:,botLim:limSup+1] = sampInterp
547 558 dataOut.data[:,botLim:topLim+1] = sampInterp
548 559 else:
549 560 nHeights = dataOut.data.shape[2]
550 561 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
551 562 y = dataOut.data[:,:,list(range(botLim))+list(range(topLim+1,nHeights))]
552 563 f = interpolate.interp1d(x, y, axis = 2)
553 564 xnew = numpy.arange(botLim,topLim+1)
554 565 ynew = f(xnew)
555 566 dataOut.data[:,:,botLim:topLim+1] = ynew
556 567
557 568 return dataOut
558 569
559 570
560 571 class CohInt(Operation):
561 572
562 573 isConfig = False
563 574 __profIndex = 0
564 575 __byTime = False
565 576 __initime = None
566 577 __lastdatatime = None
567 578 __integrationtime = None
568 579 __buffer = None
569 580 __bufferStride = []
570 581 __dataReady = False
571 582 __profIndexStride = 0
572 583 __dataToPutStride = False
573 584 n = None
574 585
575 586 def __init__(self, **kwargs):
576 587
577 588 Operation.__init__(self, **kwargs)
578 589
579 590 def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False):
580 591 """
581 592 Set the parameters of the integration class.
582 593
583 594 Inputs:
584 595
585 596 n : Number of coherent integrations
586 597 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
587 598 overlapping :
588 599 """
589 600
590 601 self.__initime = None
591 602 self.__lastdatatime = 0
592 603 self.__buffer = None
593 604 self.__dataReady = False
594 605 self.byblock = byblock
595 606 self.stride = stride
596 607
597 608 if n == None and timeInterval == None:
598 609 raise ValueError("n or timeInterval should be specified ...")
599 610
600 611 if n != None:
601 612 self.n = n
602 613 self.__byTime = False
603 614 else:
604 615 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
605 616 self.n = 9999
606 617 self.__byTime = True
607 618
608 619 if overlapping:
609 620 self.__withOverlapping = True
610 621 self.__buffer = None
611 622 else:
612 623 self.__withOverlapping = False
613 624 self.__buffer = 0
614 625
615 626 self.__profIndex = 0
616 627
617 628 def putData(self, data):
618 629
619 630 """
620 631 Add a profile to the __buffer and increase in one the __profileIndex
621 632
622 633 """
623 634
624 635 if not self.__withOverlapping:
625 636 self.__buffer += data.copy()
626 637 self.__profIndex += 1
627 638 return
628 639
629 640 #Overlapping data
630 641 nChannels, nHeis = data.shape
631 642 data = numpy.reshape(data, (1, nChannels, nHeis))
632 643
633 644 #If the buffer is empty then it takes the data value
634 645 if self.__buffer is None:
635 646 self.__buffer = data
636 647 self.__profIndex += 1
637 648 return
638 649
639 650 #If the buffer length is lower than n then stakcing the data value
640 651 if self.__profIndex < self.n:
641 652 self.__buffer = numpy.vstack((self.__buffer, data))
642 653 self.__profIndex += 1
643 654 return
644 655
645 656 #If the buffer length is equal to n then replacing the last buffer value with the data value
646 657 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
647 658 self.__buffer[self.n-1] = data
648 659 self.__profIndex = self.n
649 660 return
650 661
651 662
652 663 def pushData(self):
653 664 """
654 665 Return the sum of the last profiles and the profiles used in the sum.
655 666
656 667 Affected:
657 668
658 669 self.__profileIndex
659 670
660 671 """
661 672
662 673 if not self.__withOverlapping:
663 674 data = self.__buffer
664 675 n = self.__profIndex
665 676
666 677 self.__buffer = 0
667 678 self.__profIndex = 0
668 679
669 680 return data, n
670 681
671 682 #Integration with Overlapping
672 683 data = numpy.sum(self.__buffer, axis=0)
673 684 # print data
674 685 # raise
675 686 n = self.__profIndex
676 687
677 688 return data, n
678 689
679 690 def byProfiles(self, data):
680 691
681 692 self.__dataReady = False
682 693 avgdata = None
683 694 # n = None
684 695 # print data
685 696 # raise
686 697 self.putData(data)
687 698
688 699 if self.__profIndex == self.n:
689 700 avgdata, n = self.pushData()
690 701 self.__dataReady = True
691 702
692 703 return avgdata
693 704
694 705 def byTime(self, data, datatime):
695 706
696 707 self.__dataReady = False
697 708 avgdata = None
698 709 n = None
699 710
700 711 self.putData(data)
701 712
702 713 if (datatime - self.__initime) >= self.__integrationtime:
703 714 avgdata, n = self.pushData()
704 715 self.n = n
705 716 self.__dataReady = True
706 717
707 718 return avgdata
708 719
709 720 def integrateByStride(self, data, datatime):
710 721 # print data
711 722 if self.__profIndex == 0:
712 723 self.__buffer = [[data.copy(), datatime]]
713 724 else:
714 725 self.__buffer.append([data.copy(),datatime])
715 726 self.__profIndex += 1
716 727 self.__dataReady = False
717 728
718 729 if self.__profIndex == self.n * self.stride :
719 730 self.__dataToPutStride = True
720 731 self.__profIndexStride = 0
721 732 self.__profIndex = 0
722 733 self.__bufferStride = []
723 734 for i in range(self.stride):
724 735 current = self.__buffer[i::self.stride]
725 736 data = numpy.sum([t[0] for t in current], axis=0)
726 737 avgdatatime = numpy.average([t[1] for t in current])
727 738 # print data
728 739 self.__bufferStride.append((data, avgdatatime))
729 740
730 741 if self.__dataToPutStride:
731 742 self.__dataReady = True
732 743 self.__profIndexStride += 1
733 744 if self.__profIndexStride == self.stride:
734 745 self.__dataToPutStride = False
735 746 # print self.__bufferStride[self.__profIndexStride - 1]
736 747 # raise
737 748 return self.__bufferStride[self.__profIndexStride - 1]
738 749
739 750
740 751 return None, None
741 752
742 753 def integrate(self, data, datatime=None):
743 754
744 755 if self.__initime == None:
745 756 self.__initime = datatime
746 757
747 758 if self.__byTime:
748 759 avgdata = self.byTime(data, datatime)
749 760 else:
750 761 avgdata = self.byProfiles(data)
751 762
752 763
753 764 self.__lastdatatime = datatime
754 765
755 766 if avgdata is None:
756 767 return None, None
757 768
758 769 avgdatatime = self.__initime
759 770
760 771 deltatime = datatime - self.__lastdatatime
761 772
762 773 if not self.__withOverlapping:
763 774 self.__initime = datatime
764 775 else:
765 776 self.__initime += deltatime
766 777
767 778 return avgdata, avgdatatime
768 779
769 780 def integrateByBlock(self, dataOut):
770 781
771 782 times = int(dataOut.data.shape[1]/self.n)
772 783 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
773 784
774 785 id_min = 0
775 786 id_max = self.n
776 787
777 788 for i in range(times):
778 789 junk = dataOut.data[:,id_min:id_max,:]
779 790 avgdata[:,i,:] = junk.sum(axis=1)
780 791 id_min += self.n
781 792 id_max += self.n
782 793
783 794 timeInterval = dataOut.ippSeconds*self.n
784 795 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
785 796 self.__dataReady = True
786 797 return avgdata, avgdatatime
787 798
788 799 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
789 800
790 801 if not self.isConfig:
791 802 self.setup(n=n, stride=stride, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
792 803 self.isConfig = True
793 804
794 805 if dataOut.flagDataAsBlock:
795 806 """
796 807 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
797 808 """
798 809 avgdata, avgdatatime = self.integrateByBlock(dataOut)
799 810 dataOut.nProfiles /= self.n
800 811 else:
801 812 if stride is None:
802 813 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
803 814 else:
804 815 avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
805 816
806 817
807 818 # dataOut.timeInterval *= n
808 819 dataOut.flagNoData = True
809 820
810 821 if self.__dataReady:
811 822 dataOut.data = avgdata
812 823 if not dataOut.flagCohInt:
813 824 dataOut.nCohInt *= self.n
814 825 dataOut.flagCohInt = True
815 826 dataOut.utctime = avgdatatime
816 827 # print avgdata, avgdatatime
817 828 # raise
818 829 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
819 830 dataOut.flagNoData = False
820 831
821 832 #update Processing Header:
822 833 dataOut.processingHeaderObj.nCohInt = dataOut.nCohInt
823 834
824 835
825 836 return dataOut
826 837
827 838 class Decoder(Operation):
828 839
829 840 isConfig = False
830 841 __profIndex = 0
831 842
832 843 code = None
833 844
834 845 nCode = None
835 846 nBaud = None
836 847
837 848 def __init__(self, **kwargs):
838 849
839 850 Operation.__init__(self, **kwargs)
840 851
841 852 self.times = None
842 853 self.osamp = None
843 854 # self.__setValues = False
844 855 self.isConfig = False
845 856 self.setupReq = False
846 857 def setup(self, code, osamp, dataOut):
847 858
848 859 self.__profIndex = 0
849 860
850 861 self.code = code
851 862
852 863 self.nCode = len(code)
853 864 self.nBaud = len(code[0])
854 865 if (osamp != None) and (osamp >1):
855 866 self.osamp = osamp
856 867 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
857 868 self.nBaud = self.nBaud*self.osamp
858 869
859 870 self.__nChannels = dataOut.nChannels
860 871 self.__nProfiles = dataOut.nProfiles
861 872 self.__nHeis = dataOut.nHeights
862 873
863 874 if self.__nHeis < self.nBaud:
864 875 raise ValueError('Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud))
865 876
866 877 #Frequency
867 878 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
868 879
869 880 __codeBuffer[:,0:self.nBaud] = self.code
870 881
871 882 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
872 883
873 884 if dataOut.flagDataAsBlock:
874 885
875 886 self.ndatadec = self.__nHeis #- self.nBaud + 1
876 887
877 888 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
878 889
879 890 else:
880 891
881 892 #Time
882 893 self.ndatadec = self.__nHeis #- self.nBaud + 1
883 894
884 895 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
885 896
886 897 def __convolutionInFreq(self, data):
887 898
888 899 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
889 900
890 901 fft_data = numpy.fft.fft(data, axis=1)
891 902
892 903 conv = fft_data*fft_code
893 904
894 905 data = numpy.fft.ifft(conv,axis=1)
895 906
896 907 return data
897 908
898 909 def __convolutionInFreqOpt(self, data):
899 910
900 911 raise NotImplementedError
901 912
902 913 def __convolutionInTime(self, data):
903 914
904 915 code = self.code[self.__profIndex]
905 916 for i in range(self.__nChannels):
906 917 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
907 918
908 919 return self.datadecTime
909 920
910 921 def __convolutionByBlockInTime(self, data):
911 922
912 923 repetitions = int(self.__nProfiles / self.nCode)
913 924 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
914 925 junk = junk.flatten()
915 926 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
916 927 profilesList = range(self.__nProfiles)
917 928
918 929 for i in range(self.__nChannels):
919 930 for j in profilesList:
920 931 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
921 932 return self.datadecTime
922 933
923 934 def __convolutionByBlockInFreq(self, data):
924 935
925 936 raise NotImplementedError("Decoder by frequency fro Blocks not implemented")
926 937
927 938
928 939 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
929 940
930 941 fft_data = numpy.fft.fft(data, axis=2)
931 942
932 943 conv = fft_data*fft_code
933 944
934 945 data = numpy.fft.ifft(conv,axis=2)
935 946
936 947 return data
937 948
938 949
939 950 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
940 951
941 952 if dataOut.flagDecodeData:
942 953 print("This data is already decoded, recoding again ...")
943 954
944 955 if not self.isConfig:
945 956
946 957 if code is None:
947 958 if dataOut.code is None:
948 959 raise ValueError("Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type)
949 960
950 961 code = dataOut.code
951 962 else:
952 963 code = numpy.array(code).reshape(nCode,nBaud)
953 964 self.setup(code, osamp, dataOut)
954 965
955 966 self.isConfig = True
956 967
957 968 if mode == 3:
958 969 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
959 970
960 971 if times != None:
961 972 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
962 973
963 974 if self.code is None:
964 975 print("Fail decoding: Code is not defined.")
965 976 return
966 977
967 978 self.__nProfiles = dataOut.nProfiles
968 979 datadec = None
969 980
970 981 if mode == 3:
971 982 mode = 0
972 983
973 984 if dataOut.flagDataAsBlock:
974 985 """
975 986 Decoding when data have been read as block,
976 987 """
977 988
978 989 if mode == 0:
979 990 datadec = self.__convolutionByBlockInTime(dataOut.data)
980 991 if mode == 1:
981 992 datadec = self.__convolutionByBlockInFreq(dataOut.data)
982 993 else:
983 994 """
984 995 Decoding when data have been read profile by profile
985 996 """
986 997 if mode == 0:
987 998 datadec = self.__convolutionInTime(dataOut.data)
988 999
989 1000 if mode == 1:
990 1001 datadec = self.__convolutionInFreq(dataOut.data)
991 1002
992 1003 if mode == 2:
993 1004 datadec = self.__convolutionInFreqOpt(dataOut.data)
994 1005
995 1006 if datadec is None:
996 1007 raise ValueError("Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode)
997 1008
998 1009 dataOut.code = self.code
999 1010 dataOut.nCode = self.nCode
1000 1011 dataOut.nBaud = self.nBaud
1001 1012
1002 1013 dataOut.data = datadec
1003 1014 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
1004 1015 dataOut.flagDecodeData = True #asumo q la data esta decodificada
1005 1016
1006 1017
1007 1018 #update Processing Header:
1008 1019 dataOut.radarControllerHeaderObj.code = self.code
1009 1020 dataOut.radarControllerHeaderObj.nCode = self.nCode
1010 1021 dataOut.radarControllerHeaderObj.nBaud = self.nBaud
1011 1022 dataOut.radarControllerHeaderObj.nOsamp = osamp
1012 1023 #update Processing Header:
1013 1024 dataOut.processingHeaderObj.heightList = dataOut.heightList
1014 1025 dataOut.processingHeaderObj.heightResolution = dataOut.heightList[1]-dataOut.heightList[0]
1015 1026
1016 1027 if self.__profIndex == self.nCode-1:
1017 1028 self.__profIndex = 0
1018 1029 return dataOut
1019 1030
1020 1031 self.__profIndex += 1
1021 1032
1022 1033 return dataOut
1023 1034
1024 1035 class ProfileConcat(Operation):
1025 1036
1026 1037 isConfig = False
1027 1038 buffer = None
1028 1039
1029 1040 def __init__(self, **kwargs):
1030 1041
1031 1042 Operation.__init__(self, **kwargs)
1032 1043 self.profileIndex = 0
1033 1044
1034 1045 def reset(self):
1035 1046 self.buffer = numpy.zeros_like(self.buffer)
1036 1047 self.start_index = 0
1037 1048 self.times = 1
1038 1049
1039 1050 def setup(self, data, m, n=1):
1040 1051 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
1041 1052 self.nHeights = data.shape[1]#.nHeights
1042 1053 self.start_index = 0
1043 1054 self.times = 1
1044 1055
1045 1056 def concat(self, data):
1046 1057
1047 1058 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
1048 1059 self.start_index = self.start_index + self.nHeights
1049 1060
1050 1061 def run(self, dataOut, m):
1051 1062 dataOut.flagNoData = True
1052 1063
1053 1064 if not self.isConfig:
1054 1065 self.setup(dataOut.data, m, 1)
1055 1066 self.isConfig = True
1056 1067
1057 1068 if dataOut.flagDataAsBlock:
1058 1069 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
1059 1070
1060 1071 else:
1061 1072 self.concat(dataOut.data)
1062 1073 self.times += 1
1063 1074 if self.times > m:
1064 1075 dataOut.data = self.buffer
1065 1076 self.reset()
1066 1077 dataOut.flagNoData = False
1067 1078 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
1068 1079 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1069 1080 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
1070 1081 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
1071 1082 dataOut.ippSeconds *= m
1072 1083
1073 1084 #update Processing Header:
1074 1085 dataOut.processingHeaderObj.heightList = dataOut.heightList
1075 1086 dataOut.processingHeaderObj.ipp = dataOut.ippSeconds
1076 1087
1077 1088 return dataOut
1078 1089
1079 1090 class ProfileSelector(Operation):
1080 1091
1081 1092 profileIndex = None
1082 1093 # Tamanho total de los perfiles
1083 1094 nProfiles = None
1084 1095
1085 1096 def __init__(self, **kwargs):
1086 1097
1087 1098 Operation.__init__(self, **kwargs)
1088 1099 self.profileIndex = 0
1089 1100
1090 1101 def incProfileIndex(self):
1091 1102
1092 1103 self.profileIndex += 1
1093 1104
1094 1105 if self.profileIndex >= self.nProfiles:
1095 1106 self.profileIndex = 0
1096 1107
1097 1108 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
1098 1109
1099 1110 if profileIndex < minIndex:
1100 1111 return False
1101 1112
1102 1113 if profileIndex > maxIndex:
1103 1114 return False
1104 1115
1105 1116 return True
1106 1117
1107 1118 def isThisProfileInList(self, profileIndex, profileList):
1108 1119
1109 1120 if profileIndex not in profileList:
1110 1121 return False
1111 1122
1112 1123 return True
1113 1124
1114 1125 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
1115 1126
1116 1127 """
1117 1128 ProfileSelector:
1118 1129
1119 1130 Inputs:
1120 1131 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
1121 1132
1122 1133 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
1123 1134
1124 1135 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
1125 1136
1126 1137 """
1127 1138
1128 1139 if rangeList is not None:
1129 1140 if type(rangeList[0]) not in (tuple, list):
1130 1141 rangeList = [rangeList]
1131 1142
1132 1143 dataOut.flagNoData = True
1133 1144
1134 1145 if dataOut.flagDataAsBlock:
1135 1146 """
1136 1147 data dimension = [nChannels, nProfiles, nHeis]
1137 1148 """
1138 1149 if profileList != None:
1139 1150 dataOut.data = dataOut.data[:,profileList,:]
1140 1151
1141 1152 if profileRangeList != None:
1142 1153 minIndex = profileRangeList[0]
1143 1154 maxIndex = profileRangeList[1]
1144 1155 profileList = list(range(minIndex, maxIndex+1))
1145 1156
1146 1157 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
1147 1158
1148 1159 if rangeList != None:
1149 1160
1150 1161 profileList = []
1151 1162
1152 1163 for thisRange in rangeList:
1153 1164 minIndex = thisRange[0]
1154 1165 maxIndex = thisRange[1]
1155 1166
1156 1167 profileList.extend(list(range(minIndex, maxIndex+1)))
1157 1168
1158 1169 dataOut.data = dataOut.data[:,profileList,:]
1159 1170
1160 1171 dataOut.nProfiles = len(profileList)
1161 1172 dataOut.profileIndex = dataOut.nProfiles - 1
1162 1173 dataOut.flagNoData = False
1163 1174
1164 1175 return dataOut
1165 1176
1166 1177 """
1167 1178 data dimension = [nChannels, nHeis]
1168 1179 """
1169 1180
1170 1181 if profileList != None:
1171 1182
1172 1183 if self.isThisProfileInList(dataOut.profileIndex, profileList):
1173 1184
1174 1185 self.nProfiles = len(profileList)
1175 1186 dataOut.nProfiles = self.nProfiles
1176 1187 dataOut.profileIndex = self.profileIndex
1177 1188 dataOut.flagNoData = False
1178 1189
1179 1190 self.incProfileIndex()
1180 1191 return dataOut
1181 1192
1182 1193 if profileRangeList != None:
1183 1194
1184 1195 minIndex = profileRangeList[0]
1185 1196 maxIndex = profileRangeList[1]
1186 1197
1187 1198 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1188 1199
1189 1200 self.nProfiles = maxIndex - minIndex + 1
1190 1201 dataOut.nProfiles = self.nProfiles
1191 1202 dataOut.profileIndex = self.profileIndex
1192 1203 dataOut.flagNoData = False
1193 1204
1194 1205 self.incProfileIndex()
1195 1206 return dataOut
1196 1207
1197 1208 if rangeList != None:
1198 1209
1199 1210 nProfiles = 0
1200 1211
1201 1212 for thisRange in rangeList:
1202 1213 minIndex = thisRange[0]
1203 1214 maxIndex = thisRange[1]
1204 1215
1205 1216 nProfiles += maxIndex - minIndex + 1
1206 1217
1207 1218 for thisRange in rangeList:
1208 1219
1209 1220 minIndex = thisRange[0]
1210 1221 maxIndex = thisRange[1]
1211 1222
1212 1223 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1213 1224
1214 1225 self.nProfiles = nProfiles
1215 1226 dataOut.nProfiles = self.nProfiles
1216 1227 dataOut.profileIndex = self.profileIndex
1217 1228 dataOut.flagNoData = False
1218 1229
1219 1230 self.incProfileIndex()
1220 1231
1221 1232 break
1222 1233
1223 1234 return dataOut
1224 1235
1225 1236
1226 1237 if beam != None: #beam is only for AMISR data
1227 1238 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
1228 1239 dataOut.flagNoData = False
1229 1240 dataOut.profileIndex = self.profileIndex
1230 1241
1231 1242 self.incProfileIndex()
1232 1243
1233 1244 return dataOut
1234 1245
1235 1246 raise ValueError("ProfileSelector needs profileList, profileRangeList or rangeList parameter")
1236 1247
1237 1248
1238 1249 class Reshaper(Operation):
1239 1250
1240 1251 def __init__(self, **kwargs):
1241 1252
1242 1253 Operation.__init__(self, **kwargs)
1243 1254
1244 1255 self.__buffer = None
1245 1256 self.__nitems = 0
1246 1257
1247 1258 def __appendProfile(self, dataOut, nTxs):
1248 1259
1249 1260 if self.__buffer is None:
1250 1261 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
1251 1262 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
1252 1263
1253 1264 ini = dataOut.nHeights * self.__nitems
1254 1265 end = ini + dataOut.nHeights
1255 1266
1256 1267 self.__buffer[:, ini:end] = dataOut.data
1257 1268
1258 1269 self.__nitems += 1
1259 1270
1260 1271 return int(self.__nitems*nTxs)
1261 1272
1262 1273 def __getBuffer(self):
1263 1274
1264 1275 if self.__nitems == int(1./self.__nTxs):
1265 1276
1266 1277 self.__nitems = 0
1267 1278
1268 1279 return self.__buffer.copy()
1269 1280
1270 1281 return None
1271 1282
1272 1283 def __checkInputs(self, dataOut, shape, nTxs):
1273 1284
1274 1285 if shape is None and nTxs is None:
1275 1286 raise ValueError("Reshaper: shape of factor should be defined")
1276 1287
1277 1288 if nTxs:
1278 1289 if nTxs < 0:
1279 1290 raise ValueError("nTxs should be greater than 0")
1280 1291
1281 1292 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
1282 1293 raise ValueError("nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs)))
1283 1294
1284 1295 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
1285 1296
1286 1297 return shape, nTxs
1287 1298
1288 1299 if len(shape) != 2 and len(shape) != 3:
1289 1300 raise ValueError("shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights))
1290 1301
1291 1302 if len(shape) == 2:
1292 1303 shape_tuple = [dataOut.nChannels]
1293 1304 shape_tuple.extend(shape)
1294 1305 else:
1295 1306 shape_tuple = list(shape)
1296 1307
1297 1308 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1298 1309
1299 1310 return shape_tuple, nTxs
1300 1311
1301 1312 def run(self, dataOut, shape=None, nTxs=None):
1302 1313
1303 1314 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1304 1315
1305 1316 dataOut.flagNoData = True
1306 1317 profileIndex = None
1307 1318
1308 1319 if dataOut.flagDataAsBlock:
1309 1320
1310 1321 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1311 1322 dataOut.flagNoData = False
1312 1323
1313 1324 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1314 1325
1315 1326 else:
1316 1327
1317 1328 if self.__nTxs < 1:
1318 1329
1319 1330 self.__appendProfile(dataOut, self.__nTxs)
1320 1331 new_data = self.__getBuffer()
1321 1332
1322 1333 if new_data is not None:
1323 1334 dataOut.data = new_data
1324 1335 dataOut.flagNoData = False
1325 1336
1326 1337 profileIndex = dataOut.profileIndex*nTxs
1327 1338
1328 1339 else:
1329 1340 raise ValueError("nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)")
1330 1341
1331 1342 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1332 1343
1333 1344 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1334 1345
1335 1346 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1336 1347
1337 1348 dataOut.profileIndex = profileIndex
1338 1349
1339 1350 dataOut.ippSeconds /= self.__nTxs
1340 1351
1341 1352 return dataOut
1342 1353
1343 1354 class SplitProfiles(Operation):
1344 1355
1345 1356 def __init__(self, **kwargs):
1346 1357
1347 1358 Operation.__init__(self, **kwargs)
1348 1359
1349 1360 def run(self, dataOut, n):
1350 1361
1351 1362 dataOut.flagNoData = True
1352 1363 profileIndex = None
1353 1364
1354 1365 if dataOut.flagDataAsBlock:
1355 1366
1356 1367 #nchannels, nprofiles, nsamples
1357 1368 shape = dataOut.data.shape
1358 1369
1359 1370 if shape[2] % n != 0:
1360 1371 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
1361 1372
1362 1373 new_shape = shape[0], shape[1]*n, int(shape[2]/n)
1363 1374
1364 1375 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1365 1376 dataOut.flagNoData = False
1366 1377
1367 1378 profileIndex = int(dataOut.nProfiles/n) - 1
1368 1379
1369 1380 else:
1370 1381
1371 1382 raise ValueError("Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)")
1372 1383
1373 1384 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1374 1385
1375 1386 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1376 1387
1377 1388 dataOut.nProfiles = int(dataOut.nProfiles*n)
1378 1389
1379 1390 dataOut.profileIndex = profileIndex
1380 1391
1381 1392 dataOut.ippSeconds /= n
1382 1393
1383 1394 return dataOut
1384 1395
1385 1396 class CombineProfiles(Operation):
1386 1397 def __init__(self, **kwargs):
1387 1398
1388 1399 Operation.__init__(self, **kwargs)
1389 1400
1390 1401 self.__remData = None
1391 1402 self.__profileIndex = 0
1392 1403
1393 1404 def run(self, dataOut, n):
1394 1405
1395 1406 dataOut.flagNoData = True
1396 1407 profileIndex = None
1397 1408
1398 1409 if dataOut.flagDataAsBlock:
1399 1410
1400 1411 #nchannels, nprofiles, nsamples
1401 1412 shape = dataOut.data.shape
1402 1413 new_shape = shape[0], shape[1]/n, shape[2]*n
1403 1414
1404 1415 if shape[1] % n != 0:
1405 1416 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[1]))
1406 1417
1407 1418 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1408 1419 dataOut.flagNoData = False
1409 1420
1410 1421 profileIndex = int(dataOut.nProfiles*n) - 1
1411 1422
1412 1423 else:
1413 1424
1414 1425 #nchannels, nsamples
1415 1426 if self.__remData is None:
1416 1427 newData = dataOut.data
1417 1428 else:
1418 1429 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1419 1430
1420 1431 self.__profileIndex += 1
1421 1432
1422 1433 if self.__profileIndex < n:
1423 1434 self.__remData = newData
1424 1435 #continue
1425 1436 return
1426 1437
1427 1438 self.__profileIndex = 0
1428 1439 self.__remData = None
1429 1440
1430 1441 dataOut.data = newData
1431 1442 dataOut.flagNoData = False
1432 1443
1433 1444 profileIndex = dataOut.profileIndex/n
1434 1445
1435 1446
1436 1447 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1437 1448
1438 1449 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1439 1450
1440 1451 dataOut.nProfiles = int(dataOut.nProfiles/n)
1441 1452
1442 1453 dataOut.profileIndex = profileIndex
1443 1454
1444 1455 dataOut.ippSeconds *= n
1445 1456
1446 1457 return dataOut
1447 1458
1448 1459 class PulsePairVoltage(Operation):
1449 1460 '''
1450 1461 Function PulsePair(Signal Power, Velocity)
1451 1462 The real component of Lag[0] provides Intensity Information
1452 1463 The imag component of Lag[1] Phase provides Velocity Information
1453 1464
1454 1465 Configuration Parameters:
1455 1466 nPRF = Number of Several PRF
1456 1467 theta = Degree Azimuth angel Boundaries
1457 1468
1458 1469 Input:
1459 1470 self.dataOut
1460 1471 lag[N]
1461 1472 Affected:
1462 1473 self.dataOut.spc
1463 1474 '''
1464 1475 isConfig = False
1465 1476 __profIndex = 0
1466 1477 __initime = None
1467 1478 __lastdatatime = None
1468 1479 __buffer = None
1469 1480 noise = None
1470 1481 __dataReady = False
1471 1482 n = None
1472 1483 __nch = 0
1473 1484 __nHeis = 0
1474 1485 removeDC = False
1475 1486 ipp = None
1476 1487 lambda_ = 0
1477 1488
1478 1489 def __init__(self,**kwargs):
1479 1490 Operation.__init__(self,**kwargs)
1480 1491
1481 1492 def setup(self, dataOut, n = None, removeDC=False):
1482 1493 '''
1483 1494 n= Numero de PRF's de entrada
1484 1495 '''
1485 1496 self.__initime = None
1486 1497 self.__lastdatatime = 0
1487 1498 self.__dataReady = False
1488 1499 self.__buffer = 0
1489 1500 self.__profIndex = 0
1490 1501 self.noise = None
1491 1502 self.__nch = dataOut.nChannels
1492 1503 self.__nHeis = dataOut.nHeights
1493 1504 self.removeDC = removeDC
1494 1505 self.lambda_ = 3.0e8/(9345.0e6)
1495 1506 self.ippSec = dataOut.ippSeconds
1496 1507 self.nCohInt = dataOut.nCohInt
1497 1508
1498 1509 if n == None:
1499 1510 raise ValueError("n should be specified.")
1500 1511
1501 1512 if n != None:
1502 1513 if n<2:
1503 1514 raise ValueError("n should be greater than 2")
1504 1515
1505 1516 self.n = n
1506 1517 self.__nProf = n
1507 1518
1508 1519 self.__buffer = numpy.zeros((dataOut.nChannels,
1509 1520 n,
1510 1521 dataOut.nHeights),
1511 1522 dtype='complex')
1512 1523
1513 1524 def putData(self,data):
1514 1525 '''
1515 1526 Add a profile to he __buffer and increase in one the __profiel Index
1516 1527 '''
1517 1528 self.__buffer[:,self.__profIndex,:]= data
1518 1529 self.__profIndex += 1
1519 1530 return
1520 1531
1521 1532 def pushData(self,dataOut):
1522 1533 '''
1523 1534 Return the PULSEPAIR and the profiles used in the operation
1524 1535 Affected : self.__profileIndex
1525 1536 '''
1526 1537 #----------------- Remove DC-----------------------------------
1527 1538 if self.removeDC==True:
1528 1539 mean = numpy.mean(self.__buffer,1)
1529 1540 tmp = mean.reshape(self.__nch,1,self.__nHeis)
1530 1541 dc= numpy.tile(tmp,[1,self.__nProf,1])
1531 1542 self.__buffer = self.__buffer - dc
1532 1543 #------------------Calculo de Potencia ------------------------
1533 1544 pair0 = self.__buffer*numpy.conj(self.__buffer)
1534 1545 pair0 = pair0.real
1535 1546 lag_0 = numpy.sum(pair0,1)
1536 1547 #------------------Calculo de Ruido x canal--------------------
1537 1548 self.noise = numpy.zeros(self.__nch)
1538 1549 for i in range(self.__nch):
1539 1550 daux = numpy.sort(pair0[i,:,:],axis= None)
1540 1551 self.noise[i]=hildebrand_sekhon( daux ,self.nCohInt)
1541 1552
1542 1553 self.noise = self.noise.reshape(self.__nch,1)
1543 1554 self.noise = numpy.tile(self.noise,[1,self.__nHeis])
1544 1555 noise_buffer = self.noise.reshape(self.__nch,1,self.__nHeis)
1545 1556 noise_buffer = numpy.tile(noise_buffer,[1,self.__nProf,1])
1546 1557 #------------------ Potencia recibida= P , Potencia senal = S , Ruido= N--
1547 1558 #------------------ P= S+N ,P=lag_0/N ---------------------------------
1548 1559 #-------------------- Power --------------------------------------------------
1549 1560 data_power = lag_0/(self.n*self.nCohInt)
1550 1561 #------------------ Senal ---------------------------------------------------
1551 1562 data_intensity = pair0 - noise_buffer
1552 1563 data_intensity = numpy.sum(data_intensity,axis=1)*(self.n*self.nCohInt)#*self.nCohInt)
1553 1564 #data_intensity = (lag_0-self.noise*self.n)*(self.n*self.nCohInt)
1554 1565 for i in range(self.__nch):
1555 1566 for j in range(self.__nHeis):
1556 1567 if data_intensity[i][j] < 0:
1557 1568 data_intensity[i][j] = numpy.min(numpy.absolute(data_intensity[i][j]))
1558 1569
1559 1570 #----------------- Calculo de Frecuencia y Velocidad doppler--------
1560 1571 pair1 = self.__buffer[:,:-1,:]*numpy.conjugate(self.__buffer[:,1:,:])
1561 1572 lag_1 = numpy.sum(pair1,1)
1562 1573 data_freq = (-1/(2.0*math.pi*self.ippSec*self.nCohInt))*numpy.angle(lag_1)
1563 1574 data_velocity = (self.lambda_/2.0)*data_freq
1564 1575
1565 1576 #---------------- Potencia promedio estimada de la Senal-----------
1566 1577 lag_0 = lag_0/self.n
1567 1578 S = lag_0-self.noise
1568 1579
1569 1580 #---------------- Frecuencia Doppler promedio ---------------------
1570 1581 lag_1 = lag_1/(self.n-1)
1571 1582 R1 = numpy.abs(lag_1)
1572 1583
1573 1584 #---------------- Calculo del SNR----------------------------------
1574 1585 data_snrPP = S/self.noise
1575 1586 for i in range(self.__nch):
1576 1587 for j in range(self.__nHeis):
1577 1588 if data_snrPP[i][j] < 1.e-20:
1578 1589 data_snrPP[i][j] = 1.e-20
1579 1590
1580 1591 #----------------- Calculo del ancho espectral ----------------------
1581 1592 L = S/R1
1582 1593 L = numpy.where(L<0,1,L)
1583 1594 L = numpy.log(L)
1584 1595 tmp = numpy.sqrt(numpy.absolute(L))
1585 1596 data_specwidth = (self.lambda_/(2*math.sqrt(2)*math.pi*self.ippSec*self.nCohInt))*tmp*numpy.sign(L)
1586 1597 n = self.__profIndex
1587 1598
1588 1599 self.__buffer = numpy.zeros((self.__nch, self.__nProf,self.__nHeis), dtype='complex')
1589 1600 self.__profIndex = 0
1590 1601 return data_power,data_intensity,data_velocity,data_snrPP,data_specwidth,n
1591 1602
1592 1603
1593 1604 def pulsePairbyProfiles(self,dataOut):
1594 1605
1595 1606 self.__dataReady = False
1596 1607 data_power = None
1597 1608 data_intensity = None
1598 1609 data_velocity = None
1599 1610 data_specwidth = None
1600 1611 data_snrPP = None
1601 1612 self.putData(data=dataOut.data)
1602 1613 if self.__profIndex == self.n:
1603 1614 data_power,data_intensity, data_velocity,data_snrPP,data_specwidth, n = self.pushData(dataOut=dataOut)
1604 1615 self.__dataReady = True
1605 1616
1606 1617 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth
1607 1618
1608 1619
1609 1620 def pulsePairOp(self, dataOut, datatime= None):
1610 1621
1611 1622 if self.__initime == None:
1612 1623 self.__initime = datatime
1613 1624 data_power, data_intensity, data_velocity, data_snrPP, data_specwidth = self.pulsePairbyProfiles(dataOut)
1614 1625 self.__lastdatatime = datatime
1615 1626
1616 1627 if data_power is None:
1617 1628 return None, None, None,None,None,None
1618 1629
1619 1630 avgdatatime = self.__initime
1620 1631 deltatime = datatime - self.__lastdatatime
1621 1632 self.__initime = datatime
1622 1633
1623 1634 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth, avgdatatime
1624 1635
1625 1636 def run(self, dataOut,n = None,removeDC= False, overlapping= False,**kwargs):
1626 1637
1627 1638 if not self.isConfig:
1628 1639 self.setup(dataOut = dataOut, n = n , removeDC=removeDC , **kwargs)
1629 1640 self.isConfig = True
1630 1641 data_power, data_intensity, data_velocity,data_snrPP,data_specwidth, avgdatatime = self.pulsePairOp(dataOut, dataOut.utctime)
1631 1642 dataOut.flagNoData = True
1632 1643
1633 1644 if self.__dataReady:
1634 1645 dataOut.nCohInt *= self.n
1635 1646 dataOut.dataPP_POW = data_intensity # S
1636 1647 dataOut.dataPP_POWER = data_power # P
1637 1648 dataOut.dataPP_DOP = data_velocity
1638 1649 dataOut.dataPP_SNR = data_snrPP
1639 1650 dataOut.dataPP_WIDTH = data_specwidth
1640 1651 dataOut.PRFbyAngle = self.n #numero de PRF*cada angulo rotado que equivale a un tiempo.
1641 1652 dataOut.utctime = avgdatatime
1642 1653 dataOut.flagNoData = False
1643 1654 return dataOut
1644 1655
1645 1656
1646 1657
1647 1658 # import collections
1648 1659 # from scipy.stats import mode
1649 1660 #
1650 1661 # class Synchronize(Operation):
1651 1662 #
1652 1663 # isConfig = False
1653 1664 # __profIndex = 0
1654 1665 #
1655 1666 # def __init__(self, **kwargs):
1656 1667 #
1657 1668 # Operation.__init__(self, **kwargs)
1658 1669 # # self.isConfig = False
1659 1670 # self.__powBuffer = None
1660 1671 # self.__startIndex = 0
1661 1672 # self.__pulseFound = False
1662 1673 #
1663 1674 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1664 1675 #
1665 1676 # #Read data
1666 1677 #
1667 1678 # powerdB = dataOut.getPower(channel = channel)
1668 1679 # noisedB = dataOut.getNoise(channel = channel)[0]
1669 1680 #
1670 1681 # self.__powBuffer.extend(powerdB.flatten())
1671 1682 #
1672 1683 # dataArray = numpy.array(self.__powBuffer)
1673 1684 #
1674 1685 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1675 1686 #
1676 1687 # maxValue = numpy.nanmax(filteredPower)
1677 1688 #
1678 1689 # if maxValue < noisedB + 10:
1679 1690 # #No se encuentra ningun pulso de transmision
1680 1691 # return None
1681 1692 #
1682 1693 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1683 1694 #
1684 1695 # if len(maxValuesIndex) < 2:
1685 1696 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1686 1697 # return None
1687 1698 #
1688 1699 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1689 1700 #
1690 1701 # #Seleccionar solo valores con un espaciamiento de nSamples
1691 1702 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1692 1703 #
1693 1704 # if len(pulseIndex) < 2:
1694 1705 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1695 1706 # return None
1696 1707 #
1697 1708 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1698 1709 #
1699 1710 # #remover senales que se distancien menos de 10 unidades o muestras
1700 1711 # #(No deberian existir IPP menor a 10 unidades)
1701 1712 #
1702 1713 # realIndex = numpy.where(spacing > 10 )[0]
1703 1714 #
1704 1715 # if len(realIndex) < 2:
1705 1716 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1706 1717 # return None
1707 1718 #
1708 1719 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1709 1720 # realPulseIndex = pulseIndex[realIndex]
1710 1721 #
1711 1722 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1712 1723 #
1713 1724 # print "IPP = %d samples" %period
1714 1725 #
1715 1726 # self.__newNSamples = dataOut.nHeights #int(period)
1716 1727 # self.__startIndex = int(realPulseIndex[0])
1717 1728 #
1718 1729 # return 1
1719 1730 #
1720 1731 #
1721 1732 # def setup(self, nSamples, nChannels, buffer_size = 4):
1722 1733 #
1723 1734 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1724 1735 # maxlen = buffer_size*nSamples)
1725 1736 #
1726 1737 # bufferList = []
1727 1738 #
1728 1739 # for i in range(nChannels):
1729 1740 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1730 1741 # maxlen = buffer_size*nSamples)
1731 1742 #
1732 1743 # bufferList.append(bufferByChannel)
1733 1744 #
1734 1745 # self.__nSamples = nSamples
1735 1746 # self.__nChannels = nChannels
1736 1747 # self.__bufferList = bufferList
1737 1748 #
1738 1749 # def run(self, dataOut, channel = 0):
1739 1750 #
1740 1751 # if not self.isConfig:
1741 1752 # nSamples = dataOut.nHeights
1742 1753 # nChannels = dataOut.nChannels
1743 1754 # self.setup(nSamples, nChannels)
1744 1755 # self.isConfig = True
1745 1756 #
1746 1757 # #Append new data to internal buffer
1747 1758 # for thisChannel in range(self.__nChannels):
1748 1759 # bufferByChannel = self.__bufferList[thisChannel]
1749 1760 # bufferByChannel.extend(dataOut.data[thisChannel])
1750 1761 #
1751 1762 # if self.__pulseFound:
1752 1763 # self.__startIndex -= self.__nSamples
1753 1764 #
1754 1765 # #Finding Tx Pulse
1755 1766 # if not self.__pulseFound:
1756 1767 # indexFound = self.__findTxPulse(dataOut, channel)
1757 1768 #
1758 1769 # if indexFound == None:
1759 1770 # dataOut.flagNoData = True
1760 1771 # return
1761 1772 #
1762 1773 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1763 1774 # self.__pulseFound = True
1764 1775 # self.__startIndex = indexFound
1765 1776 #
1766 1777 # #If pulse was found ...
1767 1778 # for thisChannel in range(self.__nChannels):
1768 1779 # bufferByChannel = self.__bufferList[thisChannel]
1769 1780 # #print self.__startIndex
1770 1781 # x = numpy.array(bufferByChannel)
1771 1782 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1772 1783 #
1773 1784 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1774 1785 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1775 1786 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1776 1787 #
1777 1788 # dataOut.data = self.__arrayBuffer
1778 1789 #
1779 1790 # self.__startIndex += self.__newNSamples
1780 1791 #
1781 1792 # return
1782 1793 class SSheightProfiles(Operation):
1783 1794
1784 1795 step = None
1785 1796 nsamples = None
1786 1797 bufferShape = None
1787 1798 profileShape = None
1788 1799 sshProfiles = None
1789 1800 profileIndex = None
1790 1801
1791 1802 def __init__(self, **kwargs):
1792 1803
1793 1804 Operation.__init__(self, **kwargs)
1794 1805 self.isConfig = False
1795 1806
1796 1807 def setup(self,dataOut ,step = None , nsamples = None):
1797 1808
1798 1809 if step == None and nsamples == None:
1799 1810 raise ValueError("step or nheights should be specified ...")
1800 1811
1801 1812 self.step = step
1802 1813 self.nsamples = nsamples
1803 1814 self.__nChannels = dataOut.nChannels
1804 1815 self.__nProfiles = dataOut.nProfiles
1805 1816 self.__nHeis = dataOut.nHeights
1806 1817 shape = dataOut.data.shape #nchannels, nprofiles, nsamples
1807 1818
1808 1819 residue = (shape[1] - self.nsamples) % self.step
1809 1820 if residue != 0:
1810 1821 print("The residue is %d, step=%d should be multiple of %d to avoid loss of %d samples"%(residue,step,shape[1] - self.nsamples,residue))
1811 1822
1812 1823 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1813 1824 numberProfile = self.nsamples
1814 1825 numberSamples = (shape[1] - self.nsamples)/self.step
1815 1826
1816 1827 self.bufferShape = int(shape[0]), int(numberSamples), int(numberProfile) # nchannels, nsamples , nprofiles
1817 1828 self.profileShape = int(shape[0]), int(numberProfile), int(numberSamples) # nchannels, nprofiles, nsamples
1818 1829
1819 1830 self.buffer = numpy.zeros(self.bufferShape , dtype=numpy.complex)
1820 1831 self.sshProfiles = numpy.zeros(self.profileShape, dtype=numpy.complex)
1821 1832
1822 1833 def run(self, dataOut, step, nsamples, code = None, repeat = None):
1823 1834 dataOut.flagNoData = True
1824 1835
1825 1836 profileIndex = None
1826 1837 #print("nProfiles, nHeights ",dataOut.nProfiles, dataOut.nHeights)
1827 1838 #print(dataOut.getFreqRange(1)/1000.)
1828 1839 #exit(1)
1829 1840 if dataOut.flagDataAsBlock:
1830 1841 dataOut.data = numpy.average(dataOut.data,axis=1)
1831 1842 #print("jee")
1832 1843 dataOut.flagDataAsBlock = False
1833 1844 if not self.isConfig:
1834 1845 self.setup(dataOut, step=step , nsamples=nsamples)
1835 1846 #print("Setup done")
1836 1847 self.isConfig = True
1837 1848
1838 1849
1839 1850 if code is not None:
1840 1851 code = numpy.array(code)
1841 1852 code_block = code
1842 1853
1843 1854 if repeat is not None:
1844 1855 code_block = numpy.repeat(code_block, repeats=repeat, axis=1)
1845 1856 #print(code_block.shape)
1846 1857 for i in range(self.buffer.shape[1]):
1847 1858
1848 1859 if code is not None:
1849 1860 self.buffer[:,i] = dataOut.data[:,i*self.step:i*self.step + self.nsamples]*code_block
1850 1861
1851 1862 else:
1852 1863
1853 1864 self.buffer[:,i] = dataOut.data[:,i*self.step:i*self.step + self.nsamples]#*code[dataOut.profileIndex,:]
1854 1865
1855 1866 #self.buffer[:,j,self.__nHeis-j*self.step - self.nheights:self.__nHeis-j*self.step] = numpy.flip(dataOut.data[:,j*self.step:j*self.step + self.nheights])
1856 1867
1857 1868 for j in range(self.buffer.shape[0]):
1858 1869 self.sshProfiles[j] = numpy.transpose(self.buffer[j])
1859 1870
1860 1871 profileIndex = self.nsamples
1861 1872 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1862 1873 ippSeconds = (deltaHeight*1.0e-6)/(0.15)
1863 1874 #print("ippSeconds, dH: ",ippSeconds,deltaHeight)
1864 1875 try:
1865 1876 if dataOut.concat_m is not None:
1866 1877 ippSeconds= ippSeconds/float(dataOut.concat_m)
1867 1878 #print "Profile concat %d"%dataOut.concat_m
1868 1879 except:
1869 1880 pass
1870 1881
1871 1882 dataOut.data = self.sshProfiles
1872 1883 dataOut.flagNoData = False
1873 1884 dataOut.heightList = numpy.arange(self.buffer.shape[1]) *self.step*deltaHeight + dataOut.heightList[0]
1874 1885 dataOut.nProfiles = int(dataOut.nProfiles*self.nsamples)
1875 1886
1876 1887 dataOut.profileIndex = profileIndex
1877 1888 dataOut.flagDataAsBlock = True
1878 1889 dataOut.ippSeconds = ippSeconds
1879 1890 dataOut.step = self.step
1880 1891 #print(numpy.shape(dataOut.data))
1881 1892 #exit(1)
1882 1893 #print("new data shape and time:", dataOut.data.shape, dataOut.utctime)
1883 1894
1884 1895 return dataOut
1885 1896 ################################################################################3############################3
1886 1897 ################################################################################3############################3
1887 1898 ################################################################################3############################3
1888 1899 ################################################################################3############################3
1889 1900
1890 1901 class SSheightProfiles2(Operation):
1891 1902 '''
1892 1903 Procesa por perfiles y por bloques
1893 1904 Versión corregida y actualizada para trabajar con RemoveProfileSats2
1894 1905 Usar esto
1895 1906 '''
1896 1907
1897 1908
1898 1909 bufferShape = None
1899 1910 profileShape = None
1900 1911 sshProfiles = None
1901 1912 profileIndex = None
1902 1913 #nsamples = None
1903 1914 #step = None
1904 1915 #deltaHeight = None
1905 1916 #init_range = None
1906 1917 __slots__ = ('step', 'nsamples', 'deltaHeight', 'init_range', 'isConfig', '__nChannels',
1907 1918 '__nProfiles', '__nHeis', 'deltaHeight', 'new_nHeights')
1908 1919
1909 1920 def __init__(self, **kwargs):
1910 1921
1911 1922 Operation.__init__(self, **kwargs)
1912 1923 self.isConfig = False
1913 1924
1914 1925 def setup(self,dataOut ,step = None , nsamples = None):
1915 1926
1916 1927 if step == None and nsamples == None:
1917 1928 raise ValueError("step or nheights should be specified ...")
1918 1929
1919 1930 self.step = step
1920 1931 self.nsamples = nsamples
1921 1932 self.__nChannels = int(dataOut.nChannels)
1922 1933 self.__nProfiles = int(dataOut.nProfiles)
1923 1934 self.__nHeis = int(dataOut.nHeights)
1924 1935
1925 1936 residue = (self.__nHeis - self.nsamples) % self.step
1926 1937 if residue != 0:
1927 1938 print("The residue is %d, step=%d should be multiple of %d to avoid loss of %d samples"%(residue,step,self.__nProfiles - self.nsamples,residue))
1928 1939
1929 1940 self.deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1930 1941 self.init_range = dataOut.heightList[0]
1931 1942 #numberProfile = self.nsamples
1932 1943 numberSamples = (self.__nHeis - self.nsamples)/self.step
1933 1944
1934 1945 self.new_nHeights = numberSamples
1935 1946
1936 1947 self.bufferShape = int(self.__nChannels), int(numberSamples), int(self.nsamples) # nchannels, nsamples , nprofiles
1937 1948 self.profileShape = int(self.__nChannels), int(self.nsamples), int(numberSamples) # nchannels, nprofiles, nsamples
1938 1949
1939 1950 self.buffer = numpy.zeros(self.bufferShape , dtype=numpy.complex)
1940 1951 self.sshProfiles = numpy.zeros(self.profileShape, dtype=numpy.complex)
1941 1952
1942 1953 def getNewProfiles(self, data, code=None, repeat=None):
1943 1954
1944 1955 if code is not None:
1945 1956 code = numpy.array(code)
1946 1957 code_block = code
1947 1958
1948 1959 if repeat is not None:
1949 1960 code_block = numpy.repeat(code_block, repeats=repeat, axis=1)
1950 1961 if data.ndim < 3:
1951 1962 data = data.reshape(self.__nChannels,1,self.__nHeis )
1952 1963 #print("buff, data, :",self.buffer.shape, data.shape,self.sshProfiles.shape, code_block.shape)
1953 1964 for ch in range(self.__nChannels):
1954 1965 for i in range(int(self.new_nHeights)): #nuevas alturas
1955 1966 if code is not None:
1956 1967 self.buffer[ch,i,:] = data[ch,:,i*self.step:i*self.step + self.nsamples]*code_block
1957 1968 else:
1958 1969 self.buffer[ch,i,:] = data[ch,:,i*self.step:i*self.step + self.nsamples]#*code[dataOut.profileIndex,:]
1959 1970
1960 1971 for j in range(self.__nChannels): #en los cananles
1961 1972 self.sshProfiles[j,:,:] = numpy.transpose(self.buffer[j,:,:])
1962 1973 #print("new profs Done")
1963 1974
1964 1975
1965 1976
1966 1977 def run(self, dataOut, step, nsamples, code = None, repeat = None):
1967 1978 # print("running")
1968 1979 if dataOut.flagNoData == True:
1969 1980 return dataOut
1970 1981 dataOut.flagNoData = True
1971 1982 #print("init data shape:", dataOut.data.shape)
1972 1983 #print("ch: {} prof: {} hs: {}".format(int(dataOut.nChannels),
1973 1984 # int(dataOut.nProfiles),int(dataOut.nHeights)))
1974 1985
1975 1986 profileIndex = None
1976 1987 # if not dataOut.flagDataAsBlock:
1977 1988 # dataOut.nProfiles = 1
1978 1989
1979 1990 if not self.isConfig:
1980 1991 self.setup(dataOut, step=step , nsamples=nsamples)
1981 1992 #print("Setup done")
1982 1993 self.isConfig = True
1983 1994
1984 1995 dataBlock = None
1985 1996
1986 1997 nprof = 1
1987 1998 if dataOut.flagDataAsBlock:
1988 1999 nprof = int(dataOut.nProfiles)
1989 2000
1990 2001 #print("dataOut nProfiles:", dataOut.nProfiles)
1991 2002 for profile in range(nprof):
1992 2003 if dataOut.flagDataAsBlock:
1993 2004 #print("read blocks")
1994 2005 self.getNewProfiles(dataOut.data[:,profile,:], code=code, repeat=repeat)
1995 2006 else:
1996 2007 #print("read profiles")
1997 2008 self.getNewProfiles(dataOut.data, code=code, repeat=repeat) #only one channe
1998 2009 if profile == 0:
1999 2010 dataBlock = self.sshProfiles.copy()
2000 2011 else: #by blocks
2001 2012 dataBlock = numpy.concatenate((dataBlock,self.sshProfiles), axis=1) #profile axis
2002 2013 #print("by blocks: ",dataBlock.shape, self.sshProfiles.shape)
2003 2014
2004 2015 profileIndex = self.nsamples
2005 2016 #deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
2006 2017 ippSeconds = (self.deltaHeight*1.0e-6)/(0.15)
2007 2018
2008 2019
2009 2020 dataOut.data = dataBlock
2010 2021 #print("show me: ",self.step,self.deltaHeight, dataOut.heightList, self.new_nHeights)
2011 2022 dataOut.heightList = numpy.arange(int(self.new_nHeights)) *self.step*self.deltaHeight + self.init_range
2012 2023 dataOut.sampled_heightsFFT = self.nsamples
2013 2024 dataOut.ippSeconds = ippSeconds
2014 2025 dataOut.step = self.step
2015 2026 dataOut.deltaHeight = self.step*self.deltaHeight
2016 2027 dataOut.flagNoData = False
2017 2028 if dataOut.flagDataAsBlock:
2018 2029 dataOut.nProfiles = int(dataOut.nProfiles*self.nsamples)
2019 2030
2020 2031 else:
2021 2032 dataOut.nProfiles = int(self.nsamples)
2022 2033 dataOut.profileIndex = dataOut.nProfiles
2023 2034 dataOut.flagDataAsBlock = True
2024 2035
2025 2036 dataBlock = None
2026 2037
2027 2038 #print("new data shape:", dataOut.data.shape, dataOut.utctime)
2028 2039
2029 2040 #update Processing Header:
2030 2041 dataOut.processingHeaderObj.heightList = dataOut.heightList
2031 2042 dataOut.processingHeaderObj.ipp = ippSeconds
2032 2043 dataOut.processingHeaderObj.heightResolution = dataOut.deltaHeight
2033 2044 #dataOut.processingHeaderObj.profilesPerBlock = nProfiles
2034 2045
2035 2046 # # dataOut.data = CH, PROFILES, HEIGHTS
2036 2047 #print(dataOut.data .shape)
2037 2048 if dataOut.flagProfilesByRange:
2038 2049 # #assuming the same remotion for all channels
2039 2050 aux = [ self.nsamples - numpy.count_nonzero(dataOut.data[0, :, h]==0) for h in range(len(dataOut.heightList))]
2040 2051 dataOut.nProfilesByRange = (numpy.asarray(aux)).reshape((1,len(dataOut.heightList) ))
2041 2052 #print(dataOut.nProfilesByRange.shape)
2042 2053 else:
2043 2054 dataOut.nProfilesByRange = numpy.ones((1, len(dataOut.heightList)))*dataOut.nProfiles
2044 2055 return dataOut
2045 2056
2046 2057
2047 2058
2048 2059 class RemoveProfileSats(Operation):
2049 2060 '''
2050 2061 Escrito: Joab Apaza
2051 2062
2052 2063 Omite los perfiles contaminados con señal de satélites, usando una altura de referencia
2053 2064 In: minHei = min_sat_range
2054 2065 max_sat_range
2055 2066 min_hei_ref
2056 2067 max_hei_ref
2057 2068 th = diference between profiles mean, ref and sats
2058 2069 Out:
2059 2070 profile clean
2060 2071 '''
2061 2072
2062 2073
2063 2074 __buffer_data = []
2064 2075 __buffer_times = []
2065 2076
2066 2077 buffer = None
2067 2078
2068 2079 outliers_IDs_list = []
2069 2080
2070 2081
2071 2082 __slots__ = ('n','navg','profileMargin','thHistOutlier','minHei_idx','maxHei_idx','nHeights',
2072 2083 'first_utcBlock','__profIndex','init_prof','end_prof','lenProfileOut','nChannels',
2073 2084 '__count_exec','__initime','__dataReady','__ipp', 'minRef', 'maxRef', 'thdB')
2074 2085 def __init__(self, **kwargs):
2075 2086
2076 2087 Operation.__init__(self, **kwargs)
2077 2088 self.isConfig = False
2078 2089
2079 2090 def setup(self,dataOut, n=None , navg=0.8, profileMargin=50,thHistOutlier=15,
2080 2091 minHei=None, maxHei=None, minRef=None, maxRef=None, thdB=10):
2081 2092
2082 2093 if n == None and timeInterval == None:
2083 2094 raise ValueError("nprofiles or timeInterval should be specified ...")
2084 2095
2085 2096 if n != None:
2086 2097 self.n = n
2087 2098
2088 2099 self.navg = navg
2089 2100 self.profileMargin = profileMargin
2090 2101 self.thHistOutlier = thHistOutlier
2091 2102 self.__profIndex = 0
2092 2103 self.buffer = None
2093 2104 self._ipp = dataOut.ippSeconds
2094 2105 self.n_prof_released = 0
2095 2106 self.heightList = dataOut.heightList
2096 2107 self.init_prof = 0
2097 2108 self.end_prof = 0
2098 2109 self.__count_exec = 0
2099 2110 self.__profIndex = 0
2100 2111 self.first_utcBlock = None
2101 2112 #self.__dh = dataOut.heightList[1] - dataOut.heightList[0]
2102 2113 minHei = minHei
2103 2114 maxHei = maxHei
2104 2115 if minHei==None :
2105 2116 minHei = dataOut.heightList[0]
2106 2117 if maxHei==None :
2107 2118 maxHei = dataOut.heightList[-1]
2108 2119 self.minHei_idx,self.maxHei_idx = getHei_index(minHei, maxHei, dataOut.heightList)
2109 2120 self.min_ref, self.max_ref = getHei_index(minRef, maxRef, dataOut.heightList)
2110 2121 self.nChannels = dataOut.nChannels
2111 2122 self.nHeights = dataOut.nHeights
2112 2123 self.test_counter = 0
2113 2124 self.thdB = thdB
2114 2125
2115 2126 def filterSatsProfiles(self):
2116 2127 data = self.__buffer_data
2117 2128 #print(data.shape)
2118 2129 nChannels, profiles, heights = data.shape
2119 2130 indexes=numpy.zeros([], dtype=int)
2120 2131 outliers_IDs=[]
2121 2132 for c in range(nChannels):
2122 2133 #print(self.min_ref,self.max_ref)
2123 2134 noise_ref = 10* numpy.log10((data[c,:,self.min_ref:self.max_ref] * numpy.conjugate(data[c,:,self.min_ref:self.max_ref])).real)
2124 2135 #print("Noise ",numpy.percentile(noise_ref,95))
2125 2136 p95 = numpy.percentile(noise_ref,95)
2126 2137 noise_ref = noise_ref.mean()
2127 2138 #print("Noise ",noise_ref
2128 2139
2129 2140
2130 2141 for h in range(self.minHei_idx, self.maxHei_idx):
2131 2142 power = 10* numpy.log10((data[c,:,h] * numpy.conjugate(data[c,:,h])).real)
2132 2143 #th = noise_ref + self.thdB
2133 2144 th = noise_ref + 1.5*(p95-noise_ref)
2134 2145 index = numpy.where(power > th )
2135 2146 if index[0].size > 10 and index[0].size < int(self.navg*profiles):
2136 2147 indexes = numpy.append(indexes, index[0])
2137 2148 #print(index[0])
2138 2149 #print(index[0])
2139 2150
2140 2151 # fig,ax = plt.subplots()
2141 2152 # #ax.set_title(str(k)+" "+str(j))
2142 2153 # x=range(len(power))
2143 2154 # ax.scatter(x,power)
2144 2155 # #ax.axvline(index)
2145 2156 # plt.grid()
2146 2157 # plt.show()
2147 2158 #print(indexes)
2148 2159
2149 2160 #outliers_IDs = outliers_IDs.astype(numpy.dtype('int64'))
2150 2161 #outliers_IDs = numpy.unique(outliers_IDs)
2151 2162
2152 2163 outs_lines = numpy.unique(indexes)
2153 2164
2154 2165
2155 2166 #Agrupando el histograma de outliers,
2156 2167 my_bins = numpy.linspace(0,int(profiles), int(profiles/100), endpoint=True)
2157 2168
2158 2169
2159 2170 hist, bins = numpy.histogram(outs_lines,bins=my_bins)
2160 2171 hist_outliers_indexes = numpy.where(hist > self.thHistOutlier) #es outlier
2161 2172 hist_outliers_indexes = hist_outliers_indexes[0]
2162 2173 # if len(hist_outliers_indexes>0):
2163 2174 # hist_outliers_indexes = numpy.append(hist_outliers_indexes,hist_outliers_indexes[-1]+1)
2164 2175 #print(hist_outliers_indexes)
2165 2176 #print(bins, hist_outliers_indexes)
2166 2177 bins_outliers_indexes = [int(i) for i in (bins[hist_outliers_indexes])] #
2167 2178 outlier_loc_index = []
2168 2179 # for n in range(len(bins_outliers_indexes)):
2169 2180 # for e in range(bins_outliers_indexes[n]-self.profileMargin,bins_outliers_indexes[n]+ self.profileMargin):
2170 2181 # outlier_loc_index.append(e)
2171 2182 outlier_loc_index = [e for n in range(len(bins_outliers_indexes)) for e in range(bins_outliers_indexes[n]-self.profileMargin,bins_outliers_indexes[n]+ profiles//100 + self.profileMargin) ]
2172 2183 outlier_loc_index = numpy.asarray(outlier_loc_index)
2173 2184
2174 2185
2175 2186
2176 2187
2177 2188 #print("outliers Ids: ", outlier_loc_index, outlier_loc_index.shape)
2178 2189 outlier_loc_index = outlier_loc_index[ (outlier_loc_index >= 0) & (outlier_loc_index<profiles)]
2179 2190 #print("outliers final: ", outlier_loc_index)
2180 2191
2181 2192 from matplotlib import pyplot as plt
2182 2193 x, y = numpy.meshgrid(numpy.arange(profiles), self.heightList)
2183 2194 fig, ax = plt.subplots(1,2,figsize=(8, 6))
2184 2195 dat = data[0,:,:].real
2185 2196 dat = 10* numpy.log10((data[0,:,:] * numpy.conjugate(data[0,:,:])).real)
2186 2197 m = numpy.nanmean(dat)
2187 2198 o = numpy.nanstd(dat)
2188 2199 #print(m, o, x.shape, y.shape)
2189 2200 #c = ax[0].pcolormesh(x, y, dat.T, cmap ='YlGnBu', vmin = (m-2*o), vmax = (m+2*o))
2190 2201 c = ax[0].pcolormesh(x, y, dat.T, cmap ='YlGnBu', vmin = 50, vmax = 75)
2191 2202 ax[0].vlines(outs_lines,200,600, linestyles='dashed', label = 'outs', color='w')
2192 2203 fig.colorbar(c)
2193 2204 ax[0].vlines(outlier_loc_index,650,750, linestyles='dashed', label = 'outs', color='r')
2194 2205 ax[1].hist(outs_lines,bins=my_bins)
2195 2206 plt.show()
2196 2207
2197 2208
2198 2209 self.outliers_IDs_list = outlier_loc_index
2199 2210 #print("outs list: ", self.outliers_IDs_list)
2200 2211 return data
2201 2212
2202 2213
2203 2214
2204 2215 def fillBuffer(self, data, datatime):
2205 2216
2206 2217 if self.__profIndex == 0:
2207 2218 self.__buffer_data = data.copy()
2208 2219
2209 2220 else:
2210 2221 self.__buffer_data = numpy.concatenate((self.__buffer_data,data), axis=1)#en perfiles
2211 2222 self.__profIndex += 1
2212 2223 self.__buffer_times.append(datatime)
2213 2224
2214 2225 def getData(self, data, datatime=None):
2215 2226
2216 2227 if self.__profIndex == 0:
2217 2228 self.__initime = datatime
2218 2229
2219 2230
2220 2231 self.__dataReady = False
2221 2232
2222 2233 self.fillBuffer(data, datatime)
2223 2234 dataBlock = None
2224 2235
2225 2236 if self.__profIndex == self.n:
2226 2237 #print("apnd : ",data)
2227 2238 dataBlock = self.filterSatsProfiles()
2228 2239 self.__dataReady = True
2229 2240
2230 2241 return dataBlock
2231 2242
2232 2243 if dataBlock is None:
2233 2244 return None, None
2234 2245
2235 2246
2236 2247
2237 2248 return dataBlock
2238 2249
2239 2250 def releaseBlock(self):
2240 2251
2241 2252 if self.n % self.lenProfileOut != 0:
2242 2253 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n))
2243 2254 return None
2244 2255
2245 2256 data = self.buffer[:,self.init_prof:self.end_prof:,:] #ch, prof, alt
2246 2257
2247 2258 self.init_prof = self.end_prof
2248 2259 self.end_prof += self.lenProfileOut
2249 2260 #print("data release shape: ",dataOut.data.shape, self.end_prof)
2250 2261 self.n_prof_released += 1
2251 2262
2252 2263 return data
2253 2264
2254 2265 def run(self, dataOut, n=None, navg=0.8, nProfilesOut=1, profile_margin=50,
2255 2266 th_hist_outlier=15,minHei=None, maxHei=None, minRef=None, maxRef=None, thdB=10):
2256 2267
2257 2268 if not self.isConfig:
2258 2269 #print("init p idx: ", dataOut.profileIndex )
2259 2270 self.setup(dataOut,n=n, navg=navg,profileMargin=profile_margin,thHistOutlier=th_hist_outlier,
2260 2271 minHei=minHei, maxHei=maxHei, minRef=minRef, maxRef=maxRef, thdB=thdB)
2261 2272 self.isConfig = True
2262 2273
2263 2274 dataBlock = None
2264 2275
2265 2276 if not dataOut.buffer_empty: #hay datos acumulados
2266 2277
2267 2278 if self.init_prof == 0:
2268 2279 self.n_prof_released = 0
2269 2280 self.lenProfileOut = nProfilesOut
2270 2281 dataOut.flagNoData = False
2271 2282 #print("tp 2 ",dataOut.data.shape)
2272 2283
2273 2284 self.init_prof = 0
2274 2285 self.end_prof = self.lenProfileOut
2275 2286
2276 2287 dataOut.nProfiles = self.lenProfileOut
2277 2288 if nProfilesOut == 1:
2278 2289 dataOut.flagDataAsBlock = False
2279 2290 else:
2280 2291 dataOut.flagDataAsBlock = True
2281 2292 #print("prof: ",self.init_prof)
2282 2293 dataOut.flagNoData = False
2283 2294 if numpy.isin(self.n_prof_released, self.outliers_IDs_list):
2284 2295 #print("omitting: ", self.n_prof_released)
2285 2296 dataOut.flagNoData = True
2286 2297 dataOut.ippSeconds = self._ipp
2287 2298 dataOut.utctime = self.first_utcBlock + self.init_prof*self._ipp
2288 2299 # print("time: ", dataOut.utctime, self.first_utcBlock, self.init_prof,self._ipp,dataOut.ippSeconds)
2289 2300 #dataOut.data = self.releaseBlock()
2290 2301 #########################################################3
2291 2302 if self.n % self.lenProfileOut != 0:
2292 2303 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n))
2293 2304 return None
2294 2305
2295 2306 dataOut.data = None
2296 2307
2297 2308 if nProfilesOut == 1:
2298 2309 dataOut.data = self.buffer[:,self.end_prof-1,:] #ch, prof, alt
2299 2310 else:
2300 2311 dataOut.data = self.buffer[:,self.init_prof:self.end_prof,:] #ch, prof, alt
2301 2312
2302 2313 self.init_prof = self.end_prof
2303 2314 self.end_prof += self.lenProfileOut
2304 2315 #print("data release shape: ",dataOut.data.shape, self.end_prof, dataOut.flagNoData)
2305 2316 self.n_prof_released += 1
2306 2317
2307 2318 if self.end_prof >= (self.n +self.lenProfileOut):
2308 2319
2309 2320 self.init_prof = 0
2310 2321 self.__profIndex = 0
2311 2322 self.buffer = None
2312 2323 dataOut.buffer_empty = True
2313 2324 self.outliers_IDs_list = []
2314 2325 self.n_prof_released = 0
2315 2326 dataOut.flagNoData = False #enviar ultimo aunque sea outlier :(
2316 2327 #print("cleaning...", dataOut.buffer_empty)
2317 2328 dataOut.profileIndex = 0 #self.lenProfileOut
2318 2329 ####################################################################
2319 2330 return dataOut
2320 2331
2321 2332
2322 2333 #print("tp 223 ",dataOut.data.shape)
2323 2334 dataOut.flagNoData = True
2324 2335
2325 2336
2326 2337
2327 2338 try:
2328 2339 #dataBlock = self.getData(dataOut.data.reshape(self.nChannels,1,self.nHeights), dataOut.utctime)
2329 2340 dataBlock = self.getData(numpy.reshape(dataOut.data,(self.nChannels,1,self.nHeights)), dataOut.utctime)
2330 2341 self.__count_exec +=1
2331 2342 except Exception as e:
2332 2343 print("Error getting profiles data",self.__count_exec )
2333 2344 print(e)
2334 2345 sys.exit()
2335 2346
2336 2347 if self.__dataReady:
2337 2348 #print("omitting: ", len(self.outliers_IDs_list))
2338 2349 self.__count_exec = 0
2339 2350 #dataOut.data =
2340 2351 #self.buffer = numpy.flip(dataBlock, axis=1)
2341 2352 self.buffer = dataBlock
2342 2353 self.first_utcBlock = self.__initime
2343 2354 dataOut.utctime = self.__initime
2344 2355 dataOut.nProfiles = self.__profIndex
2345 2356 #dataOut.flagNoData = False
2346 2357 self.init_prof = 0
2347 2358 self.__profIndex = 0
2348 2359 self.__initime = None
2349 2360 dataBlock = None
2350 2361 self.__buffer_times = []
2351 2362 dataOut.error = False
2352 2363 dataOut.useInputBuffer = True
2353 2364 dataOut.buffer_empty = False
2354 2365 #print("1 ch: {} prof: {} hs: {}".format(int(dataOut.nChannels),int(dataOut.nProfiles),int(dataOut.nHeights)))
2355 2366
2356 2367
2357 2368
2358 2369 #print(self.__count_exec)
2359 2370
2360 2371 return dataOut
2361 2372
2362 2373
2363 2374 class RemoveProfileSats2(Operation):
2364 2375 '''
2365 2376 Escrito: Joab Apaza
2366 2377
2367 2378 Omite los perfiles contaminados con señal de satélites, usando una altura de referencia
2368 2379 promedia todas las alturas para los cálculos
2369 2380 In:
2370 2381 n = Cantidad de perfiles que se acumularan, usualmente 10 segundos
2371 2382 navg = Porcentaje de perfiles que puede considerarse como satélite, máximo 90%
2372 2383 minHei =
2373 2384 minRef =
2374 2385 maxRef =
2375 2386 nBins =
2376 2387 profile_margin =
2377 2388 th_hist_outlier =
2378 2389 nProfilesOut =
2379 2390
2380 2391 Pensado para remover interferencias de las YAGI, se puede adaptar a otras interferencias
2381 2392
2382 2393 remYagi = Activa la funcion de remoción de interferencias de la YAGI
2383 2394 nProfYagi = Cantidad de perfiles que son afectados, acorde NTX de la YAGI
2384 2395 offYagi =
2385 2396 minHJULIA = Altura mínima donde aparece la señal referencia de JULIA (-50)
2386 2397 maxHJULIA = Altura máxima donde aparece la señal referencia de JULIA (-15)
2387 2398
2388 2399 debug = Activa los gráficos, recomendable ejecutar para ajustar los parámetros
2389 2400 para un experimento en específico.
2390 2401
2391 2402 ** se modifica para remover interferencias puntuales, es decir, desde otros radares.
2392 2403 Inicialmente se ha configurado para omitir también los perfiles de la YAGI en los datos
2393 2404 de AMISR-ISR.
2394 2405
2395 2406 Out:
2396 2407 profile clean
2397 2408 '''
2398 2409
2399 2410
2400 2411 __buffer_data = []
2401 2412 __buffer_times = []
2402 2413
2403 2414 buffer = None
2404 2415
2405 2416 outliers_IDs_list = []
2406 2417
2407 2418
2408 2419 __slots__ = ('n','navg','profileMargin','thHistOutlier','minHei_idx','maxHei_idx','nHeights',
2409 2420 'first_utcBlock','__profIndex','init_prof','end_prof','lenProfileOut','nChannels','cohFactor',
2410 2421 '__count_exec','__initime','__dataReady','__ipp', 'minRef', 'maxRef', 'debug','prev_pnoise','thfactor')
2411 2422 def __init__(self, **kwargs):
2412 2423
2413 2424 Operation.__init__(self, **kwargs)
2414 2425 self.isConfig = False
2415 2426 self.currentTime = None
2416 2427
2417 2428 def setup(self,dataOut, n=None , navg=0.9, profileMargin=50,thHistOutlier=15,minHei=None, maxHei=None, nBins=10,
2418 2429 minRef=None, maxRef=None, debug=False, remYagi=False, nProfYagi = 0, offYagi=0, minHJULIA=None, maxHJULIA=None,
2419 2430 idate=None,startH=None,endH=None, thfactor=1 ):
2420 2431
2421 2432 if n == None and timeInterval == None:
2422 2433 raise ValueError("nprofiles or timeInterval should be specified ...")
2423 2434
2424 2435 if n != None:
2425 2436 self.n = n
2426 2437
2427 2438 self.navg = navg
2428 2439 self.profileMargin = profileMargin
2429 2440 self.thHistOutlier = thHistOutlier
2430 2441 self.__profIndex = 0
2431 2442 self.buffer = None
2432 2443 self._ipp = dataOut.ippSeconds
2433 2444 self.n_prof_released = 0
2434 2445 self.heightList = dataOut.heightList
2435 2446 self.init_prof = 0
2436 2447 self.end_prof = 0
2437 2448 self.__count_exec = 0
2438 2449 self.__profIndex = 0
2439 2450 self.first_utcBlock = None
2440 2451 self.prev_pnoise = None
2441 2452 self.nBins = nBins
2442 2453 self.thfactor = thfactor
2443 2454 #self.__dh = dataOut.heightList[1] - dataOut.heightList[0]
2444 2455 minHei = minHei
2445 2456 maxHei = maxHei
2446 2457 if minHei==None :
2447 2458 minHei = dataOut.heightList[0]
2448 2459 if maxHei==None :
2449 2460 maxHei = dataOut.heightList[-1]
2450 2461 self.minHei_idx,self.maxHei_idx = getHei_index(minHei, maxHei, dataOut.heightList)
2451 2462 self.min_ref, self.max_ref = getHei_index(minRef, maxRef, dataOut.heightList)
2452 2463 self.nChannels = dataOut.nChannels
2453 2464 self.nHeights = dataOut.nHeights
2454 2465 self.test_counter = 0
2455 2466 self.debug = debug
2456 2467 self.remYagi = remYagi
2457 2468 self.cohFactor = dataOut.nCohInt
2458 2469 if self.remYagi :
2459 2470 if minHJULIA==None or maxHJULIA==None:
2460 2471 raise ValueError("Parameters minHYagi and minHYagi are necessary!")
2461 2472 return
2462 2473 if idate==None or startH==None or endH==None:
2463 2474 raise ValueError("Date and hour parameters are necessary!")
2464 2475 return
2465 2476 self.minHJULIA_idx,self.maxHJULIA_idx = getHei_index(minHJULIA, maxHJULIA, dataOut.heightList)
2466 2477 self.offYagi = offYagi
2467 2478 self.nTxYagi = nProfYagi
2468 2479
2469 2480 self.startTime = datetime.datetime.combine(idate,startH)
2470 2481 self.endTime = datetime.datetime.combine(idate,endH)
2471 2482
2472 2483 log.warning("Be careful with the selection of parameters for sats removal! It is avisable to \
2473 2484 activate the debug parameter in this operation for calibration", self.name)
2474 2485
2475 2486
2476 2487 def filterSatsProfiles(self):
2477 2488
2478 2489 data = self.__buffer_data.copy()
2479 2490 #print(data.shape)
2480 2491 nChannels, profiles, heights = data.shape
2481 2492 indexes=numpy.zeros([], dtype=int)
2482 2493 indexes = numpy.delete(indexes,0)
2483 2494
2484 2495 indexesYagi=numpy.zeros([], dtype=int)
2485 2496 indexesYagi = numpy.delete(indexesYagi,0)
2486 2497
2487 2498 indexesYagi_up=numpy.zeros([], dtype=int)
2488 2499 indexesYagi_up = numpy.delete(indexesYagi_up,0)
2489 2500 indexesYagi_down=numpy.zeros([], dtype=int)
2490 2501 indexesYagi_down = numpy.delete(indexesYagi_down,0)
2491 2502
2492 2503
2493 2504 indexesJULIA=numpy.zeros([], dtype=int)
2494 2505 indexesJULIA = numpy.delete(indexesJULIA,0)
2495 2506
2496 2507 outliers_IDs=[]
2497 2508
2498 2509 div = profiles//self.nBins
2499 2510
2500 2511 for c in range(nChannels):
2501 2512 #print(self.min_ref,self.max_ref)
2502 2513
2503 2514 import scipy.signal
2504 2515 b, a = scipy.signal.butter(3, 0.5)
2505 2516 #noise_ref = (data[c,:,self.min_ref:self.max_ref] * numpy.conjugate(data[c,:,self.min_ref:self.max_ref]))
2506 2517 noise_ref = numpy.abs(data[c,:,self.min_ref:self.max_ref])
2507 2518 lnoise = len(noise_ref[0,:])
2508 2519 #print(noise_ref.shape)
2509 2520 noise_ref = noise_ref.mean(axis=1)
2510 2521 #fnoise = noise_ref
2511 2522 fnoise = scipy.signal.filtfilt(b, a, noise_ref)
2512 2523 #noise_refdB = 10* numpy.log10(noise_ref)
2513 2524 #print("Noise ",numpy.percentile(noise_ref,95))
2514 2525 p95 = numpy.percentile(fnoise,95)
2515 2526 mean_noise = fnoise.mean()
2516 2527
2517 2528 if self.prev_pnoise != None:
2518 2529 if mean_noise < (1.1 * self.prev_pnoise) and mean_noise > (0.9 * self.prev_pnoise):
2519 2530 mean_noise = 0.9*mean_noise + 0.1*self.prev_pnoise
2520 2531 self.prev_pnoise = mean_noise
2521 2532 else:
2522 2533 mean_noise = self.prev_pnoise
2523 2534 else:
2524 2535 self.prev_pnoise = mean_noise
2525 2536
2526 2537 std = fnoise.std()+ fnoise.mean()
2527 2538
2528 2539
2529 2540
2530 2541 #power = (data[c,:,self.minHei_idx:self.maxHei_idx] * numpy.conjugate(data[c,:,self.minHei_idx:self.maxHei_idx]))
2531 2542 power = numpy.abs(data[c,:,self.minHei_idx:self.maxHei_idx])
2532 2543 npower = len(power[0,:])
2533 2544 #print(power.shape)
2534 2545 power = power.mean(axis=1)
2535 2546
2536 2547 fpower = scipy.signal.filtfilt(b, a, power)
2537 2548 #print(power.shape)
2538 2549 #powerdB = 10* numpy.log10(power)
2539 2550
2540 2551 #th = p95 * self.thfactor
2541 2552 th = mean_noise * self.thfactor
2542 2553
2543 2554 index = numpy.where(fpower > th )
2544 2555 #print("Noise ",mean_noise, p95)
2545 2556 #print(index)
2546 2557
2547 2558
2548 2559
2549 2560 if index[0].size <= int(self.navg*profiles): #outliers from sats
2550 2561 indexes = numpy.append(indexes, index[0])
2551 2562
2552 2563 index2low = numpy.where(fpower < (th*0.5 )) #outliers from no TX
2553 2564 if index2low[0].size <= int(self.navg*profiles):
2554 2565 indexes = numpy.append(indexes, index2low[0])
2555 2566
2556 2567 #print("sdas ", noise_ref.mean())
2557 2568
2558 2569 if self.remYagi :
2559 2570 #print(self.minHJULIA_idx, self.maxHJULIA_idx)
2560 2571 powerJULIA = (data[c,:,self.minHJULIA_idx:self.maxHJULIA_idx] * numpy.conjugate(data[c,:,self.minHJULIA_idx:self.maxHJULIA_idx])).real
2561 2572 powerJULIA = powerJULIA.mean(axis=1)
2562 2573 th_JULIA = powerJULIA.mean()*0.85
2563 2574 indexJULIA = numpy.where(powerJULIA >= th_JULIA )
2564 2575
2565 2576 indexesJULIA= numpy.append(indexesJULIA, indexJULIA[0])
2566 2577
2567 2578 # fig, ax = plt.subplots()
2568 2579 # ax.plot(powerJULIA)
2569 2580 # ax.axhline(th_JULIA, color='r')
2570 2581 # plt.grid()
2571 2582 # plt.show()
2572 2583
2573 2584 if self.debug:
2574 2585 fig, ax = plt.subplots()
2575 2586 ax.plot(fpower, label="power")
2576 2587 #ax.plot(fnoise, label="noise ref")
2577 2588 ax.axhline(th, color='g', label="th")
2578 2589 #ax.axhline(std, color='b', label="mean")
2579 2590 ax.legend()
2580 2591 plt.grid()
2581 2592 plt.show()
2582 2593
2583 2594 #print(indexes)
2584 2595
2585 2596 #outliers_IDs = outliers_IDs.astype(numpy.dtype('int64'))
2586 2597 #outliers_IDs = numpy.unique(outliers_IDs)
2587 2598 # print(indexesJULIA)
2588 2599 if len(indexesJULIA > 1):
2589 2600 iJ = indexesJULIA
2590 2601 locs = [ (iJ[n]-iJ[n-1]) > 5 for n in range(len(iJ))]
2591 2602 locs_2 = numpy.where(locs)[0]
2592 2603 #print(locs_2, indexesJULIA[locs_2-1])
2593 2604 indexesYagi_up = numpy.append(indexesYagi_up, indexesJULIA[locs_2-1])
2594 2605 indexesYagi_down = numpy.append(indexesYagi_down, indexesJULIA[locs_2])
2595 2606
2596 2607
2597 2608 indexesYagi_up = numpy.append(indexesYagi_up,indexesJULIA[-1])
2598 2609 indexesYagi_down = numpy.append(indexesYagi_down,indexesJULIA[0])
2599 2610
2600 2611 indexesYagi_up = numpy.unique(indexesYagi_up)
2601 2612 indexesYagi_down = numpy.unique(indexesYagi_down)
2602 2613
2603 2614
2604 2615 aux_ind = [ numpy.arange( (self.offYagi + k)+1, (self.offYagi + k + self.nTxYagi)+1, 1, dtype=int) for k in indexesYagi_up]
2605 2616 indexesYagi_up = (numpy.asarray(aux_ind)).flatten()
2606 2617
2607 2618 aux_ind2 = [ numpy.arange( (k - self.nTxYagi)+1, k+1 , 1, dtype=int) for k in indexesYagi_down]
2608 2619 indexesYagi_down = (numpy.asarray(aux_ind2)).flatten()
2609 2620
2610 2621 indexesYagi = numpy.append(indexesYagi,indexesYagi_up)
2611 2622 indexesYagi = numpy.append(indexesYagi,indexesYagi_down)
2612 2623
2613 2624
2614 2625 indexesYagi = indexesYagi[ (indexesYagi >= 0) & (indexesYagi<profiles)]
2615 2626 indexesYagi = numpy.unique(indexesYagi)
2616 2627
2617 2628 #print("indexes: " ,indexes)
2618 2629 outs_lines = numpy.unique(indexes)
2619 2630 #print(outs_lines)
2620 2631
2621 2632 #Agrupando el histograma de outliers,
2622 2633 my_bins = numpy.linspace(0,int(profiles), div, endpoint=True)
2623 2634
2624 2635
2625 2636 hist, bins = numpy.histogram(outs_lines,bins=my_bins)
2626 2637 #print("hist: ",hist)
2627 2638 hist_outliers_indexes = numpy.where(hist >= self.thHistOutlier)[0] #es outlier
2628 2639 # print(hist_outliers_indexes)
2629 2640 if len(hist_outliers_indexes>0):
2630 2641 hist_outliers_indexes = numpy.append(hist_outliers_indexes,hist_outliers_indexes[-1]+1)
2631 2642
2632 2643 bins_outliers_indexes = [int(i)+1 for i in (bins[hist_outliers_indexes])] #
2633 2644 outlier_loc_index = []
2634 2645 #print("out indexes ", bins_outliers_indexes)
2635 2646
2636 2647 # if len(bins_outliers_indexes) <= 2:
2637 2648 # extprof = 0
2638 2649 # else:
2639 2650 # extprof = self.profileMargin
2640 2651
2641 2652 extprof = self.profileMargin
2642 2653
2643 2654 outlier_loc_index = [e for n in range(len(bins_outliers_indexes)) for e in range(bins_outliers_indexes[n]-extprof,bins_outliers_indexes[n] + extprof) ]
2644 2655 outlier_loc_index = numpy.asarray(outlier_loc_index)
2645 2656 # if len(outlier_loc_index)>1:
2646 2657 # ipmax = numpy.where(fpower==fpower.max())[0]
2647 2658 # print("pmax: ",ipmax)
2648 2659
2649 2660
2650 2661
2651 2662
2652 2663 #print("outliers Ids: ", outlier_loc_index, outlier_loc_index.shape)
2653 2664 outlier_loc_index = outlier_loc_index[ (outlier_loc_index >= 0) & (outlier_loc_index<profiles)]
2654 2665 #print("outliers final: ", outlier_loc_index)
2655 2666
2656 2667
2657 2668 if self.debug:
2658 2669 x, y = numpy.meshgrid(numpy.arange(profiles), self.heightList)
2659 2670 fig, ax = plt.subplots(nChannels,2,figsize=(8, 6))
2660 2671
2661 2672 for i in range(nChannels):
2662 2673 dat = data[i,:,:].real
2663 2674 dat = 10* numpy.log10((data[i,:,:] * numpy.conjugate(data[i,:,:])).real)
2664 2675 m = numpy.nanmean(dat)
2665 2676 o = numpy.nanstd(dat)
2666 2677 if nChannels>1:
2667 2678 c = ax[i][0].pcolormesh(x, y, dat.T, cmap ='jet', vmin = 60, vmax = 70)
2668 2679 ax[i][0].vlines(outs_lines,650,700, linestyles='dashed', label = 'outs', color='w')
2669 2680 #fig.colorbar(c)
2670 2681 ax[i][0].vlines(outlier_loc_index,700,750, linestyles='dashed', label = 'outs', color='r')
2671 2682 ax[i][1].hist(outs_lines,bins=my_bins)
2672 2683 if self.remYagi :
2673 2684 ax[0].vlines(indexesYagi,750,850, linestyles='dashed', label = 'yagi', color='m')
2674 2685 else:
2675 2686 c = ax[0].pcolormesh(x, y, dat.T, cmap ='jet', vmin = 60, vmax = (70+2*self.cohFactor))
2676 2687 ax[0].vlines(outs_lines,650,700, linestyles='dashed', label = 'outs', color='w')
2677 2688 #fig.colorbar(c)
2678 2689 ax[0].vlines(outlier_loc_index,700,750, linestyles='dashed', label = 'outs', color='r')
2679 2690
2680 2691 ax[1].hist(outs_lines,bins=my_bins)
2681 2692 if self.remYagi :
2682 2693 ax[0].vlines(indexesYagi,750,850, linestyles='dashed', label = 'yagi', color='m')
2683 2694 plt.show()
2684 2695
2685 2696
2686 2697
2687 2698
2688 2699 if self.remYagi and (self.currentTime < self.startTime and self.currentTime < self.endTime):
2689 2700 outlier_loc_index = numpy.append(outlier_loc_index,indexesYagi)
2690 2701
2691 2702 self.outliers_IDs_list = numpy.unique(outlier_loc_index)
2692 2703
2693 2704 #print("outs list: ", self.outliers_IDs_list)
2694 2705 return self.__buffer_data
2695 2706
2696 2707
2697 2708
2698 2709 def fillBuffer(self, data, datatime):
2699 2710
2700 2711 if self.__profIndex == 0:
2701 2712 self.__buffer_data = data.copy()
2702 2713
2703 2714 else:
2704 2715 self.__buffer_data = numpy.concatenate((self.__buffer_data,data), axis=1)#en perfiles
2705 2716 self.__profIndex += 1
2706 2717 self.__buffer_times.append(datatime)
2707 2718
2708 2719 def getData(self, data, datatime=None):
2709 2720
2710 2721 if self.__profIndex == 0:
2711 2722 self.__initime = datatime
2712 2723
2713 2724
2714 2725 self.__dataReady = False
2715 2726
2716 2727 self.fillBuffer(data, datatime)
2717 2728 dataBlock = None
2718 2729
2719 2730 if self.__profIndex == self.n:
2720 2731 #print("apnd : ",data)
2721 2732 dataBlock = self.filterSatsProfiles()
2722 2733 self.__dataReady = True
2723 2734
2724 2735 return dataBlock
2725 2736
2726 2737 if dataBlock is None:
2727 2738 return None, None
2728 2739
2729 2740
2730 2741
2731 2742 return dataBlock
2732 2743
2733 2744 def releaseBlock(self):
2734 2745
2735 2746 if self.n % self.lenProfileOut != 0:
2736 2747 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n))
2737 2748 return None
2738 2749
2739 2750 data = self.buffer[:,self.init_prof:self.end_prof:,:] #ch, prof, alt
2740 2751
2741 2752 self.init_prof = self.end_prof
2742 2753 self.end_prof += self.lenProfileOut
2743 2754 #print("data release shape: ",dataOut.data.shape, self.end_prof)
2744 2755 self.n_prof_released += 1
2745 2756
2746 2757 return data
2747 2758
2748 2759 def run(self, dataOut, n=None, navg=0.9, nProfilesOut=1, profile_margin=50, th_hist_outlier=15,minHei=None,nBins=10,
2749 2760 maxHei=None, minRef=None, maxRef=None, debug=False, remYagi=False, nProfYagi = 0, offYagi=0, minHJULIA=None, maxHJULIA=None,
2750 2761 idate=None,startH=None,endH=None, thfactor=1):
2751 2762
2752 2763 if not self.isConfig:
2753 2764 #print("init p idx: ", dataOut.profileIndex )
2754 2765 self.setup(dataOut,n=n, navg=navg,profileMargin=profile_margin,thHistOutlier=th_hist_outlier,minHei=minHei,
2755 2766 nBins=10, maxHei=maxHei, minRef=minRef, maxRef=maxRef, debug=debug, remYagi=remYagi, nProfYagi = nProfYagi,
2756 2767 offYagi=offYagi, minHJULIA=minHJULIA,maxHJULIA=maxHJULIA,idate=idate,startH=startH,endH=endH, thfactor=thfactor)
2757 2768
2758 2769 self.isConfig = True
2759 2770
2760 2771 dataBlock = None
2761 2772 self.currentTime = datetime.datetime.fromtimestamp(dataOut.utctime)
2762 2773
2763 2774 if not dataOut.buffer_empty: #hay datos acumulados
2764 2775
2765 2776 if self.init_prof == 0:
2766 2777 self.n_prof_released = 0
2767 2778 self.lenProfileOut = nProfilesOut
2768 2779 dataOut.flagNoData = False
2769 2780 #print("tp 2 ",dataOut.data.shape)
2770 2781
2771 2782 self.init_prof = 0
2772 2783 self.end_prof = self.lenProfileOut
2773 2784
2774 2785 dataOut.nProfiles = self.lenProfileOut
2775 2786 if nProfilesOut == 1:
2776 2787 dataOut.flagDataAsBlock = False
2777 2788 else:
2778 2789 dataOut.flagDataAsBlock = True
2779 2790 #print("prof: ",self.init_prof)
2780 2791 dataOut.flagNoData = False
2781 2792 if numpy.isin(self.n_prof_released, self.outliers_IDs_list):
2782 2793 #print("omitting: ", self.n_prof_released)
2783 2794 dataOut.flagNoData = True
2784 2795 dataOut.ippSeconds = self._ipp
2785 2796 dataOut.utctime = self.first_utcBlock + self.init_prof*self._ipp
2786 2797 # print("time: ", dataOut.utctime, self.first_utcBlock, self.init_prof,self._ipp,dataOut.ippSeconds)
2787 2798 #dataOut.data = self.releaseBlock()
2788 2799 #########################################################3
2789 2800 if self.n % self.lenProfileOut != 0:
2790 2801 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n))
2791 2802 return None
2792 2803
2793 2804 dataOut.data = None
2794 2805
2795 2806 if nProfilesOut == 1:
2796 2807 dataOut.data = self.buffer[:,self.end_prof-1,:] #ch, prof, alt
2797 2808 else:
2798 2809 dataOut.data = self.buffer[:,self.init_prof:self.end_prof,:] #ch, prof, alt
2799 2810
2800 2811 self.init_prof = self.end_prof
2801 2812 self.end_prof += self.lenProfileOut
2802 2813 #print("data release shape: ",dataOut.data.shape, self.end_prof, dataOut.flagNoData)
2803 2814 self.n_prof_released += 1
2804 2815
2805 2816 if self.end_prof >= (self.n +self.lenProfileOut):
2806 2817
2807 2818 self.init_prof = 0
2808 2819 self.__profIndex = 0
2809 2820 self.buffer = None
2810 2821 dataOut.buffer_empty = True
2811 2822 self.outliers_IDs_list = []
2812 2823 self.n_prof_released = 0
2813 2824 dataOut.flagNoData = False #enviar ultimo aunque sea outlier :(
2814 2825 #print("cleaning...", dataOut.buffer_empty)
2815 2826 dataOut.profileIndex = self.__profIndex
2816 2827 ####################################################################
2817 2828 return dataOut
2818 2829
2819 2830
2820 2831 #print("tp 223 ",dataOut.data.shape)
2821 2832 dataOut.flagNoData = True
2822 2833
2823 2834
2824 2835
2825 2836 try:
2826 2837 #dataBlock = self.getData(dataOut.data.reshape(self.nChannels,1,self.nHeights), dataOut.utctime)
2827 2838 dataBlock = self.getData(numpy.reshape(dataOut.data,(self.nChannels,1,self.nHeights)), dataOut.utctime)
2828 2839 self.__count_exec +=1
2829 2840 except Exception as e:
2830 2841 print("Error getting profiles data",self.__count_exec )
2831 2842 print(e)
2832 2843 sys.exit()
2833 2844
2834 2845 if self.__dataReady:
2835 2846 #print("omitting: ", len(self.outliers_IDs_list))
2836 2847 self.__count_exec = 0
2837 2848 #dataOut.data =
2838 2849 #self.buffer = numpy.flip(dataBlock, axis=1)
2839 2850 self.buffer = dataBlock
2840 2851 self.first_utcBlock = self.__initime
2841 2852 dataOut.utctime = self.__initime
2842 2853 dataOut.nProfiles = self.__profIndex
2843 2854 #dataOut.flagNoData = False
2844 2855 self.init_prof = 0
2845 2856 self.__profIndex = 0
2846 2857 self.__initime = None
2847 2858 dataBlock = None
2848 2859 self.__buffer_times = []
2849 2860 dataOut.error = False
2850 2861 dataOut.useInputBuffer = True
2851 2862 dataOut.buffer_empty = False
2852 2863 #print("1 ch: {} prof: {} hs: {}".format(int(dataOut.nChannels),int(dataOut.nProfiles),int(dataOut.nHeights)))
2853 2864
2854 2865
2855 2866
2856 2867 #print(self.__count_exec)
2857 2868
2858 2869 return dataOut
2859 2870
2860 2871
2861 2872
2862 2873
2863 2874 class remHeightsIppInterf(Operation):
2864 2875
2865 2876 def __init__(self, **kwargs):
2866 2877
2867 2878
2868 2879 Operation.__init__(self, **kwargs)
2869 2880
2870 2881 self.isConfig = False
2871 2882
2872 2883 self.heights_indx = None
2873 2884 self.heightsList = []
2874 2885
2875 2886 self.ipp1 = None
2876 2887 self.ipp2 = None
2877 2888 self.tx1 = None
2878 2889 self.tx2 = None
2879 2890 self.dh1 = None
2880 2891
2881 2892
2882 2893 def setup(self, dataOut, ipp1=None, ipp2=None, tx1=None, tx2=None, dh1=None,
2883 2894 idate=None, startH=None, endH=None):
2884 2895
2885 2896
2886 2897 self.ipp1 = ipp1
2887 2898 self.ipp2 = ipp2
2888 2899 self.tx1 = tx1
2889 2900 self.tx2 = tx2
2890 2901 self.dh1 = dh1
2891 2902
2892 2903 _maxIpp1R = dataOut.heightList.max()
2893 2904
2894 2905 _n_repeats = int(_maxIpp1R / ipp2)
2895 2906 _init_hIntf = (tx1 + ipp2/2)+ dh1
2896 2907 _n_hIntf = int(tx2 / dh1)
2897 2908
2898 2909 self.heightsList = [_init_hIntf+n*ipp2 for n in range(_n_repeats) ]
2899 2910 heiList = dataOut.heightList
2900 2911 self.heights_indx = [getHei_index(h,h,heiList)[0] for h in self.heightsList]
2901 2912
2902 2913 self.heights_indx = [ numpy.asarray([k for k in range(_n_hIntf+2)])+(getHei_index(h,h,heiList)[0] -1) for h in self.heightsList]
2903 2914
2904 2915 self.heights_indx = numpy.asarray(self.heights_indx )
2905 2916 self.isConfig = True
2906 2917 self.startTime = datetime.datetime.combine(idate,startH)
2907 2918 self.endTime = datetime.datetime.combine(idate,endH)
2908 2919 #print(self.startTime, self.endTime)
2909 2920 #print("nrepeats: ", _n_repeats, " _nH: ",_n_hIntf )
2910 2921
2911 2922 log.warning("Heights set to zero (km): ", self.name)
2912 2923 log.warning(str((dataOut.heightList[self.heights_indx].flatten())), self.name)
2913 2924 log.warning("Be careful with the selection of heights for noise calculation!")
2914 2925
2915 2926 def run(self, dataOut, ipp1=None, ipp2=None, tx1=None, tx2=None, dh1=None, idate=None,
2916 2927 startH=None, endH=None):
2917 2928 #print(locals().values())
2918 2929 if None in locals().values():
2919 2930 log.warning('Missing kwargs, invalid values """None""" ', self.name)
2920 2931 return dataOut
2921 2932
2922 2933
2923 2934 if not self.isConfig:
2924 2935 self.setup(dataOut, ipp1=ipp1, ipp2=ipp2, tx1=tx1, tx2=tx2, dh1=dh1,
2925 2936 idate=idate, startH=startH, endH=endH)
2926 2937
2927 2938 dataOut.flagProfilesByRange = False
2928 2939 currentTime = datetime.datetime.fromtimestamp(dataOut.utctime)
2929 2940
2930 2941 if currentTime < self.startTime or currentTime > self.endTime:
2931 2942 return dataOut
2932 2943
2933 2944 for ch in range(dataOut.data.shape[0]):
2934 2945
2935 2946 for hk in self.heights_indx.flatten():
2936 2947 if dataOut.data.ndim < 3:
2937 2948 dataOut.data[ch,hk] = 0.0 + 0.0j
2938 2949 else:
2939 2950 dataOut.data[ch,:,hk] = 0.0 + 0.0j
2940 2951
2941 2952 dataOut.flagProfilesByRange = True
2942 2953
2943 2954 return dataOut
2944 2955
2945 2956
2946 2957
2947 2958
2948 2959 class profiles2Block(Operation):
2949 2960 '''
2950 2961 Escrito: Joab Apaza
2951 2962
2952 2963 genera un bloque de perfiles
2953 2964
2954 2965
2955 2966 Out:
2956 2967 block
2957 2968 '''
2958 2969
2959 2970 isConfig = False
2960 2971 __buffer_data = []
2961 2972 __buffer_times = []
2962 2973 __profIndex = 0
2963 2974 __byTime = False
2964 2975 __initime = None
2965 2976 __lastdatatime = None
2966 2977 buffer = None
2967 2978 n = None
2968 2979 __dataReady = False
2969 2980 __nChannels = None
2970 2981 __nHeis = None
2971 2982
2972 2983 def __init__(self, **kwargs):
2973 2984
2974 2985 Operation.__init__(self, **kwargs)
2975 2986 self.isConfig = False
2976 2987
2977 2988 def setup(self,n=None, timeInterval=None):
2978 2989
2979 2990 if n == None and timeInterval == None:
2980 2991 raise ValueError("n or timeInterval should be specified ...")
2981 2992
2982 2993 if n != None:
2983 2994 self.n = n
2984 2995 self.__byTime = False
2985 2996 else:
2986 2997 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
2987 2998 self.n = 9999
2988 2999 self.__byTime = True
2989 3000
2990 3001 self.__profIndex = 0
2991 3002
2992 3003
2993 3004 def fillBuffer(self, data, datatime):
2994 3005
2995 3006 if self.__profIndex == 0:
2996 3007 self.__buffer_data = data.copy()
2997 3008
2998 3009 else:
2999 3010 self.__buffer_data = numpy.concatenate((self.__buffer_data,data), axis=1)#en perfiles
3000 3011 self.__profIndex += 1
3001 3012 self.__buffer_times.append(datatime)
3002 3013
3003 3014 def getData(self, data, datatime=None):
3004 3015 if self.__initime == None:
3005 3016 self.__initime = datatime
3006 3017
3007 3018 if data.ndim < 3:
3008 3019 data = data.reshape(self.__nChannels,1,self.__nHeis )
3009 3020
3010 3021 if self.__byTime:
3011 3022 dataBlock = self.byTime(data, datatime)
3012 3023 else:
3013 3024 dataBlock = self.byProfiles(data, datatime)
3014 3025
3015 3026
3016 3027 self.__lastdatatime = datatime
3017 3028
3018 3029 if dataBlock is None:
3019 3030 return None, None
3020 3031
3021 3032 return dataBlock, self.__buffer_times
3022 3033
3023 3034 def byProfiles(self, data, datatime):
3024 3035
3025 3036 self.__dataReady = False
3026 3037 dataBlock = None
3027 3038 # n = None
3028 3039 # print data
3029 3040 # raise
3030 3041 self.fillBuffer(data, datatime)
3031 3042
3032 3043 if self.__profIndex == self.n:
3033 3044 dataBlock = self.__buffer_data
3034 3045 self.__dataReady = True
3035 3046
3036 3047 return dataBlock
3037 3048
3038 3049 def byTime(self, data, datatime):
3039 3050
3040 3051 self.__dataReady = False
3041 3052 dataBlock = None
3042 3053 n = None
3043 3054
3044 3055 self.fillBuffer(data, datatime)
3045 3056
3046 3057 if (datatime - self.__initime) >= self.__integrationtime:
3047 3058 dataBlock = self.__buffer_data
3048 3059 self.n = self.__profIndex
3049 3060 self.__dataReady = True
3050 3061
3051 3062 return dataBlock
3052 3063
3053 3064
3054 3065 def run(self, dataOut, n=None, timeInterval=None, **kwargs):
3055 3066
3056 3067 if not self.isConfig:
3057 3068 self.setup(n=n, timeInterval=timeInterval, **kwargs)
3058 3069 self.__nChannels = dataOut.nChannels
3059 3070 self.__nHeis = len(dataOut.heightList)
3060 3071 self.isConfig = True
3061 3072
3062 3073 if dataOut.flagDataAsBlock:
3063 3074 """
3064 3075 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
3065 3076 """
3066 3077 raise ValueError("The data is already a block")
3067 3078 return
3068 3079 else:
3069 3080
3070 3081 dataBlock, timeBlock = self.getData(dataOut.data, dataOut.utctime)
3071 3082
3072 3083
3073 3084 # print(dataOut.data.shape)
3074 3085 # dataOut.timeInterval *= n
3075 3086 dataOut.flagNoData = True
3076 3087
3077 3088 if self.__dataReady:
3078 3089 dataOut.data = dataBlock
3079 3090 dataOut.flagDataAsBlock = True
3080 3091 dataOut.utctime = timeBlock[-1]
3081 3092 dataOut.nProfiles = self.__profIndex
3082 3093 # print avgdata, avgdatatime
3083 3094 # raise
3084 3095 dataOut.flagNoData = False
3085 3096 self.__profIndex = 0
3086 3097 self.__initime = None
3087 3098 #update Processing Header:
3088 3099 # print(dataOut.data.shape)
3089 3100
3090 3101 return dataOut No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now