##// END OF EJS Templates
schain-jc
RSS
Multiprocessing for writing Units(Spectral, Voltage and Parameters)
George Yong -
r1179:6414333e2ace
parent child
Show More
Side by Side Unified
Context file:
r1179:6414333e2ace Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1234 +1,1251
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
5 5 '''
6 6
7 7 import copy
8 8 import numpy
9 9 import datetime
10 10
11 11 from .jroheaderIO import SystemHeader, RadarControllerHeader
12 12
13 13
14 14 def getNumpyDtype(dataTypeCode):
15 15
16 16 if dataTypeCode == 0:
17 17 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
18 18 elif dataTypeCode == 1:
19 19 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
20 20 elif dataTypeCode == 2:
21 21 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
22 22 elif dataTypeCode == 3:
23 23 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
24 24 elif dataTypeCode == 4:
25 25 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
26 26 elif dataTypeCode == 5:
27 27 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
28 28 else:
29 29 raise ValueError('dataTypeCode was not defined')
30 30
31 31 return numpyDtype
32 32
33 33
34 34 def getDataTypeCode(numpyDtype):
35 35
36 36 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
37 37 datatype = 0
38 38 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
39 39 datatype = 1
40 40 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
41 41 datatype = 2
42 42 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
43 43 datatype = 3
44 44 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
45 45 datatype = 4
46 46 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
47 47 datatype = 5
48 48 else:
49 49 datatype = None
50 50
51 51 return datatype
52 52
53 53
54 54 def hildebrand_sekhon(data, navg):
55 55 """
56 56 This method is for the objective determination of the noise level in Doppler spectra. This
57 57 implementation technique is based on the fact that the standard deviation of the spectral
58 58 densities is equal to the mean spectral density for white Gaussian noise
59 59
60 60 Inputs:
61 61 Data : heights
62 62 navg : numbers of averages
63 63
64 64 Return:
65 65 mean : noise's level
66 66 """
67 67
68 sorted_spectrum = numpy.sort(data, axis=None)
69 nnoise = len(sorted_spectrum) # default to all points in the spectrum as noise
70 for npts in range(1, len(sorted_spectrum)+1):
71 partial = sorted_spectrum[:npts]
72 mean = partial.mean()
73 var = partial.var()
74 if var * navg < mean**2.:
75 nnoise = npts
76 else:
77 # partial spectrum no longer has characteristics of white noise
78 break
68 sortdata = numpy.sort(data, axis=None)
69 lenOfData = len(sortdata)
70 nums_min = lenOfData*0.2
71
72 if nums_min <= 5:
73
74 nums_min = 5
75
76 sump = 0.
77 sumq = 0.
78
79 j = 0
80 cont = 1
81
82 while((cont==1)and(j<lenOfData)):
83
84 sump += sortdata[j]
85 sumq += sortdata[j]**2
86
87 if j > nums_min:
88 rtest = float(j)/(j-1) + 1.0/navg
89 if ((sumq*j) > (rtest*sump**2)):
90 j = j - 1
91 sump = sump - sortdata[j]
92 sumq = sumq - sortdata[j]**2
93 cont = 0
94
95 j += 1
96
97 lnoise = sump /j
79 98
80 noise_spectrum = sorted_spectrum[:nnoise]
81 mean = noise_spectrum.mean()
82 return mean
99 return lnoise
83 100
84 101
85 102 class Beam:
86 103
87 104 def __init__(self):
88 105 self.codeList = []
89 106 self.azimuthList = []
90 107 self.zenithList = []
91 108
92 109
93 110 class GenericData(object):
94 111
95 112 flagNoData = True
96 113
97 114 def copy(self, inputObj=None):
98 115
99 116 if inputObj == None:
100 117 return copy.deepcopy(self)
101 118
102 119 for key in list(inputObj.__dict__.keys()):
103 120
104 121 attribute = inputObj.__dict__[key]
105 122
106 123 # If this attribute is a tuple or list
107 124 if type(inputObj.__dict__[key]) in (tuple, list):
108 125 self.__dict__[key] = attribute[:]
109 126 continue
110 127
111 128 # If this attribute is another object or instance
112 129 if hasattr(attribute, '__dict__'):
113 130 self.__dict__[key] = attribute.copy()
114 131 continue
115 132
116 133 self.__dict__[key] = inputObj.__dict__[key]
117 134
118 135 def deepcopy(self):
119 136
120 137 return copy.deepcopy(self)
121 138
122 139 def isEmpty(self):
123 140
124 141 return self.flagNoData
125 142
126 143
127 144 class JROData(GenericData):
128 145
129 146 # m_BasicHeader = BasicHeader()
130 147 # m_ProcessingHeader = ProcessingHeader()
131 148
132 149 systemHeaderObj = SystemHeader()
133 150
134 151 radarControllerHeaderObj = RadarControllerHeader()
135 152
136 153 # data = None
137 154
138 155 type = None
139 156
140 157 datatype = None # dtype but in string
141 158
142 159 # dtype = None
143 160
144 161 # nChannels = None
145 162
146 163 # nHeights = None
147 164
148 165 nProfiles = None
149 166
150 167 heightList = None
151 168
152 169 channelList = None
153 170
154 171 flagDiscontinuousBlock = False
155 172
156 173 useLocalTime = False
157 174
158 175 utctime = None
159 176
160 177 timeZone = None
161 178
162 179 dstFlag = None
163 180
164 181 errorCount = None
165 182
166 183 blocksize = None
167 184
168 185 # nCode = None
169 186 #
170 187 # nBaud = None
171 188 #
172 189 # code = None
173 190
174 191 flagDecodeData = False # asumo q la data no esta decodificada
175 192
176 193 flagDeflipData = False # asumo q la data no esta sin flip
177 194
178 195 flagShiftFFT = False
179 196
180 197 # ippSeconds = None
181 198
182 199 # timeInterval = None
183 200
184 201 nCohInt = None
185 202
186 203 # noise = None
187 204
188 205 windowOfFilter = 1
189 206
190 207 # Speed of ligth
191 208 C = 3e8
192 209
193 210 frequency = 49.92e6
194 211
195 212 realtime = False
196 213
197 214 beacon_heiIndexList = None
198 215
199 216 last_block = None
200 217
201 218 blocknow = None
202 219
203 220 azimuth = None
204 221
205 222 zenith = None
206 223
207 224 beam = Beam()
208 225
209 226 profileIndex = None
210 227
211 228 error = (0, '')
212 229
213 230 def __str__(self):
214 231
215 232 return '{} - {}'.format(self.type, self.getDatatime())
216 233
217 234 def getNoise(self):
218 235
219 236 raise NotImplementedError
220 237
221 238 def getNChannels(self):
222 239
223 240 return len(self.channelList)
224 241
225 242 def getChannelIndexList(self):
226 243
227 244 return list(range(self.nChannels))
228 245
229 246 def getNHeights(self):
230 247
231 248 return len(self.heightList)
232 249
233 250 def getHeiRange(self, extrapoints=0):
234 251
235 252 heis = self.heightList
236 253 # deltah = self.heightList[1] - self.heightList[0]
237 254 #
238 255 # heis.append(self.heightList[-1])
239 256
240 257 return heis
241 258
242 259 def getDeltaH(self):
243 260
244 261 delta = self.heightList[1] - self.heightList[0]
245 262
246 263 return delta
247 264
248 265 def getltctime(self):
249 266
250 267 if self.useLocalTime:
251 268 return self.utctime - self.timeZone * 60
252 269
253 270 return self.utctime
254 271
255 272 def getDatatime(self):
256 273
257 274 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
258 275 return datatimeValue
259 276
260 277 def getTimeRange(self):
261 278
262 279 datatime = []
263 280
264 281 datatime.append(self.ltctime)
265 282 datatime.append(self.ltctime + self.timeInterval + 1)
266 283
267 284 datatime = numpy.array(datatime)
268 285
269 286 return datatime
270 287
271 288 def getFmaxTimeResponse(self):
272 289
273 290 period = (10**-6) * self.getDeltaH() / (0.15)
274 291
275 292 PRF = 1. / (period * self.nCohInt)
276 293
277 294 fmax = PRF
278 295
279 296 return fmax
280 297
281 298 def getFmax(self):
282 299 PRF = 1. / (self.ippSeconds * self.nCohInt)
283 300
284 301 fmax = PRF
285 302 return fmax
286 303
287 304 def getVmax(self):
288 305
289 306 _lambda = self.C / self.frequency
290 307
291 308 vmax = self.getFmax() * _lambda / 2
292 309
293 310 return vmax
294 311
295 312 def get_ippSeconds(self):
296 313 '''
297 314 '''
298 315 return self.radarControllerHeaderObj.ippSeconds
299 316
300 317 def set_ippSeconds(self, ippSeconds):
301 318 '''
302 319 '''
303 320
304 321 self.radarControllerHeaderObj.ippSeconds = ippSeconds
305 322
306 323 return
307 324
308 325 def get_dtype(self):
309 326 '''
310 327 '''
311 328 return getNumpyDtype(self.datatype)
312 329
313 330 def set_dtype(self, numpyDtype):
314 331 '''
315 332 '''
316 333
317 334 self.datatype = getDataTypeCode(numpyDtype)
318 335
319 336 def get_code(self):
320 337 '''
321 338 '''
322 339 return self.radarControllerHeaderObj.code
323 340
324 341 def set_code(self, code):
325 342 '''
326 343 '''
327 344 self.radarControllerHeaderObj.code = code
328 345
329 346 return
330 347
331 348 def get_ncode(self):
332 349 '''
333 350 '''
334 351 return self.radarControllerHeaderObj.nCode
335 352
336 353 def set_ncode(self, nCode):
337 354 '''
338 355 '''
339 356 self.radarControllerHeaderObj.nCode = nCode
340 357
341 358 return
342 359
343 360 def get_nbaud(self):
344 361 '''
345 362 '''
346 363 return self.radarControllerHeaderObj.nBaud
347 364
348 365 def set_nbaud(self, nBaud):
349 366 '''
350 367 '''
351 368 self.radarControllerHeaderObj.nBaud = nBaud
352 369
353 370 return
354 371
355 372 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
356 373 channelIndexList = property(
357 374 getChannelIndexList, "I'm the 'channelIndexList' property.")
358 375 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
359 376 #noise = property(getNoise, "I'm the 'nHeights' property.")
360 377 datatime = property(getDatatime, "I'm the 'datatime' property")
361 378 ltctime = property(getltctime, "I'm the 'ltctime' property")
362 379 ippSeconds = property(get_ippSeconds, set_ippSeconds)
363 380 dtype = property(get_dtype, set_dtype)
364 381 # timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
365 382 code = property(get_code, set_code)
366 383 nCode = property(get_ncode, set_ncode)
367 384 nBaud = property(get_nbaud, set_nbaud)
368 385
369 386
370 387 class Voltage(JROData):
371 388
372 389 # data es un numpy array de 2 dmensiones (canales, alturas)
373 390 data = None
374 391
375 392 def __init__(self):
376 393 '''
377 394 Constructor
378 395 '''
379 396
380 397 self.useLocalTime = True
381 398
382 399 self.radarControllerHeaderObj = RadarControllerHeader()
383 400
384 401 self.systemHeaderObj = SystemHeader()
385 402
386 403 self.type = "Voltage"
387 404
388 405 self.data = None
389 406
390 407 # self.dtype = None
391 408
392 409 # self.nChannels = 0
393 410
394 411 # self.nHeights = 0
395 412
396 413 self.nProfiles = None
397 414
398 415 self.heightList = None
399 416
400 417 self.channelList = None
401 418
402 419 # self.channelIndexList = None
403 420
404 421 self.flagNoData = True
405 422
406 423 self.flagDiscontinuousBlock = False
407 424
408 425 self.utctime = None
409 426
410 427 self.timeZone = None
411 428
412 429 self.dstFlag = None
413 430
414 431 self.errorCount = None
415 432
416 433 self.nCohInt = None
417 434
418 435 self.blocksize = None
419 436
420 437 self.flagDecodeData = False # asumo q la data no esta decodificada
421 438
422 439 self.flagDeflipData = False # asumo q la data no esta sin flip
423 440
424 441 self.flagShiftFFT = False
425 442
426 443 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
427 444
428 445 self.profileIndex = 0
429 446
430 447 def getNoisebyHildebrand(self, channel=None):
431 448 """
432 449 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
433 450
434 451 Return:
435 452 noiselevel
436 453 """
437 454
438 455 if channel != None:
439 456 data = self.data[channel]
440 457 nChannels = 1
441 458 else:
442 459 data = self.data
443 460 nChannels = self.nChannels
444 461
445 462 noise = numpy.zeros(nChannels)
446 463 power = data * numpy.conjugate(data)
447 464
448 465 for thisChannel in range(nChannels):
449 466 if nChannels == 1:
450 467 daux = power[:].real
451 468 else:
452 469 daux = power[thisChannel, :].real
453 470 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
454 471
455 472 return noise
456 473
457 474 def getNoise(self, type=1, channel=None):
458 475
459 476 if type == 1:
460 477 noise = self.getNoisebyHildebrand(channel)
461 478
462 479 return noise
463 480
464 481 def getPower(self, channel=None):
465 482
466 483 if channel != None:
467 484 data = self.data[channel]
468 485 else:
469 486 data = self.data
470 487
471 488 power = data * numpy.conjugate(data)
472 489 powerdB = 10 * numpy.log10(power.real)
473 490 powerdB = numpy.squeeze(powerdB)
474 491
475 492 return powerdB
476 493
477 494 def getTimeInterval(self):
478 495
479 496 timeInterval = self.ippSeconds * self.nCohInt
480 497
481 498 return timeInterval
482 499
483 500 noise = property(getNoise, "I'm the 'nHeights' property.")
484 501 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
485 502
486 503
487 504 class Spectra(JROData):
488 505
489 506 # data spc es un numpy array de 2 dmensiones (canales, perfiles, alturas)
490 507 data_spc = None
491 508
492 509 # data cspc es un numpy array de 2 dmensiones (canales, pares, alturas)
493 510 data_cspc = None
494 511
495 512 # data dc es un numpy array de 2 dmensiones (canales, alturas)
496 513 data_dc = None
497 514
498 515 # data power
499 516 data_pwr = None
500 517
501 518 nFFTPoints = None
502 519
503 520 # nPairs = None
504 521
505 522 pairsList = None
506 523
507 524 nIncohInt = None
508 525
509 526 wavelength = None # Necesario para cacular el rango de velocidad desde la frecuencia
510 527
511 528 nCohInt = None # se requiere para determinar el valor de timeInterval
512 529
513 530 ippFactor = None
514 531
515 532 profileIndex = 0
516 533
517 534 plotting = "spectra"
518 535
519 536 def __init__(self):
520 537 '''
521 538 Constructor
522 539 '''
523 540
524 541 self.useLocalTime = True
525 542
526 543 self.radarControllerHeaderObj = RadarControllerHeader()
527 544
528 545 self.systemHeaderObj = SystemHeader()
529 546
530 547 self.type = "Spectra"
531 548
532 549 # self.data = None
533 550
534 551 # self.dtype = None
535 552
536 553 # self.nChannels = 0
537 554
538 555 # self.nHeights = 0
539 556
540 557 self.nProfiles = None
541 558
542 559 self.heightList = None
543 560
544 561 self.channelList = None
545 562
546 563 # self.channelIndexList = None
547 564
548 565 self.pairsList = None
549 566
550 567 self.flagNoData = True
551 568
552 569 self.flagDiscontinuousBlock = False
553 570
554 571 self.utctime = None
555 572
556 573 self.nCohInt = None
557 574
558 575 self.nIncohInt = None
559 576
560 577 self.blocksize = None
561 578
562 579 self.nFFTPoints = None
563 580
564 581 self.wavelength = None
565 582
566 583 self.flagDecodeData = False # asumo q la data no esta decodificada
567 584
568 585 self.flagDeflipData = False # asumo q la data no esta sin flip
569 586
570 587 self.flagShiftFFT = False
571 588
572 589 self.ippFactor = 1
573 590
574 591 #self.noise = None
575 592
576 593 self.beacon_heiIndexList = []
577 594
578 595 self.noise_estimation = None
579 596
580 597 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
581 598 """
582 599 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
583 600
584 601 Return:
585 602 noiselevel
586 603 """
587 604
588 605 noise = numpy.zeros(self.nChannels)
589 606
590 607 for channel in range(self.nChannels):
591 608 daux = self.data_spc[channel,
592 609 xmin_index:xmax_index, ymin_index:ymax_index]
593 610 noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
594 611
595 612 return noise
596 613
597 614 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
598 615
599 616 if self.noise_estimation is not None:
600 617 # this was estimated by getNoise Operation defined in jroproc_spectra.py
601 618 return self.noise_estimation
602 619 else:
603 620 noise = self.getNoisebyHildebrand(
604 621 xmin_index, xmax_index, ymin_index, ymax_index)
605 622 return noise
606 623
607 624 def getFreqRangeTimeResponse(self, extrapoints=0):
608 625
609 626 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
610 627 freqrange = deltafreq * \
611 628 (numpy.arange(self.nFFTPoints + extrapoints) -
612 629 self.nFFTPoints / 2.) - deltafreq / 2
613 630
614 631 return freqrange
615 632
616 633 def getAcfRange(self, extrapoints=0):
617 634
618 635 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
619 636 freqrange = deltafreq * \
620 637 (numpy.arange(self.nFFTPoints + extrapoints) -
621 638 self.nFFTPoints / 2.) - deltafreq / 2
622 639
623 640 return freqrange
624 641
625 642 def getFreqRange(self, extrapoints=0):
626 643
627 644 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
628 645 freqrange = deltafreq * \
629 646 (numpy.arange(self.nFFTPoints + extrapoints) -
630 647 self.nFFTPoints / 2.) - deltafreq / 2
631 648
632 649 return freqrange
633 650
634 651 def getVelRange(self, extrapoints=0):
635 652
636 653 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
637 654 velrange = deltav * (numpy.arange(self.nFFTPoints +
638 655 extrapoints) - self.nFFTPoints / 2.) # - deltav/2
639 656
640 657 return velrange
641 658
642 659 def getNPairs(self):
643 660
644 661 return len(self.pairsList)
645 662
646 663 def getPairsIndexList(self):
647 664
648 665 return list(range(self.nPairs))
649 666
650 667 def getNormFactor(self):
651 668
652 669 pwcode = 1
653 670
654 671 if self.flagDecodeData:
655 672 pwcode = numpy.sum(self.code[0]**2)
656 673 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
657 674 normFactor = self.nProfiles * self.nIncohInt * \
658 675 self.nCohInt * pwcode * self.windowOfFilter
659 676
660 677 return normFactor
661 678
662 679 def getFlagCspc(self):
663 680
664 681 if self.data_cspc is None:
665 682 return True
666 683
667 684 return False
668 685
669 686 def getFlagDc(self):
670 687
671 688 if self.data_dc is None:
672 689 return True
673 690
674 691 return False
675 692
676 693 def getTimeInterval(self):
677 694
678 695 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
679 696
680 697 return timeInterval
681 698
682 699 def getPower(self):
683 700
684 701 factor = self.normFactor
685 702 z = self.data_spc / factor
686 703 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
687 704 avg = numpy.average(z, axis=1)
688 705
689 706 return 10 * numpy.log10(avg)
690 707
691 708 def getCoherence(self, pairsList=None, phase=False):
692 709
693 710 z = []
694 711 if pairsList is None:
695 712 pairsIndexList = self.pairsIndexList
696 713 else:
697 714 pairsIndexList = []
698 715 for pair in pairsList:
699 716 if pair not in self.pairsList:
700 717 raise ValueError("Pair %s is not in dataOut.pairsList" % (
701 718 pair))
702 719 pairsIndexList.append(self.pairsList.index(pair))
703 720 for i in range(len(pairsIndexList)):
704 721 pair = self.pairsList[pairsIndexList[i]]
705 722 ccf = numpy.average(
706 723 self.data_cspc[pairsIndexList[i], :, :], axis=0)
707 724 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
708 725 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
709 726 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
710 727 if phase:
711 728 data = numpy.arctan2(avgcoherenceComplex.imag,
712 729 avgcoherenceComplex.real) * 180 / numpy.pi
713 730 else:
714 731 data = numpy.abs(avgcoherenceComplex)
715 732
716 733 z.append(data)
717 734
718 735 return numpy.array(z)
719 736
720 737 def setValue(self, value):
721 738
722 739 print("This property should not be initialized")
723 740
724 741 return
725 742
726 743 nPairs = property(getNPairs, setValue, "I'm the 'nPairs' property.")
727 744 pairsIndexList = property(
728 745 getPairsIndexList, setValue, "I'm the 'pairsIndexList' property.")
729 746 normFactor = property(getNormFactor, setValue,
730 747 "I'm the 'getNormFactor' property.")
731 748 flag_cspc = property(getFlagCspc, setValue)
732 749 flag_dc = property(getFlagDc, setValue)
733 750 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
734 751 timeInterval = property(getTimeInterval, setValue,
735 752 "I'm the 'timeInterval' property")
736 753
737 754
738 755 class SpectraHeis(Spectra):
739 756
740 757 data_spc = None
741 758
742 759 data_cspc = None
743 760
744 761 data_dc = None
745 762
746 763 nFFTPoints = None
747 764
748 765 # nPairs = None
749 766
750 767 pairsList = None
751 768
752 769 nCohInt = None
753 770
754 771 nIncohInt = None
755 772
756 773 def __init__(self):
757 774
758 775 self.radarControllerHeaderObj = RadarControllerHeader()
759 776
760 777 self.systemHeaderObj = SystemHeader()
761 778
762 779 self.type = "SpectraHeis"
763 780
764 781 # self.dtype = None
765 782
766 783 # self.nChannels = 0
767 784
768 785 # self.nHeights = 0
769 786
770 787 self.nProfiles = None
771 788
772 789 self.heightList = None
773 790
774 791 self.channelList = None
775 792
776 793 # self.channelIndexList = None
777 794
778 795 self.flagNoData = True
779 796
780 797 self.flagDiscontinuousBlock = False
781 798
782 799 # self.nPairs = 0
783 800
784 801 self.utctime = None
785 802
786 803 self.blocksize = None
787 804
788 805 self.profileIndex = 0
789 806
790 807 self.nCohInt = 1
791 808
792 809 self.nIncohInt = 1
793 810
794 811 def getNormFactor(self):
795 812 pwcode = 1
796 813 if self.flagDecodeData:
797 814 pwcode = numpy.sum(self.code[0]**2)
798 815
799 816 normFactor = self.nIncohInt * self.nCohInt * pwcode
800 817
801 818 return normFactor
802 819
803 820 def getTimeInterval(self):
804 821
805 822 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
806 823
807 824 return timeInterval
808 825
809 826 normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
810 827 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
811 828
812 829
813 830 class Fits(JROData):
814 831
815 832 heightList = None
816 833
817 834 channelList = None
818 835
819 836 flagNoData = True
820 837
821 838 flagDiscontinuousBlock = False
822 839
823 840 useLocalTime = False
824 841
825 842 utctime = None
826 843
827 844 timeZone = None
828 845
829 846 # ippSeconds = None
830 847
831 848 # timeInterval = None
832 849
833 850 nCohInt = None
834 851
835 852 nIncohInt = None
836 853
837 854 noise = None
838 855
839 856 windowOfFilter = 1
840 857
841 858 # Speed of ligth
842 859 C = 3e8
843 860
844 861 frequency = 49.92e6
845 862
846 863 realtime = False
847 864
848 865 def __init__(self):
849 866
850 867 self.type = "Fits"
851 868
852 869 self.nProfiles = None
853 870
854 871 self.heightList = None
855 872
856 873 self.channelList = None
857 874
858 875 # self.channelIndexList = None
859 876
860 877 self.flagNoData = True
861 878
862 879 self.utctime = None
863 880
864 881 self.nCohInt = 1
865 882
866 883 self.nIncohInt = 1
867 884
868 885 self.useLocalTime = True
869 886
870 887 self.profileIndex = 0
871 888
872 889 # self.utctime = None
873 890 # self.timeZone = None
874 891 # self.ltctime = None
875 892 # self.timeInterval = None
876 893 # self.header = None
877 894 # self.data_header = None
878 895 # self.data = None
879 896 # self.datatime = None
880 897 # self.flagNoData = False
881 898 # self.expName = ''
882 899 # self.nChannels = None
883 900 # self.nSamples = None
884 901 # self.dataBlocksPerFile = None
885 902 # self.comments = ''
886 903 #
887 904
888 905 def getltctime(self):
889 906
890 907 if self.useLocalTime:
891 908 return self.utctime - self.timeZone * 60
892 909
893 910 return self.utctime
894 911
895 912 def getDatatime(self):
896 913
897 914 datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
898 915 return datatime
899 916
900 917 def getTimeRange(self):
901 918
902 919 datatime = []
903 920
904 921 datatime.append(self.ltctime)
905 922 datatime.append(self.ltctime + self.timeInterval)
906 923
907 924 datatime = numpy.array(datatime)
908 925
909 926 return datatime
910 927
911 928 def getHeiRange(self):
912 929
913 930 heis = self.heightList
914 931
915 932 return heis
916 933
917 934 def getNHeights(self):
918 935
919 936 return len(self.heightList)
920 937
921 938 def getNChannels(self):
922 939
923 940 return len(self.channelList)
924 941
925 942 def getChannelIndexList(self):
926 943
927 944 return list(range(self.nChannels))
928 945
929 946 def getNoise(self, type=1):
930 947
931 948 #noise = numpy.zeros(self.nChannels)
932 949
933 950 if type == 1:
934 951 noise = self.getNoisebyHildebrand()
935 952
936 953 if type == 2:
937 954 noise = self.getNoisebySort()
938 955
939 956 if type == 3:
940 957 noise = self.getNoisebyWindow()
941 958
942 959 return noise
943 960
944 961 def getTimeInterval(self):
945 962
946 963 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
947 964
948 965 return timeInterval
949 966
950 967 datatime = property(getDatatime, "I'm the 'datatime' property")
951 968 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
952 969 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
953 970 channelIndexList = property(
954 971 getChannelIndexList, "I'm the 'channelIndexList' property.")
955 972 noise = property(getNoise, "I'm the 'nHeights' property.")
956 973
957 974 ltctime = property(getltctime, "I'm the 'ltctime' property")
958 975 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
959 976
960 977
961 978 class Correlation(JROData):
962 979
963 980 noise = None
964 981
965 982 SNR = None
966 983
967 984 #--------------------------------------------------
968 985
969 986 mode = None
970 987
971 988 split = False
972 989
973 990 data_cf = None
974 991
975 992 lags = None
976 993
977 994 lagRange = None
978 995
979 996 pairsList = None
980 997
981 998 normFactor = None
982 999
983 1000 #--------------------------------------------------
984 1001
985 1002 # calculateVelocity = None
986 1003
987 1004 nLags = None
988 1005
989 1006 nPairs = None
990 1007
991 1008 nAvg = None
992 1009
993 1010 def __init__(self):
994 1011 '''
995 1012 Constructor
996 1013 '''
997 1014 self.radarControllerHeaderObj = RadarControllerHeader()
998 1015
999 1016 self.systemHeaderObj = SystemHeader()
1000 1017
1001 1018 self.type = "Correlation"
1002 1019
1003 1020 self.data = None
1004 1021
1005 1022 self.dtype = None
1006 1023
1007 1024 self.nProfiles = None
1008 1025
1009 1026 self.heightList = None
1010 1027
1011 1028 self.channelList = None
1012 1029
1013 1030 self.flagNoData = True
1014 1031
1015 1032 self.flagDiscontinuousBlock = False
1016 1033
1017 1034 self.utctime = None
1018 1035
1019 1036 self.timeZone = None
1020 1037
1021 1038 self.dstFlag = None
1022 1039
1023 1040 self.errorCount = None
1024 1041
1025 1042 self.blocksize = None
1026 1043
1027 1044 self.flagDecodeData = False # asumo q la data no esta decodificada
1028 1045
1029 1046 self.flagDeflipData = False # asumo q la data no esta sin flip
1030 1047
1031 1048 self.pairsList = None
1032 1049
1033 1050 self.nPoints = None
1034 1051
1035 1052 def getPairsList(self):
1036 1053
1037 1054 return self.pairsList
1038 1055
1039 1056 def getNoise(self, mode=2):
1040 1057
1041 1058 indR = numpy.where(self.lagR == 0)[0][0]
1042 1059 indT = numpy.where(self.lagT == 0)[0][0]
1043 1060
1044 1061 jspectra0 = self.data_corr[:, :, indR, :]
1045 1062 jspectra = copy.copy(jspectra0)
1046 1063
1047 1064 num_chan = jspectra.shape[0]
1048 1065 num_hei = jspectra.shape[2]
1049 1066
1050 1067 freq_dc = jspectra.shape[1] / 2
1051 1068 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
1052 1069
1053 1070 if ind_vel[0] < 0:
1054 1071 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
1055 1072
1056 1073 if mode == 1:
1057 1074 jspectra[:, freq_dc, :] = (
1058 1075 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
1059 1076
1060 1077 if mode == 2:
1061 1078
1062 1079 vel = numpy.array([-2, -1, 1, 2])
1063 1080 xx = numpy.zeros([4, 4])
1064 1081
1065 1082 for fil in range(4):
1066 1083 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
1067 1084
1068 1085 xx_inv = numpy.linalg.inv(xx)
1069 1086 xx_aux = xx_inv[0, :]
1070 1087
1071 1088 for ich in range(num_chan):
1072 1089 yy = jspectra[ich, ind_vel, :]
1073 1090 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
1074 1091
1075 1092 junkid = jspectra[ich, freq_dc, :] <= 0
1076 1093 cjunkid = sum(junkid)
1077 1094
1078 1095 if cjunkid.any():
1079 1096 jspectra[ich, freq_dc, junkid.nonzero()] = (
1080 1097 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
1081 1098
1082 1099 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
1083 1100
1084 1101 return noise
1085 1102
1086 1103 def getTimeInterval(self):
1087 1104
1088 1105 timeInterval = self.ippSeconds * self.nCohInt * self.nProfiles
1089 1106
1090 1107 return timeInterval
1091 1108
1092 1109 def splitFunctions(self):
1093 1110
1094 1111 pairsList = self.pairsList
1095 1112 ccf_pairs = []
1096 1113 acf_pairs = []
1097 1114 ccf_ind = []
1098 1115 acf_ind = []
1099 1116 for l in range(len(pairsList)):
1100 1117 chan0 = pairsList[l][0]
1101 1118 chan1 = pairsList[l][1]
1102 1119
1103 1120 # Obteniendo pares de Autocorrelacion
1104 1121 if chan0 == chan1:
1105 1122 acf_pairs.append(chan0)
1106 1123 acf_ind.append(l)
1107 1124 else:
1108 1125 ccf_pairs.append(pairsList[l])
1109 1126 ccf_ind.append(l)
1110 1127
1111 1128 data_acf = self.data_cf[acf_ind]
1112 1129 data_ccf = self.data_cf[ccf_ind]
1113 1130
1114 1131 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
1115 1132
1116 1133 def getNormFactor(self):
1117 1134 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
1118 1135 acf_pairs = numpy.array(acf_pairs)
1119 1136 normFactor = numpy.zeros((self.nPairs, self.nHeights))
1120 1137
1121 1138 for p in range(self.nPairs):
1122 1139 pair = self.pairsList[p]
1123 1140
1124 1141 ch0 = pair[0]
1125 1142 ch1 = pair[1]
1126 1143
1127 1144 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
1128 1145 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
1129 1146 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
1130 1147
1131 1148 return normFactor
1132 1149
1133 1150 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
1134 1151 normFactor = property(getNormFactor, "I'm the 'normFactor property'")
1135 1152
1136 1153
1137 1154 class Parameters(Spectra):
1138 1155
1139 1156 experimentInfo = None # Information about the experiment
1140 1157
1141 1158 # Information from previous data
1142 1159
1143 1160 inputUnit = None # Type of data to be processed
1144 1161
1145 1162 operation = None # Type of operation to parametrize
1146 1163
1147 1164 # normFactor = None #Normalization Factor
1148 1165
1149 1166 groupList = None # List of Pairs, Groups, etc
1150 1167
1151 1168 # Parameters
1152 1169
1153 1170 data_param = None # Parameters obtained
1154 1171
1155 1172 data_pre = None # Data Pre Parametrization
1156 1173
1157 1174 data_SNR = None # Signal to Noise Ratio
1158 1175
1159 1176 # heightRange = None #Heights
1160 1177
1161 1178 abscissaList = None # Abscissa, can be velocities, lags or time
1162 1179
1163 1180 # noise = None #Noise Potency
1164 1181
1165 1182 utctimeInit = None # Initial UTC time
1166 1183
1167 1184 paramInterval = None # Time interval to calculate Parameters in seconds
1168 1185
1169 1186 useLocalTime = True
1170 1187
1171 1188 # Fitting
1172 1189
1173 1190 data_error = None # Error of the estimation
1174 1191
1175 1192 constants = None
1176 1193
1177 1194 library = None
1178 1195
1179 1196 # Output signal
1180 1197
1181 1198 outputInterval = None # Time interval to calculate output signal in seconds
1182 1199
1183 1200 data_output = None # Out signal
1184 1201
1185 1202 nAvg = None
1186 1203
1187 1204 noise_estimation = None
1188 1205
1189 1206 GauSPC = None # Fit gaussian SPC
1190 1207
1191 1208 def __init__(self):
1192 1209 '''
1193 1210 Constructor
1194 1211 '''
1195 1212 self.radarControllerHeaderObj = RadarControllerHeader()
1196 1213
1197 1214 self.systemHeaderObj = SystemHeader()
1198 1215
1199 1216 self.type = "Parameters"
1200 1217
1201 1218 def getTimeRange1(self, interval):
1202 1219
1203 1220 datatime = []
1204 1221
1205 1222 if self.useLocalTime:
1206 1223 time1 = self.utctimeInit - self.timeZone * 60
1207 1224 else:
1208 1225 time1 = self.utctimeInit
1209 1226
1210 1227 datatime.append(time1)
1211 1228 datatime.append(time1 + interval)
1212 1229 datatime = numpy.array(datatime)
1213 1230
1214 1231 return datatime
1215 1232
1216 1233 def getTimeInterval(self):
1217 1234
1218 1235 if hasattr(self, 'timeInterval1'):
1219 1236 return self.timeInterval1
1220 1237 else:
1221 1238 return self.paramInterval
1222 1239
1223 1240 def setValue(self, value):
1224 1241
1225 1242 print("This property should not be initialized")
1226 1243
1227 1244 return
1228 1245
1229 1246 def getNoise(self):
1230 1247
1231 1248 return self.spc_noise
1232 1249
1233 1250 timeInterval = property(getTimeInterval)
1234 1251 noise = property(getNoise, setValue, "I'm the 'Noise' property.") No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,2151 +1,2158
1 1 import os
2 2 import datetime
3 3 import numpy
4 4 import inspect
5 5 from .figure import Figure, isRealtime, isTimeInHourRange
6 6 from .plotting_codes import *
7
7 from schainpy.model.proc.jroproc_base import MPDecorator
8 from schainpy.utils import log
8 9
9 10 class FitGauPlot(Figure):
10 11
11 12 isConfig = None
12 13 __nsubplots = None
13 14
14 15 WIDTHPROF = None
15 16 HEIGHTPROF = None
16 17 PREFIX = 'fitgau'
17 18
18 19 def __init__(self, **kwargs):
19 20 Figure.__init__(self, **kwargs)
20 21 self.isConfig = False
21 22 self.__nsubplots = 1
22 23
23 24 self.WIDTH = 250
24 25 self.HEIGHT = 250
25 26 self.WIDTHPROF = 120
26 27 self.HEIGHTPROF = 0
27 28 self.counter_imagwr = 0
28 29
29 30 self.PLOT_CODE = SPEC_CODE
30 31
31 32 self.FTP_WEI = None
32 33 self.EXP_CODE = None
33 34 self.SUB_EXP_CODE = None
34 35 self.PLOT_POS = None
35 36
36 37 self.__xfilter_ena = False
37 38 self.__yfilter_ena = False
38 39
39 40 def getSubplots(self):
40 41
41 42 ncol = int(numpy.sqrt(self.nplots)+0.9)
42 43 nrow = int(self.nplots*1./ncol + 0.9)
43 44
44 45 return nrow, ncol
45 46
46 47 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
47 48
48 49 self.__showprofile = showprofile
49 50 self.nplots = nplots
50 51
51 52 ncolspan = 1
52 53 colspan = 1
53 54 if showprofile:
54 55 ncolspan = 3
55 56 colspan = 2
56 57 self.__nsubplots = 2
57 58
58 59 self.createFigure(id = id,
59 60 wintitle = wintitle,
60 61 widthplot = self.WIDTH + self.WIDTHPROF,
61 62 heightplot = self.HEIGHT + self.HEIGHTPROF,
62 63 show=show)
63 64
64 65 nrow, ncol = self.getSubplots()
65 66
66 67 counter = 0
67 68 for y in range(nrow):
68 69 for x in range(ncol):
69 70
70 71 if counter >= self.nplots:
71 72 break
72 73
73 74 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
74 75
75 76 if showprofile:
76 77 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
77 78
78 79 counter += 1
79 80
80 81 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
81 82 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
82 83 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
83 84 server=None, folder=None, username=None, password=None,
84 85 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
85 86 xaxis="frequency", colormap='jet', normFactor=None , GauSelector = 1):
86 87
87 88 """
88 89
89 90 Input:
90 91 dataOut :
91 92 id :
92 93 wintitle :
93 94 channelList :
94 95 showProfile :
95 96 xmin : None,
96 97 xmax : None,
97 98 ymin : None,
98 99 ymax : None,
99 100 zmin : None,
100 101 zmax : None
101 102 """
102 103 if realtime:
103 104 if not(isRealtime(utcdatatime = dataOut.utctime)):
104 105 print('Skipping this plot function')
105 106 return
106 107
107 108 if channelList == None:
108 109 channelIndexList = dataOut.channelIndexList
109 110 else:
110 111 channelIndexList = []
111 112 for channel in channelList:
112 113 if channel not in dataOut.channelList:
113 114 raise ValueError("Channel %d is not in dataOut.channelList" %channel)
114 115 channelIndexList.append(dataOut.channelList.index(channel))
115 116
116 117 # if normFactor is None:
117 118 # factor = dataOut.normFactor
118 119 # else:
119 120 # factor = normFactor
120 121 if xaxis == "frequency":
121 122 x = dataOut.spc_range[0]
122 123 xlabel = "Frequency (kHz)"
123 124
124 125 elif xaxis == "time":
125 126 x = dataOut.spc_range[1]
126 127 xlabel = "Time (ms)"
127 128
128 129 else:
129 130 x = dataOut.spc_range[2]
130 131 xlabel = "Velocity (m/s)"
131 132
132 133 ylabel = "Range (Km)"
133 134
134 135 y = dataOut.getHeiRange()
135 136
136 137 z = dataOut.GauSPC[:,GauSelector,:,:] #GauSelector] #dataOut.data_spc/factor
137 138 print('GausSPC', z[0,32,10:40])
138 139 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
139 140 zdB = 10*numpy.log10(z)
140 141
141 142 avg = numpy.average(z, axis=1)
142 143 avgdB = 10*numpy.log10(avg)
143 144
144 145 noise = dataOut.spc_noise
145 146 noisedB = 10*numpy.log10(noise)
146 147
147 148 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
148 149 title = wintitle + " Spectra"
149 150 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
150 151 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
151 152
152 153 if not self.isConfig:
153 154
154 155 nplots = len(channelIndexList)
155 156
156 157 self.setup(id=id,
157 158 nplots=nplots,
158 159 wintitle=wintitle,
159 160 showprofile=showprofile,
160 161 show=show)
161 162
162 163 if xmin == None: xmin = numpy.nanmin(x)
163 164 if xmax == None: xmax = numpy.nanmax(x)
164 165 if ymin == None: ymin = numpy.nanmin(y)
165 166 if ymax == None: ymax = numpy.nanmax(y)
166 167 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
167 168 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
168 169
169 170 self.FTP_WEI = ftp_wei
170 171 self.EXP_CODE = exp_code
171 172 self.SUB_EXP_CODE = sub_exp_code
172 173 self.PLOT_POS = plot_pos
173 174
174 175 self.isConfig = True
175 176
176 177 self.setWinTitle(title)
177 178
178 179 for i in range(self.nplots):
179 180 index = channelIndexList[i]
180 181 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
181 182 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
182 183 if len(dataOut.beam.codeList) != 0:
183 184 title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
184 185
185 186 axes = self.axesList[i*self.__nsubplots]
186 187 axes.pcolor(x, y, zdB[index,:,:],
187 188 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
188 189 xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
189 190 ticksize=9, cblabel='')
190 191
191 192 if self.__showprofile:
192 193 axes = self.axesList[i*self.__nsubplots +1]
193 194 axes.pline(avgdB[index,:], y,
194 195 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
195 196 xlabel='dB', ylabel='', title='',
196 197 ytick_visible=False,
197 198 grid='x')
198 199
199 200 noiseline = numpy.repeat(noisedB[index], len(y))
200 201 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
201 202
202 203 self.draw()
203 204
204 205 if figfile == None:
205 206 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
206 207 name = str_datetime
207 208 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
208 209 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
209 210 figfile = self.getFilename(name)
210 211
211 212 self.save(figpath=figpath,
212 213 figfile=figfile,
213 214 save=save,
214 215 ftp=ftp,
215 216 wr_period=wr_period,
216 217 thisDatetime=thisDatetime)
217 218
218 219
219 220
220 221 class MomentsPlot(Figure):
221 222
222 223 isConfig = None
223 224 __nsubplots = None
224 225
225 226 WIDTHPROF = None
226 227 HEIGHTPROF = None
227 228 PREFIX = 'prm'
228 def __init__(self, **kwargs):
229 Figure.__init__(self, **kwargs)
229 def __init__(self):
230 Figure.__init__(self)
230 231 self.isConfig = False
231 232 self.__nsubplots = 1
232 233
233 234 self.WIDTH = 280
234 235 self.HEIGHT = 250
235 236 self.WIDTHPROF = 120
236 237 self.HEIGHTPROF = 0
237 238 self.counter_imagwr = 0
238 239
239 240 self.PLOT_CODE = MOMENTS_CODE
240 241
241 242 self.FTP_WEI = None
242 243 self.EXP_CODE = None
243 244 self.SUB_EXP_CODE = None
244 245 self.PLOT_POS = None
245 246
246 247 def getSubplots(self):
247 248
248 249 ncol = int(numpy.sqrt(self.nplots)+0.9)
249 250 nrow = int(self.nplots*1./ncol + 0.9)
250 251
251 252 return nrow, ncol
252 253
253 254 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
254 255
255 256 self.__showprofile = showprofile
256 257 self.nplots = nplots
257 258
258 259 ncolspan = 1
259 260 colspan = 1
260 261 if showprofile:
261 262 ncolspan = 3
262 263 colspan = 2
263 264 self.__nsubplots = 2
264 265
265 266 self.createFigure(id = id,
266 267 wintitle = wintitle,
267 268 widthplot = self.WIDTH + self.WIDTHPROF,
268 269 heightplot = self.HEIGHT + self.HEIGHTPROF,
269 270 show=show)
270 271
271 272 nrow, ncol = self.getSubplots()
272 273
273 274 counter = 0
274 275 for y in range(nrow):
275 276 for x in range(ncol):
276 277
277 278 if counter >= self.nplots:
278 279 break
279 280
280 281 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
281 282
282 283 if showprofile:
283 284 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
284 285
285 286 counter += 1
286 287
287 288 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
288 289 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
289 290 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
290 291 server=None, folder=None, username=None, password=None,
291 292 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
292 293
293 294 """
294 295
295 296 Input:
296 297 dataOut :
297 298 id :
298 299 wintitle :
299 300 channelList :
300 301 showProfile :
301 302 xmin : None,
302 303 xmax : None,
303 304 ymin : None,
304 305 ymax : None,
305 306 zmin : None,
306 307 zmax : None
307 308 """
308 309
309 310 if dataOut.flagNoData:
310 311 return None
311 312
312 313 if realtime:
313 314 if not(isRealtime(utcdatatime = dataOut.utctime)):
314 315 print('Skipping this plot function')
315 316 return
316 317
317 318 if channelList == None:
318 319 channelIndexList = dataOut.channelIndexList
319 320 else:
320 321 channelIndexList = []
321 322 for channel in channelList:
322 323 if channel not in dataOut.channelList:
323 324 raise ValueError("Channel %d is not in dataOut.channelList")
324 325 channelIndexList.append(dataOut.channelList.index(channel))
325 326
326 327 factor = dataOut.normFactor
327 328 x = dataOut.abscissaList
328 329 y = dataOut.heightList
329 330
330 331 z = dataOut.data_pre[channelIndexList,:,:]/factor
331 332 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
332 333 avg = numpy.average(z, axis=1)
333 334 noise = dataOut.noise/factor
334 335
335 336 zdB = 10*numpy.log10(z)
336 337 avgdB = 10*numpy.log10(avg)
337 338 noisedB = 10*numpy.log10(noise)
338 339
339 340 #thisDatetime = dataOut.datatime
340 341 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
341 342 title = wintitle + " Parameters"
342 343 xlabel = "Velocity (m/s)"
343 344 ylabel = "Range (Km)"
344 345
345 346 update_figfile = False
346 347
347 348 if not self.isConfig:
348 349
349 350 nplots = len(channelIndexList)
350 351
351 352 self.setup(id=id,
352 353 nplots=nplots,
353 354 wintitle=wintitle,
354 355 showprofile=showprofile,
355 356 show=show)
356 357
357 358 if xmin == None: xmin = numpy.nanmin(x)
358 359 if xmax == None: xmax = numpy.nanmax(x)
359 360 if ymin == None: ymin = numpy.nanmin(y)
360 361 if ymax == None: ymax = numpy.nanmax(y)
361 362 if zmin == None: zmin = numpy.nanmin(avgdB)*0.9
362 363 if zmax == None: zmax = numpy.nanmax(avgdB)*0.9
363 364
364 365 self.FTP_WEI = ftp_wei
365 366 self.EXP_CODE = exp_code
366 367 self.SUB_EXP_CODE = sub_exp_code
367 368 self.PLOT_POS = plot_pos
368 369
369 370 self.isConfig = True
370 371 update_figfile = True
371 372
372 373 self.setWinTitle(title)
373 374
374 375 for i in range(self.nplots):
375 376 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
376 377 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[i], noisedB[i], str_datetime)
377 378 axes = self.axesList[i*self.__nsubplots]
378 379 axes.pcolor(x, y, zdB[i,:,:],
379 380 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
380 381 xlabel=xlabel, ylabel=ylabel, title=title,
381 382 ticksize=9, cblabel='')
382 383 #Mean Line
383 384 mean = dataOut.data_param[i, 1, :]
384 385 axes.addpline(mean, y, idline=0, color="black", linestyle="solid", lw=1)
385 386
386 387 if self.__showprofile:
387 388 axes = self.axesList[i*self.__nsubplots +1]
388 389 axes.pline(avgdB[i], y,
389 390 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
390 391 xlabel='dB', ylabel='', title='',
391 392 ytick_visible=False,
392 393 grid='x')
393 394
394 395 noiseline = numpy.repeat(noisedB[i], len(y))
395 396 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
396 397
397 398 self.draw()
398 399
399 400 self.save(figpath=figpath,
400 401 figfile=figfile,
401 402 save=save,
402 403 ftp=ftp,
403 404 wr_period=wr_period,
404 405 thisDatetime=thisDatetime)
405 406
406 407
407
408 408 class SkyMapPlot(Figure):
409 409
410 410 __isConfig = None
411 411 __nsubplots = None
412 412
413 413 WIDTHPROF = None
414 414 HEIGHTPROF = None
415 415 PREFIX = 'mmap'
416 416
417 417 def __init__(self, **kwargs):
418 418 Figure.__init__(self, **kwargs)
419 419 self.isConfig = False
420 420 self.__nsubplots = 1
421 421
422 422 # self.WIDTH = 280
423 423 # self.HEIGHT = 250
424 424 self.WIDTH = 600
425 425 self.HEIGHT = 600
426 426 self.WIDTHPROF = 120
427 427 self.HEIGHTPROF = 0
428 428 self.counter_imagwr = 0
429 429
430 430 self.PLOT_CODE = MSKYMAP_CODE
431 431
432 432 self.FTP_WEI = None
433 433 self.EXP_CODE = None
434 434 self.SUB_EXP_CODE = None
435 435 self.PLOT_POS = None
436 436
437 437 def getSubplots(self):
438 438
439 439 ncol = int(numpy.sqrt(self.nplots)+0.9)
440 440 nrow = int(self.nplots*1./ncol + 0.9)
441 441
442 442 return nrow, ncol
443 443
444 444 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
445 445
446 446 self.__showprofile = showprofile
447 447 self.nplots = nplots
448 448
449 449 ncolspan = 1
450 450 colspan = 1
451 451
452 452 self.createFigure(id = id,
453 453 wintitle = wintitle,
454 454 widthplot = self.WIDTH, #+ self.WIDTHPROF,
455 455 heightplot = self.HEIGHT,# + self.HEIGHTPROF,
456 456 show=show)
457 457
458 458 nrow, ncol = 1,1
459 459 counter = 0
460 460 x = 0
461 461 y = 0
462 462 self.addAxes(1, 1, 0, 0, 1, 1, True)
463 463
464 464 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
465 465 tmin=0, tmax=24, timerange=None,
466 466 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
467 467 server=None, folder=None, username=None, password=None,
468 468 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
469 469
470 470 """
471 471
472 472 Input:
473 473 dataOut :
474 474 id :
475 475 wintitle :
476 476 channelList :
477 477 showProfile :
478 478 xmin : None,
479 479 xmax : None,
480 480 ymin : None,
481 481 ymax : None,
482 482 zmin : None,
483 483 zmax : None
484 484 """
485 485
486 486 arrayParameters = dataOut.data_param
487 487 error = arrayParameters[:,-1]
488 488 indValid = numpy.where(error == 0)[0]
489 489 finalMeteor = arrayParameters[indValid,:]
490 490 finalAzimuth = finalMeteor[:,3]
491 491 finalZenith = finalMeteor[:,4]
492 492
493 493 x = finalAzimuth*numpy.pi/180
494 494 y = finalZenith
495 495 x1 = [dataOut.ltctime, dataOut.ltctime]
496 496
497 497 #thisDatetime = dataOut.datatime
498 498 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
499 499 title = wintitle + " Parameters"
500 500 xlabel = "Zonal Zenith Angle (deg) "
501 501 ylabel = "Meridional Zenith Angle (deg)"
502 502 update_figfile = False
503 503
504 504 if not self.isConfig:
505 505
506 506 nplots = 1
507 507
508 508 self.setup(id=id,
509 509 nplots=nplots,
510 510 wintitle=wintitle,
511 511 showprofile=showprofile,
512 512 show=show)
513 513
514 514 if self.xmin is None and self.xmax is None:
515 515 self.xmin, self.xmax = self.getTimeLim(x1, tmin, tmax, timerange)
516 516
517 517 if timerange != None:
518 518 self.timerange = timerange
519 519 else:
520 520 self.timerange = self.xmax - self.xmin
521 521
522 522 self.FTP_WEI = ftp_wei
523 523 self.EXP_CODE = exp_code
524 524 self.SUB_EXP_CODE = sub_exp_code
525 525 self.PLOT_POS = plot_pos
526 526 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
527 527 self.firstdate = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
528 528 self.isConfig = True
529 529 update_figfile = True
530 530
531 531 self.setWinTitle(title)
532 532
533 533 i = 0
534 534 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
535 535
536 536 axes = self.axesList[i*self.__nsubplots]
537 537 nevents = axes.x_buffer.shape[0] + x.shape[0]
538 538 title = "Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n" %(self.firstdate,str_datetime,nevents)
539 539 axes.polar(x, y,
540 540 title=title, xlabel=xlabel, ylabel=ylabel,
541 541 ticksize=9, cblabel='')
542 542
543 543 self.draw()
544 544
545 545 self.save(figpath=figpath,
546 546 figfile=figfile,
547 547 save=save,
548 548 ftp=ftp,
549 549 wr_period=wr_period,
550 550 thisDatetime=thisDatetime,
551 551 update_figfile=update_figfile)
552 552
553 553 if dataOut.ltctime >= self.xmax:
554 554 self.isConfigmagwr = wr_period
555 555 self.isConfig = False
556 556 update_figfile = True
557 557 axes.__firsttime = True
558 558 self.xmin += self.timerange
559 559 self.xmax += self.timerange
560 560
561 561
562 562
563 563
564 564 class WindProfilerPlot(Figure):
565 565
566 566 __isConfig = None
567 567 __nsubplots = None
568 568
569 569 WIDTHPROF = None
570 570 HEIGHTPROF = None
571 571 PREFIX = 'wind'
572 572
573 573 def __init__(self, **kwargs):
574 574 Figure.__init__(self, **kwargs)
575 575 self.timerange = None
576 576 self.isConfig = False
577 577 self.__nsubplots = 1
578 578
579 579 self.WIDTH = 800
580 580 self.HEIGHT = 300
581 581 self.WIDTHPROF = 120
582 582 self.HEIGHTPROF = 0
583 583 self.counter_imagwr = 0
584 584
585 585 self.PLOT_CODE = WIND_CODE
586 586
587 587 self.FTP_WEI = None
588 588 self.EXP_CODE = None
589 589 self.SUB_EXP_CODE = None
590 590 self.PLOT_POS = None
591 591 self.tmin = None
592 592 self.tmax = None
593 593
594 594 self.xmin = None
595 595 self.xmax = None
596 596
597 597 self.figfile = None
598 598
599 599 def getSubplots(self):
600 600
601 601 ncol = 1
602 602 nrow = self.nplots
603 603
604 604 return nrow, ncol
605 605
606 606 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
607 607
608 608 self.__showprofile = showprofile
609 609 self.nplots = nplots
610 610
611 611 ncolspan = 1
612 612 colspan = 1
613 613
614 614 self.createFigure(id = id,
615 615 wintitle = wintitle,
616 616 widthplot = self.WIDTH + self.WIDTHPROF,
617 617 heightplot = self.HEIGHT + self.HEIGHTPROF,
618 618 show=show)
619 619
620 620 nrow, ncol = self.getSubplots()
621 621
622 622 counter = 0
623 623 for y in range(nrow):
624 624 if counter >= self.nplots:
625 625 break
626 626
627 627 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
628 628 counter += 1
629 629
630 630 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='False',
631 631 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
632 632 zmax_ver = None, zmin_ver = None, SNRmin = None, SNRmax = None,
633 633 timerange=None, SNRthresh = None,
634 634 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
635 635 server=None, folder=None, username=None, password=None,
636 636 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
637 637 """
638 638
639 639 Input:
640 640 dataOut :
641 641 id :
642 642 wintitle :
643 643 channelList :
644 644 showProfile :
645 645 xmin : None,
646 646 xmax : None,
647 647 ymin : None,
648 648 ymax : None,
649 649 zmin : None,
650 650 zmax : None
651 651 """
652 652
653 653 # if timerange is not None:
654 654 # self.timerange = timerange
655 655 #
656 656 # tmin = None
657 657 # tmax = None
658 658
659 659 x = dataOut.getTimeRange1(dataOut.paramInterval)
660 660 y = dataOut.heightList
661 661 z = dataOut.data_output.copy()
662 662 nplots = z.shape[0] #Number of wind dimensions estimated
663 663 nplotsw = nplots
664 664
665 665
666 666 #If there is a SNR function defined
667 667 if dataOut.data_SNR is not None:
668 668 nplots += 1
669 669 SNR = dataOut.data_SNR
670 670 SNRavg = numpy.average(SNR, axis=0)
671 671
672 672 SNRdB = 10*numpy.log10(SNR)
673 673 SNRavgdB = 10*numpy.log10(SNRavg)
674 674
675 675 if SNRthresh == None: SNRthresh = -5.0
676 676 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
677 677
678 678 for i in range(nplotsw):
679 679 z[i,ind] = numpy.nan
680 680
681 681 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
682 682 #thisDatetime = datetime.datetime.now()
683 683 title = wintitle + "Wind"
684 684 xlabel = ""
685 685 ylabel = "Height (km)"
686 686 update_figfile = False
687 687
688 688 if not self.isConfig:
689 689
690 690 self.setup(id=id,
691 691 nplots=nplots,
692 692 wintitle=wintitle,
693 693 showprofile=showprofile,
694 694 show=show)
695 695
696 696 if timerange is not None:
697 697 self.timerange = timerange
698 698
699 699 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
700 700
701 701 if ymin == None: ymin = numpy.nanmin(y)
702 702 if ymax == None: ymax = numpy.nanmax(y)
703 703
704 704 if zmax == None: zmax = numpy.nanmax(abs(z[list(range(2)),:]))
705 705 #if numpy.isnan(zmax): zmax = 50
706 706 if zmin == None: zmin = -zmax
707 707
708 708 if nplotsw == 3:
709 709 if zmax_ver == None: zmax_ver = numpy.nanmax(abs(z[2,:]))
710 710 if zmin_ver == None: zmin_ver = -zmax_ver
711 711
712 712 if dataOut.data_SNR is not None:
713 713 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
714 714 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
715 715
716 716
717 717 self.FTP_WEI = ftp_wei
718 718 self.EXP_CODE = exp_code
719 719 self.SUB_EXP_CODE = sub_exp_code
720 720 self.PLOT_POS = plot_pos
721 721
722 722 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
723 723 self.isConfig = True
724 724 self.figfile = figfile
725 725 update_figfile = True
726 726
727 727 self.setWinTitle(title)
728 728
729 729 if ((self.xmax - x[1]) < (x[1]-x[0])):
730 730 x[1] = self.xmax
731 731
732 732 strWind = ['Zonal', 'Meridional', 'Vertical']
733 733 strCb = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
734 734 zmaxVector = [zmax, zmax, zmax_ver]
735 735 zminVector = [zmin, zmin, zmin_ver]
736 736 windFactor = [1,1,100]
737 737
738 738 for i in range(nplotsw):
739 739
740 740 title = "%s Wind: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
741 741 axes = self.axesList[i*self.__nsubplots]
742 742
743 743 z1 = z[i,:].reshape((1,-1))*windFactor[i]
744 744 #z1=numpy.ma.masked_where(z1==0.,z1)
745 745
746 746 axes.pcolorbuffer(x, y, z1,
747 747 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
748 748 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
749 749 ticksize=9, cblabel=strCb[i], cbsize="1%", colormap="seismic" )
750 750
751 751 if dataOut.data_SNR is not None:
752 752 i += 1
753 753 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
754 754 axes = self.axesList[i*self.__nsubplots]
755 755 SNRavgdB = SNRavgdB.reshape((1,-1))
756 756 axes.pcolorbuffer(x, y, SNRavgdB,
757 757 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
758 758 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
759 759 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
760 760
761 761 self.draw()
762 762
763 763 self.save(figpath=figpath,
764 764 figfile=figfile,
765 765 save=save,
766 766 ftp=ftp,
767 767 wr_period=wr_period,
768 768 thisDatetime=thisDatetime,
769 769 update_figfile=update_figfile)
770 770
771 771 if dataOut.ltctime + dataOut.paramInterval >= self.xmax:
772 772 self.counter_imagwr = wr_period
773 773 self.isConfig = False
774 774 update_figfile = True
775 775
776
776 @MPDecorator
777 777 class ParametersPlot(Figure):
778 778
779 779 __isConfig = None
780 780 __nsubplots = None
781 781
782 782 WIDTHPROF = None
783 783 HEIGHTPROF = None
784 784 PREFIX = 'param'
785 785
786 786 nplots = None
787 787 nchan = None
788 788
789 def __init__(self, **kwargs):
790 Figure.__init__(self, **kwargs)
789 def __init__(self):#, **kwargs):
790 Figure.__init__(self)#, **kwargs)
791 791 self.timerange = None
792 792 self.isConfig = False
793 793 self.__nsubplots = 1
794 794
795 795 self.WIDTH = 800
796 796 self.HEIGHT = 180
797 797 self.WIDTHPROF = 120
798 798 self.HEIGHTPROF = 0
799 799 self.counter_imagwr = 0
800 800
801 801 self.PLOT_CODE = RTI_CODE
802 802
803 803 self.FTP_WEI = None
804 804 self.EXP_CODE = None
805 805 self.SUB_EXP_CODE = None
806 806 self.PLOT_POS = None
807 807 self.tmin = None
808 808 self.tmax = None
809 809
810 810 self.xmin = None
811 811 self.xmax = None
812 812
813 813 self.figfile = None
814 814
815 815 def getSubplots(self):
816 816
817 817 ncol = 1
818 818 nrow = self.nplots
819 819
820 820 return nrow, ncol
821 821
822 822 def setup(self, id, nplots, wintitle, show=True):
823 823
824 824 self.nplots = nplots
825 825
826 826 ncolspan = 1
827 827 colspan = 1
828 828
829 829 self.createFigure(id = id,
830 830 wintitle = wintitle,
831 831 widthplot = self.WIDTH + self.WIDTHPROF,
832 832 heightplot = self.HEIGHT + self.HEIGHTPROF,
833 833 show=show)
834 834
835 835 nrow, ncol = self.getSubplots()
836 836
837 837 counter = 0
838 838 for y in range(nrow):
839 839 for x in range(ncol):
840 840
841 841 if counter >= self.nplots:
842 842 break
843 843
844 844 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
845 845
846 846 counter += 1
847 847
848 848 def run(self, dataOut, id, wintitle="", channelList=None, paramIndex = 0, colormap="jet",
849 849 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None, timerange=None,
850 850 showSNR=False, SNRthresh = -numpy.inf, SNRmin=None, SNRmax=None,
851 851 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
852 852 server=None, folder=None, username=None, password=None,
853 853 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, HEIGHT=None):
854 854 """
855 855
856 856 Input:
857 857 dataOut :
858 858 id :
859 859 wintitle :
860 860 channelList :
861 861 showProfile :
862 862 xmin : None,
863 863 xmax : None,
864 864 ymin : None,
865 865 ymax : None,
866 866 zmin : None,
867 867 zmax : None
868 868 """
869 if dataOut.flagNoData:
870 return dataOut
871
869 872
870 873 if HEIGHT is not None:
871 874 self.HEIGHT = HEIGHT
872 875
873 876
874 877 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
875 878 return
876 879
877 880 if channelList == None:
878 881 channelIndexList = list(range(dataOut.data_param.shape[0]))
879 882 else:
880 883 channelIndexList = []
881 884 for channel in channelList:
882 885 if channel not in dataOut.channelList:
883 886 raise ValueError("Channel %d is not in dataOut.channelList")
884 887 channelIndexList.append(dataOut.channelList.index(channel))
885 888
886 889 x = dataOut.getTimeRange1(dataOut.paramInterval)
887 890 y = dataOut.getHeiRange()
888 891
889 892 if dataOut.data_param.ndim == 3:
890 893 z = dataOut.data_param[channelIndexList,paramIndex,:]
891 894 else:
892 895 z = dataOut.data_param[channelIndexList,:]
893 896
894 897 if showSNR:
895 898 #SNR data
896 899 SNRarray = dataOut.data_SNR[channelIndexList,:]
897 900 SNRdB = 10*numpy.log10(SNRarray)
898 901 ind = numpy.where(SNRdB < SNRthresh)
899 902 z[ind] = numpy.nan
900 903
901 904 thisDatetime = dataOut.datatime
902 905 # thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
903 906 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
904 907 xlabel = ""
905 908 ylabel = "Range (Km)"
906 909
907 910 update_figfile = False
908 911
909 912 if not self.isConfig:
910 913
911 914 nchan = len(channelIndexList)
912 915 self.nchan = nchan
913 916 self.plotFact = 1
914 917 nplots = nchan
915 918
916 919 if showSNR:
917 920 nplots = nchan*2
918 921 self.plotFact = 2
919 922 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
920 923 if SNRmax == None: SNRmax = numpy.nanmax(SNRdB)
921 924
922 925 self.setup(id=id,
923 926 nplots=nplots,
924 927 wintitle=wintitle,
925 928 show=show)
926 929
927 930 if timerange != None:
928 931 self.timerange = timerange
929 932
930 933 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
931 934
932 935 if ymin == None: ymin = numpy.nanmin(y)
933 936 if ymax == None: ymax = numpy.nanmax(y)
934 937 if zmin == None: zmin = numpy.nanmin(z)
935 938 if zmax == None: zmax = numpy.nanmax(z)
936 939
937 940 self.FTP_WEI = ftp_wei
938 941 self.EXP_CODE = exp_code
939 942 self.SUB_EXP_CODE = sub_exp_code
940 943 self.PLOT_POS = plot_pos
941 944
942 945 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
943 946 self.isConfig = True
944 947 self.figfile = figfile
945 948 update_figfile = True
946 949
947 950 self.setWinTitle(title)
948 951
949 952 for i in range(self.nchan):
950 953 index = channelIndexList[i]
951 954 title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
952 955 axes = self.axesList[i*self.plotFact]
953 956 z1 = z[i,:].reshape((1,-1))
954 957 axes.pcolorbuffer(x, y, z1,
955 958 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
956 959 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
957 960 ticksize=9, cblabel='', cbsize="1%",colormap=colormap)
958 961
959 962 if showSNR:
960 963 title = "Channel %d SNR: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
961 964 axes = self.axesList[i*self.plotFact + 1]
962 965 SNRdB1 = SNRdB[i,:].reshape((1,-1))
963 966 axes.pcolorbuffer(x, y, SNRdB1,
964 967 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
965 968 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
966 969 ticksize=9, cblabel='', cbsize="1%",colormap='jet')
967 970
968 971
969 972 self.draw()
970 973
971 974 if dataOut.ltctime >= self.xmax:
972 975 self.counter_imagwr = wr_period
973 976 self.isConfig = False
974 977 update_figfile = True
975 978
976 979 self.save(figpath=figpath,
977 980 figfile=figfile,
978 981 save=save,
979 982 ftp=ftp,
980 983 wr_period=wr_period,
981 984 thisDatetime=thisDatetime,
982 985 update_figfile=update_figfile)
983 986
984
985
987 return dataOut
988 @MPDecorator
986 989 class Parameters1Plot(Figure):
987 990
988 991 __isConfig = None
989 992 __nsubplots = None
990 993
991 994 WIDTHPROF = None
992 995 HEIGHTPROF = None
993 996 PREFIX = 'prm'
994 997
995 def __init__(self, **kwargs):
996 Figure.__init__(self, **kwargs)
998 def __init__(self):
999 Figure.__init__(self)
997 1000 self.timerange = 2*60*60
998 1001 self.isConfig = False
999 1002 self.__nsubplots = 1
1000 1003
1001 1004 self.WIDTH = 800
1002 1005 self.HEIGHT = 180
1003 1006 self.WIDTHPROF = 120
1004 1007 self.HEIGHTPROF = 0
1005 1008 self.counter_imagwr = 0
1006 1009
1007 1010 self.PLOT_CODE = PARMS_CODE
1008 1011
1009 1012 self.FTP_WEI = None
1010 1013 self.EXP_CODE = None
1011 1014 self.SUB_EXP_CODE = None
1012 1015 self.PLOT_POS = None
1013 1016 self.tmin = None
1014 1017 self.tmax = None
1015 1018
1016 1019 self.xmin = None
1017 1020 self.xmax = None
1018 1021
1019 1022 self.figfile = None
1020 1023
1021 1024 def getSubplots(self):
1022 1025
1023 1026 ncol = 1
1024 1027 nrow = self.nplots
1025 1028
1026 1029 return nrow, ncol
1027 1030
1028 1031 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1029 1032
1030 1033 self.__showprofile = showprofile
1031 1034 self.nplots = nplots
1032 1035
1033 1036 ncolspan = 1
1034 1037 colspan = 1
1035 1038
1036 1039 self.createFigure(id = id,
1037 1040 wintitle = wintitle,
1038 1041 widthplot = self.WIDTH + self.WIDTHPROF,
1039 1042 heightplot = self.HEIGHT + self.HEIGHTPROF,
1040 1043 show=show)
1041 1044
1042 1045 nrow, ncol = self.getSubplots()
1043 1046
1044 1047 counter = 0
1045 1048 for y in range(nrow):
1046 1049 for x in range(ncol):
1047 1050
1048 1051 if counter >= self.nplots:
1049 1052 break
1050 1053
1051 1054 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1052 1055
1053 1056 if showprofile:
1054 1057 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
1055 1058
1056 1059 counter += 1
1057 1060
1058 1061 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
1059 1062 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,timerange=None,
1060 1063 parameterIndex = None, onlyPositive = False,
1061 1064 SNRthresh = -numpy.inf, SNR = True, SNRmin = None, SNRmax = None, onlySNR = False,
1062 1065 DOP = True,
1063 1066 zlabel = "", parameterName = "", parameterObject = "data_param",
1064 1067 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
1065 1068 server=None, folder=None, username=None, password=None,
1066 1069 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1067 1070 #print inspect.getargspec(self.run).args
1068 1071 """
1069 1072
1070 1073 Input:
1071 1074 dataOut :
1072 1075 id :
1073 1076 wintitle :
1074 1077 channelList :
1075 1078 showProfile :
1076 1079 xmin : None,
1077 1080 xmax : None,
1078 1081 ymin : None,
1079 1082 ymax : None,
1080 1083 zmin : None,
1081 1084 zmax : None
1082 1085 """
1086 if dataOut.flagNoData:
1087 return dataOut
1083 1088
1084 1089 data_param = getattr(dataOut, parameterObject)
1085 1090
1086 1091 if channelList == None:
1087 1092 channelIndexList = numpy.arange(data_param.shape[0])
1088 1093 else:
1089 1094 channelIndexList = numpy.array(channelList)
1090 1095
1091 1096 nchan = len(channelIndexList) #Number of channels being plotted
1092 1097
1093 1098 if nchan < 1:
1094 1099 return
1095 1100
1096 1101 nGraphsByChannel = 0
1097 1102
1098 1103 if SNR:
1099 1104 nGraphsByChannel += 1
1100 1105 if DOP:
1101 1106 nGraphsByChannel += 1
1102 1107
1103 1108 if nGraphsByChannel < 1:
1104 1109 return
1105 1110
1106 1111 nplots = nGraphsByChannel*nchan
1107 1112
1108 1113 if timerange is not None:
1109 1114 self.timerange = timerange
1110 1115
1111 1116 #tmin = None
1112 1117 #tmax = None
1113 1118 if parameterIndex == None:
1114 1119 parameterIndex = 1
1115 1120
1116 1121 x = dataOut.getTimeRange1(dataOut.paramInterval)
1117 1122 y = dataOut.heightList
1118 1123
1119 1124 if dataOut.data_param.ndim == 3:
1120 1125 z = dataOut.data_param[channelIndexList,parameterIndex,:]
1121 1126 else:
1122 1127 z = dataOut.data_param[channelIndexList,:]
1123 1128
1124 1129 if dataOut.data_SNR is not None:
1125 1130 if dataOut.data_SNR.ndim == 2:
1126 1131 SNRavg = numpy.average(dataOut.data_SNR, axis=0)
1127 1132 else:
1128 1133 SNRavg = dataOut.data_SNR
1129 1134 SNRdB = 10*numpy.log10(SNRavg)
1130 1135
1131 1136 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1132 1137 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
1133 1138 xlabel = ""
1134 1139 ylabel = "Range (Km)"
1135 1140
1136 1141 if onlyPositive:
1137 1142 colormap = "jet"
1138 1143 zmin = 0
1139 1144 else: colormap = "RdBu_r"
1140 1145
1141 1146 if not self.isConfig:
1142 1147
1143 1148 self.setup(id=id,
1144 1149 nplots=nplots,
1145 1150 wintitle=wintitle,
1146 1151 showprofile=showprofile,
1147 1152 show=show)
1148 1153
1149 1154 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1150 1155
1151 1156 if ymin == None: ymin = numpy.nanmin(y)
1152 1157 if ymax == None: ymax = numpy.nanmax(y)
1153 1158 if zmin == None: zmin = numpy.nanmin(z)
1154 1159 if zmax == None: zmax = numpy.nanmax(z)
1155 1160
1156 1161 if SNR:
1157 1162 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
1158 1163 if SNRmax == None: SNRmax = numpy.nanmax(SNRdB)
1159 1164
1160 1165 self.FTP_WEI = ftp_wei
1161 1166 self.EXP_CODE = exp_code
1162 1167 self.SUB_EXP_CODE = sub_exp_code
1163 1168 self.PLOT_POS = plot_pos
1164 1169
1165 1170 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1166 1171 self.isConfig = True
1167 1172 self.figfile = figfile
1168 1173
1169 1174 self.setWinTitle(title)
1170 1175
1171 1176 if ((self.xmax - x[1]) < (x[1]-x[0])):
1172 1177 x[1] = self.xmax
1173 1178
1174 1179 for i in range(nchan):
1175 1180
1176 1181 if (SNR and not onlySNR): j = 2*i
1177 1182 else: j = i
1178 1183
1179 1184 j = nGraphsByChannel*i
1180 1185
1181 1186 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1182 1187 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
1183 1188
1184 1189 if not onlySNR:
1185 1190 axes = self.axesList[j*self.__nsubplots]
1186 1191 z1 = z[i,:].reshape((1,-1))
1187 1192 axes.pcolorbuffer(x, y, z1,
1188 1193 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
1189 1194 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
1190 1195 ticksize=9, cblabel=zlabel, cbsize="1%")
1191 1196
1192 1197 if DOP:
1193 1198 title = "%s Channel %d: %s" %(parameterName, channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1194 1199
1195 1200 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1196 1201 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
1197 1202 axes = self.axesList[j]
1198 1203 z1 = z[i,:].reshape((1,-1))
1199 1204 axes.pcolorbuffer(x, y, z1,
1200 1205 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
1201 1206 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
1202 1207 ticksize=9, cblabel=zlabel, cbsize="1%")
1203 1208
1204 1209 if SNR:
1205 1210 title = "Channel %d Signal Noise Ratio (SNR): %s" %(channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1206 1211 axes = self.axesList[(j)*self.__nsubplots]
1207 1212 if not onlySNR:
1208 1213 axes = self.axesList[(j + 1)*self.__nsubplots]
1209 1214
1210 1215 axes = self.axesList[(j + nGraphsByChannel-1)]
1211 1216 z1 = SNRdB.reshape((1,-1))
1212 1217 axes.pcolorbuffer(x, y, z1,
1213 1218 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1214 1219 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap="jet",
1215 1220 ticksize=9, cblabel=zlabel, cbsize="1%")
1216 1221
1217 1222
1218 1223
1219 1224 self.draw()
1220 1225
1221 1226 if x[1] >= self.axesList[0].xmax:
1222 1227 self.counter_imagwr = wr_period
1223 1228 self.isConfig = False
1224 1229 self.figfile = None
1225 1230
1226 1231 self.save(figpath=figpath,
1227 1232 figfile=figfile,
1228 1233 save=save,
1229 1234 ftp=ftp,
1230 1235 wr_period=wr_period,
1231 1236 thisDatetime=thisDatetime,
1232 1237 update_figfile=False)
1238 return dataOut
1233 1239
1234 1240 class SpectralFittingPlot(Figure):
1235 1241
1236 1242 __isConfig = None
1237 1243 __nsubplots = None
1238 1244
1239 1245 WIDTHPROF = None
1240 1246 HEIGHTPROF = None
1241 1247 PREFIX = 'prm'
1242 1248
1243 1249
1244 1250 N = None
1245 1251 ippSeconds = None
1246 1252
1247 1253 def __init__(self, **kwargs):
1248 1254 Figure.__init__(self, **kwargs)
1249 1255 self.isConfig = False
1250 1256 self.__nsubplots = 1
1251 1257
1252 1258 self.PLOT_CODE = SPECFIT_CODE
1253 1259
1254 1260 self.WIDTH = 450
1255 1261 self.HEIGHT = 250
1256 1262 self.WIDTHPROF = 0
1257 1263 self.HEIGHTPROF = 0
1258 1264
1259 1265 def getSubplots(self):
1260 1266
1261 1267 ncol = int(numpy.sqrt(self.nplots)+0.9)
1262 1268 nrow = int(self.nplots*1./ncol + 0.9)
1263 1269
1264 1270 return nrow, ncol
1265 1271
1266 1272 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
1267 1273
1268 1274 showprofile = False
1269 1275 self.__showprofile = showprofile
1270 1276 self.nplots = nplots
1271 1277
1272 1278 ncolspan = 5
1273 1279 colspan = 4
1274 1280 if showprofile:
1275 1281 ncolspan = 5
1276 1282 colspan = 4
1277 1283 self.__nsubplots = 2
1278 1284
1279 1285 self.createFigure(id = id,
1280 1286 wintitle = wintitle,
1281 1287 widthplot = self.WIDTH + self.WIDTHPROF,
1282 1288 heightplot = self.HEIGHT + self.HEIGHTPROF,
1283 1289 show=show)
1284 1290
1285 1291 nrow, ncol = self.getSubplots()
1286 1292
1287 1293 counter = 0
1288 1294 for y in range(nrow):
1289 1295 for x in range(ncol):
1290 1296
1291 1297 if counter >= self.nplots:
1292 1298 break
1293 1299
1294 1300 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1295 1301
1296 1302 if showprofile:
1297 1303 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
1298 1304
1299 1305 counter += 1
1300 1306
1301 1307 def run(self, dataOut, id, cutHeight=None, fit=False, wintitle="", channelList=None, showprofile=True,
1302 1308 xmin=None, xmax=None, ymin=None, ymax=None,
1303 1309 save=False, figpath='./', figfile=None, show=True):
1304 1310
1305 1311 """
1306 1312
1307 1313 Input:
1308 1314 dataOut :
1309 1315 id :
1310 1316 wintitle :
1311 1317 channelList :
1312 1318 showProfile :
1313 1319 xmin : None,
1314 1320 xmax : None,
1315 1321 zmin : None,
1316 1322 zmax : None
1317 1323 """
1318 1324
1319 1325 if cutHeight==None:
1320 1326 h=270
1321 1327 heightindex = numpy.abs(cutHeight - dataOut.heightList).argmin()
1322 1328 cutHeight = dataOut.heightList[heightindex]
1323 1329
1324 1330 factor = dataOut.normFactor
1325 1331 x = dataOut.abscissaList[:-1]
1326 1332 #y = dataOut.getHeiRange()
1327 1333
1328 1334 z = dataOut.data_pre[:,:,heightindex]/factor
1329 1335 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1330 1336 avg = numpy.average(z, axis=1)
1331 1337 listChannels = z.shape[0]
1332 1338
1333 1339 #Reconstruct Function
1334 1340 if fit==True:
1335 1341 groupArray = dataOut.groupList
1336 1342 listChannels = groupArray.reshape((groupArray.size))
1337 1343 listChannels.sort()
1338 1344 spcFitLine = numpy.zeros(z.shape)
1339 1345 constants = dataOut.constants
1340 1346
1341 1347 nGroups = groupArray.shape[0]
1342 1348 nChannels = groupArray.shape[1]
1343 1349 nProfiles = z.shape[1]
1344 1350
1345 1351 for f in range(nGroups):
1346 1352 groupChann = groupArray[f,:]
1347 1353 p = dataOut.data_param[f,:,heightindex]
1348 1354 # p = numpy.array([ 89.343967,0.14036615,0.17086219,18.89835291,1.58388365,1.55099167])
1349 1355 fitLineAux = dataOut.library.modelFunction(p, constants)*nProfiles
1350 1356 fitLineAux = fitLineAux.reshape((nChannels,nProfiles))
1351 1357 spcFitLine[groupChann,:] = fitLineAux
1352 1358 # spcFitLine = spcFitLine/factor
1353 1359
1354 1360 z = z[listChannels,:]
1355 1361 spcFitLine = spcFitLine[listChannels,:]
1356 1362 spcFitLinedB = 10*numpy.log10(spcFitLine)
1357 1363
1358 1364 zdB = 10*numpy.log10(z)
1359 1365 #thisDatetime = dataOut.datatime
1360 1366 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1361 1367 title = wintitle + " Doppler Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1362 1368 xlabel = "Velocity (m/s)"
1363 1369 ylabel = "Spectrum"
1364 1370
1365 1371 if not self.isConfig:
1366 1372
1367 1373 nplots = listChannels.size
1368 1374
1369 1375 self.setup(id=id,
1370 1376 nplots=nplots,
1371 1377 wintitle=wintitle,
1372 1378 showprofile=showprofile,
1373 1379 show=show)
1374 1380
1375 1381 if xmin == None: xmin = numpy.nanmin(x)
1376 1382 if xmax == None: xmax = numpy.nanmax(x)
1377 1383 if ymin == None: ymin = numpy.nanmin(zdB)
1378 1384 if ymax == None: ymax = numpy.nanmax(zdB)+2
1379 1385
1380 1386 self.isConfig = True
1381 1387
1382 1388 self.setWinTitle(title)
1383 1389 for i in range(self.nplots):
1384 1390 # title = "Channel %d: %4.2fdB" %(dataOut.channelList[i]+1, noisedB[i])
1385 1391 title = "Height %4.1f km\nChannel %d:" %(cutHeight, listChannels[i])
1386 1392 axes = self.axesList[i*self.__nsubplots]
1387 1393 if fit == False:
1388 1394 axes.pline(x, zdB[i,:],
1389 1395 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1390 1396 xlabel=xlabel, ylabel=ylabel, title=title
1391 1397 )
1392 1398 if fit == True:
1393 1399 fitline=spcFitLinedB[i,:]
1394 1400 y=numpy.vstack([zdB[i,:],fitline] )
1395 1401 legendlabels=['Data','Fitting']
1396 1402 axes.pmultilineyaxis(x, y,
1397 1403 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1398 1404 xlabel=xlabel, ylabel=ylabel, title=title,
1399 1405 legendlabels=legendlabels, marker=None,
1400 1406 linestyle='solid', grid='both')
1401 1407
1402 1408 self.draw()
1403 1409
1404 1410 self.save(figpath=figpath,
1405 1411 figfile=figfile,
1406 1412 save=save,
1407 1413 ftp=ftp,
1408 1414 wr_period=wr_period,
1409 1415 thisDatetime=thisDatetime)
1410 1416
1411 1417
1412 1418 class EWDriftsPlot(Figure):
1413 1419
1414 1420 __isConfig = None
1415 1421 __nsubplots = None
1416 1422
1417 1423 WIDTHPROF = None
1418 1424 HEIGHTPROF = None
1419 1425 PREFIX = 'drift'
1420 1426
1421 1427 def __init__(self, **kwargs):
1422 1428 Figure.__init__(self, **kwargs)
1423 1429 self.timerange = 2*60*60
1424 1430 self.isConfig = False
1425 1431 self.__nsubplots = 1
1426 1432
1427 1433 self.WIDTH = 800
1428 1434 self.HEIGHT = 150
1429 1435 self.WIDTHPROF = 120
1430 1436 self.HEIGHTPROF = 0
1431 1437 self.counter_imagwr = 0
1432 1438
1433 1439 self.PLOT_CODE = EWDRIFT_CODE
1434 1440
1435 1441 self.FTP_WEI = None
1436 1442 self.EXP_CODE = None
1437 1443 self.SUB_EXP_CODE = None
1438 1444 self.PLOT_POS = None
1439 1445 self.tmin = None
1440 1446 self.tmax = None
1441 1447
1442 1448 self.xmin = None
1443 1449 self.xmax = None
1444 1450
1445 1451 self.figfile = None
1446 1452
1447 1453 def getSubplots(self):
1448 1454
1449 1455 ncol = 1
1450 1456 nrow = self.nplots
1451 1457
1452 1458 return nrow, ncol
1453 1459
1454 1460 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1455 1461
1456 1462 self.__showprofile = showprofile
1457 1463 self.nplots = nplots
1458 1464
1459 1465 ncolspan = 1
1460 1466 colspan = 1
1461 1467
1462 1468 self.createFigure(id = id,
1463 1469 wintitle = wintitle,
1464 1470 widthplot = self.WIDTH + self.WIDTHPROF,
1465 1471 heightplot = self.HEIGHT + self.HEIGHTPROF,
1466 1472 show=show)
1467 1473
1468 1474 nrow, ncol = self.getSubplots()
1469 1475
1470 1476 counter = 0
1471 1477 for y in range(nrow):
1472 1478 if counter >= self.nplots:
1473 1479 break
1474 1480
1475 1481 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
1476 1482 counter += 1
1477 1483
1478 1484 def run(self, dataOut, id, wintitle="", channelList=None,
1479 1485 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
1480 1486 zmaxVertical = None, zminVertical = None, zmaxZonal = None, zminZonal = None,
1481 1487 timerange=None, SNRthresh = -numpy.inf, SNRmin = None, SNRmax = None, SNR_1 = False,
1482 1488 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
1483 1489 server=None, folder=None, username=None, password=None,
1484 1490 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1485 1491 """
1486 1492
1487 1493 Input:
1488 1494 dataOut :
1489 1495 id :
1490 1496 wintitle :
1491 1497 channelList :
1492 1498 showProfile :
1493 1499 xmin : None,
1494 1500 xmax : None,
1495 1501 ymin : None,
1496 1502 ymax : None,
1497 1503 zmin : None,
1498 1504 zmax : None
1499 1505 """
1500 1506
1501 1507 if timerange is not None:
1502 1508 self.timerange = timerange
1503 1509
1504 1510 tmin = None
1505 1511 tmax = None
1506 1512
1507 1513 x = dataOut.getTimeRange1(dataOut.outputInterval)
1508 1514 # y = dataOut.heightList
1509 1515 y = dataOut.heightList
1510 1516
1511 1517 z = dataOut.data_output
1512 1518 nplots = z.shape[0] #Number of wind dimensions estimated
1513 1519 nplotsw = nplots
1514 1520
1515 1521 #If there is a SNR function defined
1516 1522 if dataOut.data_SNR is not None:
1517 1523 nplots += 1
1518 1524 SNR = dataOut.data_SNR
1519 1525
1520 1526 if SNR_1:
1521 1527 SNR += 1
1522 1528
1523 1529 SNRavg = numpy.average(SNR, axis=0)
1524 1530
1525 1531 SNRdB = 10*numpy.log10(SNR)
1526 1532 SNRavgdB = 10*numpy.log10(SNRavg)
1527 1533
1528 1534 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
1529 1535
1530 1536 for i in range(nplotsw):
1531 1537 z[i,ind] = numpy.nan
1532 1538
1533 1539
1534 1540 showprofile = False
1535 1541 # thisDatetime = dataOut.datatime
1536 1542 thisDatetime = datetime.datetime.utcfromtimestamp(x[1])
1537 1543 title = wintitle + " EW Drifts"
1538 1544 xlabel = ""
1539 1545 ylabel = "Height (Km)"
1540 1546
1541 1547 if not self.isConfig:
1542 1548
1543 1549 self.setup(id=id,
1544 1550 nplots=nplots,
1545 1551 wintitle=wintitle,
1546 1552 showprofile=showprofile,
1547 1553 show=show)
1548 1554
1549 1555 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1550 1556
1551 1557 if ymin == None: ymin = numpy.nanmin(y)
1552 1558 if ymax == None: ymax = numpy.nanmax(y)
1553 1559
1554 1560 if zmaxZonal == None: zmaxZonal = numpy.nanmax(abs(z[0,:]))
1555 1561 if zminZonal == None: zminZonal = -zmaxZonal
1556 1562 if zmaxVertical == None: zmaxVertical = numpy.nanmax(abs(z[1,:]))
1557 1563 if zminVertical == None: zminVertical = -zmaxVertical
1558 1564
1559 1565 if dataOut.data_SNR is not None:
1560 1566 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
1561 1567 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
1562 1568
1563 1569 self.FTP_WEI = ftp_wei
1564 1570 self.EXP_CODE = exp_code
1565 1571 self.SUB_EXP_CODE = sub_exp_code
1566 1572 self.PLOT_POS = plot_pos
1567 1573
1568 1574 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1569 1575 self.isConfig = True
1570 1576
1571 1577
1572 1578 self.setWinTitle(title)
1573 1579
1574 1580 if ((self.xmax - x[1]) < (x[1]-x[0])):
1575 1581 x[1] = self.xmax
1576 1582
1577 1583 strWind = ['Zonal','Vertical']
1578 1584 strCb = 'Velocity (m/s)'
1579 1585 zmaxVector = [zmaxZonal, zmaxVertical]
1580 1586 zminVector = [zminZonal, zminVertical]
1581 1587
1582 1588 for i in range(nplotsw):
1583 1589
1584 1590 title = "%s Drifts: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1585 1591 axes = self.axesList[i*self.__nsubplots]
1586 1592
1587 1593 z1 = z[i,:].reshape((1,-1))
1588 1594
1589 1595 axes.pcolorbuffer(x, y, z1,
1590 1596 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
1591 1597 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1592 1598 ticksize=9, cblabel=strCb, cbsize="1%", colormap="RdBu_r")
1593 1599
1594 1600 if dataOut.data_SNR is not None:
1595 1601 i += 1
1596 1602 if SNR_1:
1597 1603 title = "Signal Noise Ratio + 1 (SNR+1): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1598 1604 else:
1599 1605 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1600 1606 axes = self.axesList[i*self.__nsubplots]
1601 1607 SNRavgdB = SNRavgdB.reshape((1,-1))
1602 1608
1603 1609 axes.pcolorbuffer(x, y, SNRavgdB,
1604 1610 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1605 1611 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1606 1612 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
1607 1613
1608 1614 self.draw()
1609 1615
1610 1616 if x[1] >= self.axesList[0].xmax:
1611 1617 self.counter_imagwr = wr_period
1612 1618 self.isConfig = False
1613 1619 self.figfile = None
1614 1620
1615 1621
1616 1622
1617 1623
1618 1624 class PhasePlot(Figure):
1619 1625
1620 1626 __isConfig = None
1621 1627 __nsubplots = None
1622 1628
1623 1629 PREFIX = 'mphase'
1624 1630
1625 1631
1626 1632 def __init__(self, **kwargs):
1627 1633 Figure.__init__(self, **kwargs)
1628 1634 self.timerange = 24*60*60
1629 1635 self.isConfig = False
1630 1636 self.__nsubplots = 1
1631 1637 self.counter_imagwr = 0
1632 1638 self.WIDTH = 600
1633 1639 self.HEIGHT = 300
1634 1640 self.WIDTHPROF = 120
1635 1641 self.HEIGHTPROF = 0
1636 1642 self.xdata = None
1637 1643 self.ydata = None
1638 1644
1639 1645 self.PLOT_CODE = MPHASE_CODE
1640 1646
1641 1647 self.FTP_WEI = None
1642 1648 self.EXP_CODE = None
1643 1649 self.SUB_EXP_CODE = None
1644 1650 self.PLOT_POS = None
1645 1651
1646 1652
1647 1653 self.filename_phase = None
1648 1654
1649 1655 self.figfile = None
1650 1656
1651 1657 def getSubplots(self):
1652 1658
1653 1659 ncol = 1
1654 1660 nrow = 1
1655 1661
1656 1662 return nrow, ncol
1657 1663
1658 1664 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1659 1665
1660 1666 self.__showprofile = showprofile
1661 1667 self.nplots = nplots
1662 1668
1663 1669 ncolspan = 7
1664 1670 colspan = 6
1665 1671 self.__nsubplots = 2
1666 1672
1667 1673 self.createFigure(id = id,
1668 1674 wintitle = wintitle,
1669 1675 widthplot = self.WIDTH+self.WIDTHPROF,
1670 1676 heightplot = self.HEIGHT+self.HEIGHTPROF,
1671 1677 show=show)
1672 1678
1673 1679 nrow, ncol = self.getSubplots()
1674 1680
1675 1681 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1676 1682
1677 1683
1678 1684 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1679 1685 xmin=None, xmax=None, ymin=None, ymax=None,
1680 1686 timerange=None,
1681 1687 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1682 1688 server=None, folder=None, username=None, password=None,
1683 1689 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1684 1690
1685 1691
1686 1692 tmin = None
1687 1693 tmax = None
1688 1694 x = dataOut.getTimeRange1(dataOut.outputInterval)
1689 1695 y = dataOut.getHeiRange()
1690 1696
1691 1697
1692 1698 #thisDatetime = dataOut.datatime
1693 1699 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
1694 1700 title = wintitle + " Phase of Beacon Signal" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1695 1701 xlabel = "Local Time"
1696 1702 ylabel = "Phase"
1697 1703
1698 1704
1699 1705 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1700 1706 phase_beacon = dataOut.data_output
1701 1707 update_figfile = False
1702 1708
1703 1709 if not self.isConfig:
1704 1710
1705 1711 self.nplots = phase_beacon.size
1706 1712
1707 1713 self.setup(id=id,
1708 1714 nplots=self.nplots,
1709 1715 wintitle=wintitle,
1710 1716 showprofile=showprofile,
1711 1717 show=show)
1712 1718
1713 1719 if timerange is not None:
1714 1720 self.timerange = timerange
1715 1721
1716 1722 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1717 1723
1718 1724 if ymin == None: ymin = numpy.nanmin(phase_beacon) - 10.0
1719 1725 if ymax == None: ymax = numpy.nanmax(phase_beacon) + 10.0
1720 1726
1721 1727 self.FTP_WEI = ftp_wei
1722 1728 self.EXP_CODE = exp_code
1723 1729 self.SUB_EXP_CODE = sub_exp_code
1724 1730 self.PLOT_POS = plot_pos
1725 1731
1726 1732 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1727 1733 self.isConfig = True
1728 1734 self.figfile = figfile
1729 1735 self.xdata = numpy.array([])
1730 1736 self.ydata = numpy.array([])
1731 1737
1732 1738 #open file beacon phase
1733 1739 path = '%s%03d' %(self.PREFIX, self.id)
1734 1740 beacon_file = os.path.join(path,'%s.txt'%self.name)
1735 1741 self.filename_phase = os.path.join(figpath,beacon_file)
1736 1742 update_figfile = True
1737 1743
1738 1744
1739 1745 #store data beacon phase
1740 1746 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1741 1747
1742 1748 self.setWinTitle(title)
1743 1749
1744 1750
1745 1751 title = "Phase Offset %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1746 1752
1747 1753 legendlabels = ["phase %d"%(chan) for chan in numpy.arange(self.nplots)]
1748 1754
1749 1755 axes = self.axesList[0]
1750 1756
1751 1757 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1752 1758
1753 1759 if len(self.ydata)==0:
1754 1760 self.ydata = phase_beacon.reshape(-1,1)
1755 1761 else:
1756 1762 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1757 1763
1758 1764
1759 1765 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1760 1766 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1761 1767 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1762 1768 XAxisAsTime=True, grid='both'
1763 1769 )
1764 1770
1765 1771 self.draw()
1766 1772
1767 1773 self.save(figpath=figpath,
1768 1774 figfile=figfile,
1769 1775 save=save,
1770 1776 ftp=ftp,
1771 1777 wr_period=wr_period,
1772 1778 thisDatetime=thisDatetime,
1773 1779 update_figfile=update_figfile)
1774 1780
1775 1781 if dataOut.ltctime + dataOut.outputInterval >= self.xmax:
1776 1782 self.counter_imagwr = wr_period
1777 1783 self.isConfig = False
1778 1784 update_figfile = True
1779 1785
1780 1786
1781 1787
1782 1788 class NSMeteorDetection1Plot(Figure):
1783 1789
1784 1790 isConfig = None
1785 1791 __nsubplots = None
1786 1792
1787 1793 WIDTHPROF = None
1788 1794 HEIGHTPROF = None
1789 1795 PREFIX = 'nsm'
1790 1796
1791 1797 zminList = None
1792 1798 zmaxList = None
1793 1799 cmapList = None
1794 1800 titleList = None
1795 1801 nPairs = None
1796 1802 nChannels = None
1797 1803 nParam = None
1798 1804
1799 1805 def __init__(self, **kwargs):
1800 1806 Figure.__init__(self, **kwargs)
1801 1807 self.isConfig = False
1802 1808 self.__nsubplots = 1
1803 1809
1804 1810 self.WIDTH = 750
1805 1811 self.HEIGHT = 250
1806 1812 self.WIDTHPROF = 120
1807 1813 self.HEIGHTPROF = 0
1808 1814 self.counter_imagwr = 0
1809 1815
1810 1816 self.PLOT_CODE = SPEC_CODE
1811 1817
1812 1818 self.FTP_WEI = None
1813 1819 self.EXP_CODE = None
1814 1820 self.SUB_EXP_CODE = None
1815 1821 self.PLOT_POS = None
1816 1822
1817 1823 self.__xfilter_ena = False
1818 1824 self.__yfilter_ena = False
1819 1825
1820 1826 def getSubplots(self):
1821 1827
1822 1828 ncol = 3
1823 1829 nrow = int(numpy.ceil(self.nplots/3.0))
1824 1830
1825 1831 return nrow, ncol
1826 1832
1827 1833 def setup(self, id, nplots, wintitle, show=True):
1828 1834
1829 1835 self.nplots = nplots
1830 1836
1831 1837 ncolspan = 1
1832 1838 colspan = 1
1833 1839
1834 1840 self.createFigure(id = id,
1835 1841 wintitle = wintitle,
1836 1842 widthplot = self.WIDTH + self.WIDTHPROF,
1837 1843 heightplot = self.HEIGHT + self.HEIGHTPROF,
1838 1844 show=show)
1839 1845
1840 1846 nrow, ncol = self.getSubplots()
1841 1847
1842 1848 counter = 0
1843 1849 for y in range(nrow):
1844 1850 for x in range(ncol):
1845 1851
1846 1852 if counter >= self.nplots:
1847 1853 break
1848 1854
1849 1855 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1850 1856
1851 1857 counter += 1
1852 1858
1853 1859 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
1854 1860 xmin=None, xmax=None, ymin=None, ymax=None, SNRmin=None, SNRmax=None,
1855 1861 vmin=None, vmax=None, wmin=None, wmax=None, mode = 'SA',
1856 1862 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1857 1863 server=None, folder=None, username=None, password=None,
1858 1864 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
1859 1865 xaxis="frequency"):
1860 1866
1861 1867 """
1862 1868
1863 1869 Input:
1864 1870 dataOut :
1865 1871 id :
1866 1872 wintitle :
1867 1873 channelList :
1868 1874 showProfile :
1869 1875 xmin : None,
1870 1876 xmax : None,
1871 1877 ymin : None,
1872 1878 ymax : None,
1873 1879 zmin : None,
1874 1880 zmax : None
1875 1881 """
1876 1882 #SEPARAR EN DOS PLOTS
1877 1883 nParam = dataOut.data_param.shape[1] - 3
1878 1884
1879 1885 utctime = dataOut.data_param[0,0]
1880 1886 tmet = dataOut.data_param[:,1].astype(int)
1881 1887 hmet = dataOut.data_param[:,2].astype(int)
1882 1888
1883 1889 x = dataOut.abscissaList
1884 1890 y = dataOut.heightList
1885 1891
1886 1892 z = numpy.zeros((nParam, y.size, x.size - 1))
1887 1893 z[:,:] = numpy.nan
1888 1894 z[:,hmet,tmet] = dataOut.data_param[:,3:].T
1889 1895 z[0,:,:] = 10*numpy.log10(z[0,:,:])
1890 1896
1891 1897 xlabel = "Time (s)"
1892 1898 ylabel = "Range (km)"
1893 1899
1894 1900 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
1895 1901
1896 1902 if not self.isConfig:
1897 1903
1898 1904 nplots = nParam
1899 1905
1900 1906 self.setup(id=id,
1901 1907 nplots=nplots,
1902 1908 wintitle=wintitle,
1903 1909 show=show)
1904 1910
1905 1911 if xmin is None: xmin = numpy.nanmin(x)
1906 1912 if xmax is None: xmax = numpy.nanmax(x)
1907 1913 if ymin is None: ymin = numpy.nanmin(y)
1908 1914 if ymax is None: ymax = numpy.nanmax(y)
1909 1915 if SNRmin is None: SNRmin = numpy.nanmin(z[0,:])
1910 1916 if SNRmax is None: SNRmax = numpy.nanmax(z[0,:])
1911 1917 if vmax is None: vmax = numpy.nanmax(numpy.abs(z[1,:]))
1912 1918 if vmin is None: vmin = -vmax
1913 1919 if wmin is None: wmin = 0
1914 1920 if wmax is None: wmax = 50
1915 1921
1916 1922 pairsList = dataOut.groupList
1917 1923 self.nPairs = len(dataOut.groupList)
1918 1924
1919 1925 zminList = [SNRmin, vmin, cmin] + [pmin]*self.nPairs
1920 1926 zmaxList = [SNRmax, vmax, cmax] + [pmax]*self.nPairs
1921 1927 titleList = ["SNR","Radial Velocity","Coherence"]
1922 1928 cmapList = ["jet","RdBu_r","jet"]
1923 1929
1924 1930 for i in range(self.nPairs):
1925 1931 strAux1 = "Phase Difference "+ str(pairsList[i][0]) + str(pairsList[i][1])
1926 1932 titleList = titleList + [strAux1]
1927 1933 cmapList = cmapList + ["RdBu_r"]
1928 1934
1929 1935 self.zminList = zminList
1930 1936 self.zmaxList = zmaxList
1931 1937 self.cmapList = cmapList
1932 1938 self.titleList = titleList
1933 1939
1934 1940 self.FTP_WEI = ftp_wei
1935 1941 self.EXP_CODE = exp_code
1936 1942 self.SUB_EXP_CODE = sub_exp_code
1937 1943 self.PLOT_POS = plot_pos
1938 1944
1939 1945 self.isConfig = True
1940 1946
1941 1947 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
1942 1948
1943 1949 for i in range(nParam):
1944 1950 title = self.titleList[i] + ": " +str_datetime
1945 1951 axes = self.axesList[i]
1946 1952 axes.pcolor(x, y, z[i,:].T,
1947 1953 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=self.zminList[i], zmax=self.zmaxList[i],
1948 1954 xlabel=xlabel, ylabel=ylabel, title=title, colormap=self.cmapList[i],ticksize=9, cblabel='')
1949 1955 self.draw()
1950 1956
1951 1957 if figfile == None:
1952 1958 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
1953 1959 name = str_datetime
1954 1960 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1955 1961 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
1956 1962 figfile = self.getFilename(name)
1957 1963
1958 1964 self.save(figpath=figpath,
1959 1965 figfile=figfile,
1960 1966 save=save,
1961 1967 ftp=ftp,
1962 1968 wr_period=wr_period,
1963 1969 thisDatetime=thisDatetime)
1964 1970
1965 1971
1966 1972 class NSMeteorDetection2Plot(Figure):
1967 1973
1968 1974 isConfig = None
1969 1975 __nsubplots = None
1970 1976
1971 1977 WIDTHPROF = None
1972 1978 HEIGHTPROF = None
1973 1979 PREFIX = 'nsm'
1974 1980
1975 1981 zminList = None
1976 1982 zmaxList = None
1977 1983 cmapList = None
1978 1984 titleList = None
1979 1985 nPairs = None
1980 1986 nChannels = None
1981 1987 nParam = None
1982 1988
1983 1989 def __init__(self, **kwargs):
1984 1990 Figure.__init__(self, **kwargs)
1985 1991 self.isConfig = False
1986 1992 self.__nsubplots = 1
1987 1993
1988 1994 self.WIDTH = 750
1989 1995 self.HEIGHT = 250
1990 1996 self.WIDTHPROF = 120
1991 1997 self.HEIGHTPROF = 0
1992 1998 self.counter_imagwr = 0
1993 1999
1994 2000 self.PLOT_CODE = SPEC_CODE
1995 2001
1996 2002 self.FTP_WEI = None
1997 2003 self.EXP_CODE = None
1998 2004 self.SUB_EXP_CODE = None
1999 2005 self.PLOT_POS = None
2000 2006
2001 2007 self.__xfilter_ena = False
2002 2008 self.__yfilter_ena = False
2003 2009
2004 2010 def getSubplots(self):
2005 2011
2006 2012 ncol = 3
2007 2013 nrow = int(numpy.ceil(self.nplots/3.0))
2008 2014
2009 2015 return nrow, ncol
2010 2016
2011 2017 def setup(self, id, nplots, wintitle, show=True):
2012 2018
2013 2019 self.nplots = nplots
2014 2020
2015 2021 ncolspan = 1
2016 2022 colspan = 1
2017 2023
2018 2024 self.createFigure(id = id,
2019 2025 wintitle = wintitle,
2020 2026 widthplot = self.WIDTH + self.WIDTHPROF,
2021 2027 heightplot = self.HEIGHT + self.HEIGHTPROF,
2022 2028 show=show)
2023 2029
2024 2030 nrow, ncol = self.getSubplots()
2025 2031
2026 2032 counter = 0
2027 2033 for y in range(nrow):
2028 2034 for x in range(ncol):
2029 2035
2030 2036 if counter >= self.nplots:
2031 2037 break
2032 2038
2033 2039 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
2034 2040
2035 2041 counter += 1
2036 2042
2037 2043 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
2038 2044 xmin=None, xmax=None, ymin=None, ymax=None, SNRmin=None, SNRmax=None,
2039 2045 vmin=None, vmax=None, wmin=None, wmax=None, mode = 'SA',
2040 2046 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
2041 2047 server=None, folder=None, username=None, password=None,
2042 2048 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
2043 2049 xaxis="frequency"):
2044 2050
2045 2051 """
2046 2052
2047 2053 Input:
2048 2054 dataOut :
2049 2055 id :
2050 2056 wintitle :
2051 2057 channelList :
2052 2058 showProfile :
2053 2059 xmin : None,
2054 2060 xmax : None,
2055 2061 ymin : None,
2056 2062 ymax : None,
2057 2063 zmin : None,
2058 2064 zmax : None
2059 2065 """
2060 2066 #Rebuild matrix
2061 2067 utctime = dataOut.data_param[0,0]
2062 2068 cmet = dataOut.data_param[:,1].astype(int)
2063 2069 tmet = dataOut.data_param[:,2].astype(int)
2064 2070 hmet = dataOut.data_param[:,3].astype(int)
2065 2071
2066 2072 nParam = 3
2067 2073 nChan = len(dataOut.groupList)
2068 2074 x = dataOut.abscissaList
2069 2075 y = dataOut.heightList
2070 2076
2071 2077 z = numpy.full((nChan, nParam, y.size, x.size - 1),numpy.nan)
2072 2078 z[cmet,:,hmet,tmet] = dataOut.data_param[:,4:]
2073 2079 z[:,0,:,:] = 10*numpy.log10(z[:,0,:,:]) #logarithmic scale
2074 2080 z = numpy.reshape(z, (nChan*nParam, y.size, x.size-1))
2075 2081
2076 2082 xlabel = "Time (s)"
2077 2083 ylabel = "Range (km)"
2078 2084
2079 2085 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
2080 2086
2081 2087 if not self.isConfig:
2082 2088
2083 2089 nplots = nParam*nChan
2084 2090
2085 2091 self.setup(id=id,
2086 2092 nplots=nplots,
2087 2093 wintitle=wintitle,
2088 2094 show=show)
2089 2095
2090 2096 if xmin is None: xmin = numpy.nanmin(x)
2091 2097 if xmax is None: xmax = numpy.nanmax(x)
2092 2098 if ymin is None: ymin = numpy.nanmin(y)
2093 2099 if ymax is None: ymax = numpy.nanmax(y)
2094 2100 if SNRmin is None: SNRmin = numpy.nanmin(z[0,:])
2095 2101 if SNRmax is None: SNRmax = numpy.nanmax(z[0,:])
2096 2102 if vmax is None: vmax = numpy.nanmax(numpy.abs(z[1,:]))
2097 2103 if vmin is None: vmin = -vmax
2098 2104 if wmin is None: wmin = 0
2099 2105 if wmax is None: wmax = 50
2100 2106
2101 2107 self.nChannels = nChan
2102 2108
2103 2109 zminList = []
2104 2110 zmaxList = []
2105 2111 titleList = []
2106 2112 cmapList = []
2107 2113 for i in range(self.nChannels):
2108 2114 strAux1 = "SNR Channel "+ str(i)
2109 2115 strAux2 = "Radial Velocity Channel "+ str(i)
2110 2116 strAux3 = "Spectral Width Channel "+ str(i)
2111 2117
2112 2118 titleList = titleList + [strAux1,strAux2,strAux3]
2113 2119 cmapList = cmapList + ["jet","RdBu_r","jet"]
2114 2120 zminList = zminList + [SNRmin,vmin,wmin]
2115 2121 zmaxList = zmaxList + [SNRmax,vmax,wmax]
2116 2122
2117 2123 self.zminList = zminList
2118 2124 self.zmaxList = zmaxList
2119 2125 self.cmapList = cmapList
2120 2126 self.titleList = titleList
2121 2127
2122 2128 self.FTP_WEI = ftp_wei
2123 2129 self.EXP_CODE = exp_code
2124 2130 self.SUB_EXP_CODE = sub_exp_code
2125 2131 self.PLOT_POS = plot_pos
2126 2132
2127 2133 self.isConfig = True
2128 2134
2129 2135 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
2130 2136
2131 2137 for i in range(self.nplots):
2132 2138 title = self.titleList[i] + ": " +str_datetime
2133 2139 axes = self.axesList[i]
2134 2140 axes.pcolor(x, y, z[i,:].T,
2135 2141 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=self.zminList[i], zmax=self.zmaxList[i],
2136 2142 xlabel=xlabel, ylabel=ylabel, title=title, colormap=self.cmapList[i],ticksize=9, cblabel='')
2137 2143 self.draw()
2138 2144
2139 2145 if figfile == None:
2140 2146 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
2141 2147 name = str_datetime
2142 2148 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
2143 2149 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
2144 2150 figfile = self.getFilename(name)
2145 2151
2146 2152 self.save(figpath=figpath,
2147 2153 figfile=figfile,
2148 2154 save=save,
2149 2155 ftp=ftp,
2150 2156 wr_period=wr_period,
2151 thisDatetime=thisDatetime) No newline at end of file
2157 thisDatetime=thisDatetime)
2158 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1587 +1,1587
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9
10 10 from .figure import Figure, isRealtime, isTimeInHourRange
11 11 from .plotting_codes import *
12 12 from schainpy.model.proc.jroproc_base import MPDecorator
13 13
14 14 from schainpy.utils import log
15 15
16 16 @MPDecorator
17 17 class SpectraPlot(Figure):
18 18
19 19 isConfig = None
20 20 __nsubplots = None
21 21
22 22 WIDTHPROF = None
23 23 HEIGHTPROF = None
24 24 PREFIX = 'spc'
25 25
26 def __init__(self):#, **kwargs):
27 Figure.__init__(self)#, **kwargs)
26 def __init__(self):
27 Figure.__init__(self)
28 28 self.isConfig = False
29 29 self.__nsubplots = 1
30 30 self.WIDTH = 250
31 31 self.HEIGHT = 250
32 32 self.WIDTHPROF = 120
33 33 self.HEIGHTPROF = 0
34 34 self.counter_imagwr = 0
35 35
36 36 self.PLOT_CODE = SPEC_CODE
37 37
38 38 self.FTP_WEI = None
39 39 self.EXP_CODE = None
40 40 self.SUB_EXP_CODE = None
41 41 self.PLOT_POS = None
42 42
43 43 self.__xfilter_ena = False
44 44 self.__yfilter_ena = False
45 45
46 46 def getSubplots(self):
47 47
48 48 ncol = int(numpy.sqrt(self.nplots)+0.9)
49 49 nrow = int(self.nplots*1./ncol + 0.9)
50 50
51 51 return nrow, ncol
52 52
53 53 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
54 54
55 55 self.__showprofile = showprofile
56 56 self.nplots = nplots
57 57
58 58 ncolspan = 1
59 59 colspan = 1
60 60 if showprofile:
61 61 ncolspan = 3
62 62 colspan = 2
63 63 self.__nsubplots = 2
64 64
65 65 self.createFigure(id = id,
66 66 wintitle = wintitle,
67 67 widthplot = self.WIDTH + self.WIDTHPROF,
68 68 heightplot = self.HEIGHT + self.HEIGHTPROF,
69 69 show=show)
70 70
71 71 nrow, ncol = self.getSubplots()
72 72
73 73 counter = 0
74 74 for y in range(nrow):
75 75 for x in range(ncol):
76 76
77 77 if counter >= self.nplots:
78 78 break
79 79
80 80 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
81 81
82 82 if showprofile:
83 83 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
84 84
85 85 counter += 1
86 86
87 87 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
88 88 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
89 89 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
90 90 server=None, folder=None, username=None, password=None,
91 91 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
92 92 xaxis="frequency", colormap='jet', normFactor=None):
93 93
94 94 """
95 95
96 96 Input:
97 97 dataOut :
98 98 id :
99 99 wintitle :
100 100 channelList :
101 101 showProfile :
102 102 xmin : None,
103 103 xmax : None,
104 104 ymin : None,
105 105 ymax : None,
106 106 zmin : None,
107 107 zmax : None
108 108 """
109 109 if dataOut.flagNoData:
110 110 return dataOut
111 111
112 112 if realtime:
113 113 if not(isRealtime(utcdatatime = dataOut.utctime)):
114 114 print('Skipping this plot function')
115 115 return
116 116
117 117 if channelList == None:
118 118 channelIndexList = dataOut.channelIndexList
119 119 else:
120 120 channelIndexList = []
121 121 for channel in channelList:
122 122 if channel not in dataOut.channelList:
123 123 raise ValueError("Channel %d is not in dataOut.channelList" %channel)
124 124 channelIndexList.append(dataOut.channelList.index(channel))
125 125
126 126 if normFactor is None:
127 127 factor = dataOut.normFactor
128 128 else:
129 129 factor = normFactor
130 130 if xaxis == "frequency":
131 131 x = dataOut.getFreqRange(1)/1000.
132 132 xlabel = "Frequency (kHz)"
133 133
134 134 elif xaxis == "time":
135 135 x = dataOut.getAcfRange(1)
136 136 xlabel = "Time (ms)"
137 137
138 138 else:
139 139 x = dataOut.getVelRange(1)
140 140 xlabel = "Velocity (m/s)"
141 141
142 142 ylabel = "Range (Km)"
143 143
144 144 y = dataOut.getHeiRange()
145 145
146 146 z = dataOut.data_spc/factor
147 147 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
148 148 zdB = 10*numpy.log10(z)
149 149
150 150 avg = numpy.average(z, axis=1)
151 151 avgdB = 10*numpy.log10(avg)
152 152
153 153 noise = dataOut.getNoise()/factor
154 154 noisedB = 10*numpy.log10(noise)
155 155
156 156 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
157 157 title = wintitle + " Spectra"
158 158 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
159 159 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
160 160
161 161 if not self.isConfig:
162 162
163 163 nplots = len(channelIndexList)
164 164
165 165 self.setup(id=id,
166 166 nplots=nplots,
167 167 wintitle=wintitle,
168 168 showprofile=showprofile,
169 169 show=show)
170 170
171 171 if xmin == None: xmin = numpy.nanmin(x)
172 172 if xmax == None: xmax = numpy.nanmax(x)
173 173 if ymin == None: ymin = numpy.nanmin(y)
174 174 if ymax == None: ymax = numpy.nanmax(y)
175 175 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
176 176 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
177 177
178 178 self.FTP_WEI = ftp_wei
179 179 self.EXP_CODE = exp_code
180 180 self.SUB_EXP_CODE = sub_exp_code
181 181 self.PLOT_POS = plot_pos
182 182
183 183 self.isConfig = True
184 184
185 185 self.setWinTitle(title)
186 186
187 187 for i in range(self.nplots):
188 188 index = channelIndexList[i]
189 189 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
190 190 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
191 191 if len(dataOut.beam.codeList) != 0:
192 192 title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
193 193
194 194 axes = self.axesList[i*self.__nsubplots]
195 195 axes.pcolor(x, y, zdB[index,:,:],
196 196 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
197 197 xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
198 198 ticksize=9, cblabel='')
199 199
200 200 if self.__showprofile:
201 201 axes = self.axesList[i*self.__nsubplots +1]
202 202 axes.pline(avgdB[index,:], y,
203 203 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
204 204 xlabel='dB', ylabel='', title='',
205 205 ytick_visible=False,
206 206 grid='x')
207 207
208 208 noiseline = numpy.repeat(noisedB[index], len(y))
209 209 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
210 210
211 211 self.draw()
212 212
213 213 if figfile == None:
214 214 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
215 215 name = str_datetime
216 216 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
217 217 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
218 218 figfile = self.getFilename(name)
219 219
220 220 self.save(figpath=figpath,
221 221 figfile=figfile,
222 222 save=save,
223 223 ftp=ftp,
224 224 wr_period=wr_period,
225 225 thisDatetime=thisDatetime)
226 226
227 227 return dataOut
228 228 @MPDecorator
229 229 class CrossSpectraPlot(Figure):
230 230
231 231 isConfig = None
232 232 __nsubplots = None
233 233
234 234 WIDTH = None
235 235 HEIGHT = None
236 236 WIDTHPROF = None
237 237 HEIGHTPROF = None
238 238 PREFIX = 'cspc'
239 239
240 def __init__(self):#, **kwargs):
241 Figure.__init__(self)#, **kwargs)
240 def __init__(self):
241 Figure.__init__(self)
242 242 self.isConfig = False
243 243 self.__nsubplots = 4
244 244 self.counter_imagwr = 0
245 245 self.WIDTH = 250
246 246 self.HEIGHT = 250
247 247 self.WIDTHPROF = 0
248 248 self.HEIGHTPROF = 0
249 249
250 250 self.PLOT_CODE = CROSS_CODE
251 251 self.FTP_WEI = None
252 252 self.EXP_CODE = None
253 253 self.SUB_EXP_CODE = None
254 254 self.PLOT_POS = None
255 255
256 256 def getSubplots(self):
257 257
258 258 ncol = 4
259 259 nrow = self.nplots
260 260
261 261 return nrow, ncol
262 262
263 263 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
264 264
265 265 self.__showprofile = showprofile
266 266 self.nplots = nplots
267 267
268 268 ncolspan = 1
269 269 colspan = 1
270 270
271 271 self.createFigure(id = id,
272 272 wintitle = wintitle,
273 273 widthplot = self.WIDTH + self.WIDTHPROF,
274 274 heightplot = self.HEIGHT + self.HEIGHTPROF,
275 275 show=True)
276 276
277 277 nrow, ncol = self.getSubplots()
278 278
279 279 counter = 0
280 280 for y in range(nrow):
281 281 for x in range(ncol):
282 282 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
283 283
284 284 counter += 1
285 285
286 286 def run(self, dataOut, id, wintitle="", pairsList=None,
287 287 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
288 288 coh_min=None, coh_max=None, phase_min=None, phase_max=None,
289 289 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
290 290 power_cmap='jet', coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
291 291 server=None, folder=None, username=None, password=None,
292 292 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None,
293 293 xaxis='frequency'):
294 294
295 295 """
296 296
297 297 Input:
298 298 dataOut :
299 299 id :
300 300 wintitle :
301 301 channelList :
302 302 showProfile :
303 303 xmin : None,
304 304 xmax : None,
305 305 ymin : None,
306 306 ymax : None,
307 307 zmin : None,
308 308 zmax : None
309 309 """
310 310
311 311 if dataOut.flagNoData:
312 312 return dataOut
313 313
314 314 if pairsList == None:
315 315 pairsIndexList = dataOut.pairsIndexList
316 316 else:
317 317 pairsIndexList = []
318 318 for pair in pairsList:
319 319 if pair not in dataOut.pairsList:
320 320 raise ValueError("Pair %s is not in dataOut.pairsList" %str(pair))
321 321 pairsIndexList.append(dataOut.pairsList.index(pair))
322 322
323 323 if not pairsIndexList:
324 324 return
325 325
326 326 if len(pairsIndexList) > 4:
327 327 pairsIndexList = pairsIndexList[0:4]
328 328
329 329 if normFactor is None:
330 330 factor = dataOut.normFactor
331 331 else:
332 332 factor = normFactor
333 333 x = dataOut.getVelRange(1)
334 334 y = dataOut.getHeiRange()
335 335 z = dataOut.data_spc[:,:,:]/factor
336 336 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
337 337
338 338 noise = dataOut.noise/factor
339 339
340 340 zdB = 10*numpy.log10(z)
341 341 noisedB = 10*numpy.log10(noise)
342 342
343 343 if coh_min == None:
344 344 coh_min = 0.0
345 345 if coh_max == None:
346 346 coh_max = 1.0
347 347
348 348 if phase_min == None:
349 349 phase_min = -180
350 350 if phase_max == None:
351 351 phase_max = 180
352 352
353 353 #thisDatetime = dataOut.datatime
354 354 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
355 355 title = wintitle + " Cross-Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
356 356 # xlabel = "Velocity (m/s)"
357 357 ylabel = "Range (Km)"
358 358
359 359 if xaxis == "frequency":
360 360 x = dataOut.getFreqRange(1)/1000.
361 361 xlabel = "Frequency (kHz)"
362 362
363 363 elif xaxis == "time":
364 364 x = dataOut.getAcfRange(1)
365 365 xlabel = "Time (ms)"
366 366
367 367 else:
368 368 x = dataOut.getVelRange(1)
369 369 xlabel = "Velocity (m/s)"
370 370
371 371 if not self.isConfig:
372 372
373 373 nplots = len(pairsIndexList)
374 374
375 375 self.setup(id=id,
376 376 nplots=nplots,
377 377 wintitle=wintitle,
378 378 showprofile=False,
379 379 show=show)
380 380
381 381 avg = numpy.abs(numpy.average(z, axis=1))
382 382 avgdB = 10*numpy.log10(avg)
383 383
384 384 if xmin == None: xmin = numpy.nanmin(x)
385 385 if xmax == None: xmax = numpy.nanmax(x)
386 386 if ymin == None: ymin = numpy.nanmin(y)
387 387 if ymax == None: ymax = numpy.nanmax(y)
388 388 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
389 389 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
390 390
391 391 self.FTP_WEI = ftp_wei
392 392 self.EXP_CODE = exp_code
393 393 self.SUB_EXP_CODE = sub_exp_code
394 394 self.PLOT_POS = plot_pos
395 395
396 396 self.isConfig = True
397 397
398 398 self.setWinTitle(title)
399 399
400 400 for i in range(self.nplots):
401 401 pair = dataOut.pairsList[pairsIndexList[i]]
402 402
403 403 chan_index0 = dataOut.channelList.index(pair[0])
404 404 chan_index1 = dataOut.channelList.index(pair[1])
405 405
406 406 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
407 407 title = "Ch%d: %4.2fdB: %s" %(pair[0], noisedB[chan_index0], str_datetime)
408 408 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index0,:,:]/factor)
409 409 axes0 = self.axesList[i*self.__nsubplots]
410 410 axes0.pcolor(x, y, zdB,
411 411 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
412 412 xlabel=xlabel, ylabel=ylabel, title=title,
413 413 ticksize=9, colormap=power_cmap, cblabel='')
414 414
415 415 title = "Ch%d: %4.2fdB: %s" %(pair[1], noisedB[chan_index1], str_datetime)
416 416 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index1,:,:]/factor)
417 417 axes0 = self.axesList[i*self.__nsubplots+1]
418 418 axes0.pcolor(x, y, zdB,
419 419 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
420 420 xlabel=xlabel, ylabel=ylabel, title=title,
421 421 ticksize=9, colormap=power_cmap, cblabel='')
422 422
423 423 coherenceComplex = dataOut.data_cspc[pairsIndexList[i],:,:]/numpy.sqrt(dataOut.data_spc[chan_index0,:,:]*dataOut.data_spc[chan_index1,:,:])
424 424 coherence = numpy.abs(coherenceComplex)
425 425 # phase = numpy.arctan(-1*coherenceComplex.imag/coherenceComplex.real)*180/numpy.pi
426 426 phase = numpy.arctan2(coherenceComplex.imag, coherenceComplex.real)*180/numpy.pi
427 427
428 428 title = "Coherence Ch%d * Ch%d" %(pair[0], pair[1])
429 429 axes0 = self.axesList[i*self.__nsubplots+2]
430 430 axes0.pcolor(x, y, coherence,
431 431 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=coh_min, zmax=coh_max,
432 432 xlabel=xlabel, ylabel=ylabel, title=title,
433 433 ticksize=9, colormap=coherence_cmap, cblabel='')
434 434
435 435 title = "Phase Ch%d * Ch%d" %(pair[0], pair[1])
436 436 axes0 = self.axesList[i*self.__nsubplots+3]
437 437 axes0.pcolor(x, y, phase,
438 438 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
439 439 xlabel=xlabel, ylabel=ylabel, title=title,
440 440 ticksize=9, colormap=phase_cmap, cblabel='')
441 441
442 442
443 443
444 444 self.draw()
445 445
446 446 self.save(figpath=figpath,
447 447 figfile=figfile,
448 448 save=save,
449 449 ftp=ftp,
450 450 wr_period=wr_period,
451 451 thisDatetime=thisDatetime)
452 452
453 453 return dataOut
454 454
455 455 @MPDecorator
456 456 class RTIPlot(Figure):
457 457
458 458 __isConfig = None
459 459 __nsubplots = None
460 460
461 461 WIDTHPROF = None
462 462 HEIGHTPROF = None
463 463 PREFIX = 'rti'
464 464
465 def __init__(self):#, **kwargs):
465 def __init__(self):
466 466
467 Figure.__init__(self)#, **kwargs)
467 Figure.__init__(self)
468 468 self.timerange = None
469 469 self.isConfig = False
470 470 self.__nsubplots = 1
471 471
472 472 self.WIDTH = 800
473 473 self.HEIGHT = 180
474 474 self.WIDTHPROF = 120
475 475 self.HEIGHTPROF = 0
476 476 self.counter_imagwr = 0
477 477
478 478 self.PLOT_CODE = RTI_CODE
479 479
480 480 self.FTP_WEI = None
481 481 self.EXP_CODE = None
482 482 self.SUB_EXP_CODE = None
483 483 self.PLOT_POS = None
484 484 self.tmin = None
485 485 self.tmax = None
486 486
487 487 self.xmin = None
488 488 self.xmax = None
489 489
490 490 self.figfile = None
491 491
492 492 def getSubplots(self):
493 493
494 494 ncol = 1
495 495 nrow = self.nplots
496 496
497 497 return nrow, ncol
498 498
499 499 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
500 500
501 501 self.__showprofile = showprofile
502 502 self.nplots = nplots
503 503
504 504 ncolspan = 1
505 505 colspan = 1
506 506 if showprofile:
507 507 ncolspan = 7
508 508 colspan = 6
509 509 self.__nsubplots = 2
510 510
511 511 self.createFigure(id = id,
512 512 wintitle = wintitle,
513 513 widthplot = self.WIDTH + self.WIDTHPROF,
514 514 heightplot = self.HEIGHT + self.HEIGHTPROF,
515 515 show=show)
516 516
517 517 nrow, ncol = self.getSubplots()
518 518
519 519 counter = 0
520 520 for y in range(nrow):
521 521 for x in range(ncol):
522 522
523 523 if counter >= self.nplots:
524 524 break
525 525
526 526 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
527 527
528 528 if showprofile:
529 529 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
530 530
531 531 counter += 1
532 532
533 533 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
534 534 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
535 535 timerange=None, colormap='jet',
536 536 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
537 537 server=None, folder=None, username=None, password=None,
538 538 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None, HEIGHT=None):
539 539
540 540 """
541 541
542 542 Input:
543 543 dataOut :
544 544 id :
545 545 wintitle :
546 546 channelList :
547 547 showProfile :
548 548 xmin : None,
549 549 xmax : None,
550 550 ymin : None,
551 551 ymax : None,
552 552 zmin : None,
553 553 zmax : None
554 554 """
555 555 if dataOut.flagNoData:
556 556 return dataOut
557 557
558 558 #colormap = kwargs.get('colormap', 'jet')
559 559 if HEIGHT is not None:
560 560 self.HEIGHT = HEIGHT
561 561
562 562 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
563 563 return
564 564
565 565 if channelList == None:
566 566 channelIndexList = dataOut.channelIndexList
567 567 else:
568 568 channelIndexList = []
569 569 for channel in channelList:
570 570 if channel not in dataOut.channelList:
571 571 raise ValueError("Channel %d is not in dataOut.channelList")
572 572 channelIndexList.append(dataOut.channelList.index(channel))
573 573
574 574 if normFactor is None:
575 575 factor = dataOut.normFactor
576 576 else:
577 577 factor = normFactor
578 578
579 # factor = dataOut.normFactor
579 #factor = dataOut.normFactor
580 580 x = dataOut.getTimeRange()
581 581 y = dataOut.getHeiRange()
582 582
583 583 z = dataOut.data_spc/factor
584 584 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
585 585 avg = numpy.average(z, axis=1)
586 586 avgdB = 10.*numpy.log10(avg)
587 587 # avgdB = dataOut.getPower()
588 588
589 589
590 590 thisDatetime = dataOut.datatime
591 # thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
591 #thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
592 592 title = wintitle + " RTI" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
593 593 xlabel = ""
594 594 ylabel = "Range (Km)"
595 595
596 596 update_figfile = False
597 597
598 598 if self.xmax is not None and dataOut.ltctime >= self.xmax: #yong
599 599 self.counter_imagwr = wr_period
600 600 self.isConfig = False
601 601 update_figfile = True
602 602
603 603 if not self.isConfig:
604 604
605 605 nplots = len(channelIndexList)
606 606
607 607 self.setup(id=id,
608 608 nplots=nplots,
609 609 wintitle=wintitle,
610 610 showprofile=showprofile,
611 611 show=show)
612 612
613 613 if timerange != None:
614 614 self.timerange = timerange
615 615
616 616 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
617 617
618 618 noise = dataOut.noise/factor
619 619 noisedB = 10*numpy.log10(noise)
620 620
621 621 if ymin == None: ymin = numpy.nanmin(y)
622 622 if ymax == None: ymax = numpy.nanmax(y)
623 623 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
624 624 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
625 625
626 626 self.FTP_WEI = ftp_wei
627 627 self.EXP_CODE = exp_code
628 628 self.SUB_EXP_CODE = sub_exp_code
629 629 self.PLOT_POS = plot_pos
630 630
631 631 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
632 632 self.isConfig = True
633 633 self.figfile = figfile
634 634 update_figfile = True
635 635
636 636 self.setWinTitle(title)
637 637
638 638 for i in range(self.nplots):
639 639 index = channelIndexList[i]
640 640 title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
641 641 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
642 642 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
643 643 axes = self.axesList[i*self.__nsubplots]
644 644 zdB = avgdB[index].reshape((1,-1))
645 645 axes.pcolorbuffer(x, y, zdB,
646 646 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
647 647 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
648 648 ticksize=9, cblabel='', cbsize="1%", colormap=colormap)
649 649
650 650 if self.__showprofile:
651 651 axes = self.axesList[i*self.__nsubplots +1]
652 652 axes.pline(avgdB[index], y,
653 653 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
654 654 xlabel='dB', ylabel='', title='',
655 655 ytick_visible=False,
656 656 grid='x')
657 657
658 658 self.draw()
659 659
660 660 self.save(figpath=figpath,
661 661 figfile=figfile,
662 662 save=save,
663 663 ftp=ftp,
664 664 wr_period=wr_period,
665 665 thisDatetime=thisDatetime,
666 666 update_figfile=update_figfile)
667 667 return dataOut
668 668
669 669 @MPDecorator
670 670 class CoherenceMap(Figure):
671 671 isConfig = None
672 672 __nsubplots = None
673 673
674 674 WIDTHPROF = None
675 675 HEIGHTPROF = None
676 676 PREFIX = 'cmap'
677 677
678 def __init__(self):#, **kwargs):
679 Figure.__init__(self)#, **kwargs)
678 def __init__(self):
679 Figure.__init__(self)
680 680 self.timerange = 2*60*60
681 681 self.isConfig = False
682 682 self.__nsubplots = 1
683 683
684 684 self.WIDTH = 800
685 685 self.HEIGHT = 180
686 686 self.WIDTHPROF = 120
687 687 self.HEIGHTPROF = 0
688 688 self.counter_imagwr = 0
689 689
690 690 self.PLOT_CODE = COH_CODE
691 691
692 692 self.FTP_WEI = None
693 693 self.EXP_CODE = None
694 694 self.SUB_EXP_CODE = None
695 695 self.PLOT_POS = None
696 696 self.counter_imagwr = 0
697 697
698 698 self.xmin = None
699 699 self.xmax = None
700 700
701 701 def getSubplots(self):
702 702 ncol = 1
703 703 nrow = self.nplots*2
704 704
705 705 return nrow, ncol
706 706
707 707 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
708 708 self.__showprofile = showprofile
709 709 self.nplots = nplots
710 710
711 711 ncolspan = 1
712 712 colspan = 1
713 713 if showprofile:
714 714 ncolspan = 7
715 715 colspan = 6
716 716 self.__nsubplots = 2
717 717
718 718 self.createFigure(id = id,
719 719 wintitle = wintitle,
720 720 widthplot = self.WIDTH + self.WIDTHPROF,
721 721 heightplot = self.HEIGHT + self.HEIGHTPROF,
722 722 show=True)
723 723
724 724 nrow, ncol = self.getSubplots()
725 725
726 726 for y in range(nrow):
727 727 for x in range(ncol):
728 728
729 729 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
730 730
731 731 if showprofile:
732 732 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
733 733
734 734 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
735 735 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
736 736 timerange=None, phase_min=None, phase_max=None,
737 737 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
738 738 coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
739 739 server=None, folder=None, username=None, password=None,
740 740 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
741 741
742 742
743 743 if dataOut.flagNoData:
744 744 return dataOut
745 745
746 746 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
747 747 return
748 748
749 749 if pairsList == None:
750 750 pairsIndexList = dataOut.pairsIndexList
751 751 else:
752 752 pairsIndexList = []
753 753 for pair in pairsList:
754 754 if pair not in dataOut.pairsList:
755 755 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
756 756 pairsIndexList.append(dataOut.pairsList.index(pair))
757 757
758 758 if pairsIndexList == []:
759 759 return
760 760
761 761 if len(pairsIndexList) > 4:
762 762 pairsIndexList = pairsIndexList[0:4]
763 763
764 764 if phase_min == None:
765 765 phase_min = -180
766 766 if phase_max == None:
767 767 phase_max = 180
768 768
769 769 x = dataOut.getTimeRange()
770 770 y = dataOut.getHeiRange()
771 771
772 772 thisDatetime = dataOut.datatime
773 773
774 774 title = wintitle + " CoherenceMap" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
775 775 xlabel = ""
776 776 ylabel = "Range (Km)"
777 777 update_figfile = False
778 778
779 779 if not self.isConfig:
780 780 nplots = len(pairsIndexList)
781 781 self.setup(id=id,
782 782 nplots=nplots,
783 783 wintitle=wintitle,
784 784 showprofile=showprofile,
785 785 show=show)
786 786
787 787 if timerange != None:
788 788 self.timerange = timerange
789 789
790 790 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
791 791
792 792 if ymin == None: ymin = numpy.nanmin(y)
793 793 if ymax == None: ymax = numpy.nanmax(y)
794 794 if zmin == None: zmin = 0.
795 795 if zmax == None: zmax = 1.
796 796
797 797 self.FTP_WEI = ftp_wei
798 798 self.EXP_CODE = exp_code
799 799 self.SUB_EXP_CODE = sub_exp_code
800 800 self.PLOT_POS = plot_pos
801 801
802 802 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
803 803
804 804 self.isConfig = True
805 805 update_figfile = True
806 806
807 807 self.setWinTitle(title)
808 808
809 809 for i in range(self.nplots):
810 810
811 811 pair = dataOut.pairsList[pairsIndexList[i]]
812 812
813 813 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i],:,:],axis=0)
814 814 powa = numpy.average(dataOut.data_spc[pair[0],:,:],axis=0)
815 815 powb = numpy.average(dataOut.data_spc[pair[1],:,:],axis=0)
816 816
817 817
818 818 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
819 819 coherence = numpy.abs(avgcoherenceComplex)
820 820
821 821 z = coherence.reshape((1,-1))
822 822
823 823 counter = 0
824 824
825 825 title = "Coherence Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
826 826 axes = self.axesList[i*self.__nsubplots*2]
827 827 axes.pcolorbuffer(x, y, z,
828 828 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
829 829 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
830 830 ticksize=9, cblabel='', colormap=coherence_cmap, cbsize="1%")
831 831
832 832 if self.__showprofile:
833 833 counter += 1
834 834 axes = self.axesList[i*self.__nsubplots*2 + counter]
835 835 axes.pline(coherence, y,
836 836 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
837 837 xlabel='', ylabel='', title='', ticksize=7,
838 838 ytick_visible=False, nxticks=5,
839 839 grid='x')
840 840
841 841 counter += 1
842 842
843 843 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
844 844
845 845 z = phase.reshape((1,-1))
846 846
847 847 title = "Phase Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
848 848 axes = self.axesList[i*self.__nsubplots*2 + counter]
849 849 axes.pcolorbuffer(x, y, z,
850 850 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
851 851 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
852 852 ticksize=9, cblabel='', colormap=phase_cmap, cbsize="1%")
853 853
854 854 if self.__showprofile:
855 855 counter += 1
856 856 axes = self.axesList[i*self.__nsubplots*2 + counter]
857 857 axes.pline(phase, y,
858 858 xmin=phase_min, xmax=phase_max, ymin=ymin, ymax=ymax,
859 859 xlabel='', ylabel='', title='', ticksize=7,
860 860 ytick_visible=False, nxticks=4,
861 861 grid='x')
862 862
863 863 self.draw()
864 864
865 865 if dataOut.ltctime >= self.xmax:
866 866 self.counter_imagwr = wr_period
867 867 self.isConfig = False
868 868 update_figfile = True
869 869
870 870 self.save(figpath=figpath,
871 871 figfile=figfile,
872 872 save=save,
873 873 ftp=ftp,
874 874 wr_period=wr_period,
875 875 thisDatetime=thisDatetime,
876 876 update_figfile=update_figfile)
877 877
878 878 return dataOut
879 879
880 880 @MPDecorator
881 881 class PowerProfilePlot(Figure):
882 882
883 883 isConfig = None
884 884 __nsubplots = None
885 885
886 886 WIDTHPROF = None
887 887 HEIGHTPROF = None
888 888 PREFIX = 'spcprofile'
889 889
890 def __init__(self):#, **kwargs):
891 Figure.__init__(self)#, **kwargs)
890 def __init__(self):
891 Figure.__init__(self)
892 892 self.isConfig = False
893 893 self.__nsubplots = 1
894 894
895 895 self.PLOT_CODE = POWER_CODE
896 896
897 897 self.WIDTH = 300
898 898 self.HEIGHT = 500
899 899 self.counter_imagwr = 0
900 900
901 901 def getSubplots(self):
902 902 ncol = 1
903 903 nrow = 1
904 904
905 905 return nrow, ncol
906 906
907 907 def setup(self, id, nplots, wintitle, show):
908 908
909 909 self.nplots = nplots
910 910
911 911 ncolspan = 1
912 912 colspan = 1
913 913
914 914 self.createFigure(id = id,
915 915 wintitle = wintitle,
916 916 widthplot = self.WIDTH,
917 917 heightplot = self.HEIGHT,
918 918 show=show)
919 919
920 920 nrow, ncol = self.getSubplots()
921 921
922 922 counter = 0
923 923 for y in range(nrow):
924 924 for x in range(ncol):
925 925 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
926 926
927 927 def run(self, dataOut, id, wintitle="", channelList=None,
928 928 xmin=None, xmax=None, ymin=None, ymax=None,
929 929 save=False, figpath='./', figfile=None, show=True,
930 930 ftp=False, wr_period=1, server=None,
931 931 folder=None, username=None, password=None):
932 932
933 933 if dataOut.flagNoData:
934 934 return dataOut
935 935
936 936
937 937 if channelList == None:
938 938 channelIndexList = dataOut.channelIndexList
939 939 channelList = dataOut.channelList
940 940 else:
941 941 channelIndexList = []
942 942 for channel in channelList:
943 943 if channel not in dataOut.channelList:
944 944 raise ValueError("Channel %d is not in dataOut.channelList")
945 945 channelIndexList.append(dataOut.channelList.index(channel))
946 946
947 947 factor = dataOut.normFactor
948 948
949 949 y = dataOut.getHeiRange()
950 950
951 951 #for voltage
952 952 if dataOut.type == 'Voltage':
953 953 x = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
954 954 x = x.real
955 955 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
956 956
957 957 #for spectra
958 958 if dataOut.type == 'Spectra':
959 959 x = dataOut.data_spc[channelIndexList,:,:]/factor
960 960 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
961 961 x = numpy.average(x, axis=1)
962 962
963 963
964 964 xdB = 10*numpy.log10(x)
965 965
966 966 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
967 967 title = wintitle + " Power Profile %s" %(thisDatetime.strftime("%d-%b-%Y"))
968 968 xlabel = "dB"
969 969 ylabel = "Range (Km)"
970 970
971 971 if not self.isConfig:
972 972
973 973 nplots = 1
974 974
975 975 self.setup(id=id,
976 976 nplots=nplots,
977 977 wintitle=wintitle,
978 978 show=show)
979 979
980 980 if ymin == None: ymin = numpy.nanmin(y)
981 981 if ymax == None: ymax = numpy.nanmax(y)
982 982 if xmin == None: xmin = numpy.nanmin(xdB)*0.9
983 983 if xmax == None: xmax = numpy.nanmax(xdB)*1.1
984 984
985 985 self.isConfig = True
986 986
987 987 self.setWinTitle(title)
988 988
989 989 title = "Power Profile: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
990 990 axes = self.axesList[0]
991 991
992 992 legendlabels = ["channel %d"%x for x in channelList]
993 993 axes.pmultiline(xdB, y,
994 994 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
995 995 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
996 996 ytick_visible=True, nxticks=5,
997 997 grid='x')
998 998
999 999 self.draw()
1000 1000
1001 1001 self.save(figpath=figpath,
1002 1002 figfile=figfile,
1003 1003 save=save,
1004 1004 ftp=ftp,
1005 1005 wr_period=wr_period,
1006 1006 thisDatetime=thisDatetime)
1007 1007
1008 1008 return dataOut
1009 1009
1010 1010 @MPDecorator
1011 1011 class SpectraCutPlot(Figure):
1012 1012
1013 1013 isConfig = None
1014 1014 __nsubplots = None
1015 1015
1016 1016 WIDTHPROF = None
1017 1017 HEIGHTPROF = None
1018 1018 PREFIX = 'spc_cut'
1019 1019
1020 def __init__(self):#, **kwargs):
1021 Figure.__init__(self)#, **kwargs)
1020 def __init__(self):
1021 Figure.__init__(self)
1022 1022 self.isConfig = False
1023 1023 self.__nsubplots = 1
1024 1024
1025 1025 self.PLOT_CODE = POWER_CODE
1026 1026
1027 1027 self.WIDTH = 700
1028 1028 self.HEIGHT = 500
1029 1029 self.counter_imagwr = 0
1030 1030
1031 1031 def getSubplots(self):
1032 1032 ncol = 1
1033 1033 nrow = 1
1034 1034
1035 1035 return nrow, ncol
1036 1036
1037 1037 def setup(self, id, nplots, wintitle, show):
1038 1038
1039 1039 self.nplots = nplots
1040 1040
1041 1041 ncolspan = 1
1042 1042 colspan = 1
1043 1043
1044 1044 self.createFigure(id = id,
1045 1045 wintitle = wintitle,
1046 1046 widthplot = self.WIDTH,
1047 1047 heightplot = self.HEIGHT,
1048 1048 show=show)
1049 1049
1050 1050 nrow, ncol = self.getSubplots()
1051 1051
1052 1052 counter = 0
1053 1053 for y in range(nrow):
1054 1054 for x in range(ncol):
1055 1055 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1056 1056
1057 1057 def run(self, dataOut, id, wintitle="", channelList=None,
1058 1058 xmin=None, xmax=None, ymin=None, ymax=None,
1059 1059 save=False, figpath='./', figfile=None, show=True,
1060 1060 ftp=False, wr_period=1, server=None,
1061 1061 folder=None, username=None, password=None,
1062 1062 xaxis="frequency"):
1063 1063
1064 1064 if dataOut.flagNoData:
1065 1065 return dataOut
1066 1066
1067 1067 if channelList == None:
1068 1068 channelIndexList = dataOut.channelIndexList
1069 1069 channelList = dataOut.channelList
1070 1070 else:
1071 1071 channelIndexList = []
1072 1072 for channel in channelList:
1073 1073 if channel not in dataOut.channelList:
1074 1074 raise ValueError("Channel %d is not in dataOut.channelList")
1075 1075 channelIndexList.append(dataOut.channelList.index(channel))
1076 1076
1077 1077 factor = dataOut.normFactor
1078 1078
1079 1079 y = dataOut.getHeiRange()
1080 1080
1081 1081 z = dataOut.data_spc/factor
1082 1082 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1083 1083
1084 1084 hei_index = numpy.arange(25)*3 + 20
1085 1085
1086 1086 if xaxis == "frequency":
1087 1087 x = dataOut.getFreqRange()/1000.
1088 1088 zdB = 10*numpy.log10(z[0,:,hei_index])
1089 1089 xlabel = "Frequency (kHz)"
1090 1090 ylabel = "Power (dB)"
1091 1091
1092 1092 elif xaxis == "time":
1093 1093 x = dataOut.getAcfRange()
1094 1094 zdB = z[0,:,hei_index]
1095 1095 xlabel = "Time (ms)"
1096 1096 ylabel = "ACF"
1097 1097
1098 1098 else:
1099 1099 x = dataOut.getVelRange()
1100 1100 zdB = 10*numpy.log10(z[0,:,hei_index])
1101 1101 xlabel = "Velocity (m/s)"
1102 1102 ylabel = "Power (dB)"
1103 1103
1104 1104 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1105 1105 title = wintitle + " Range Cuts %s" %(thisDatetime.strftime("%d-%b-%Y"))
1106 1106
1107 1107 if not self.isConfig:
1108 1108
1109 1109 nplots = 1
1110 1110
1111 1111 self.setup(id=id,
1112 1112 nplots=nplots,
1113 1113 wintitle=wintitle,
1114 1114 show=show)
1115 1115
1116 1116 if xmin == None: xmin = numpy.nanmin(x)*0.9
1117 1117 if xmax == None: xmax = numpy.nanmax(x)*1.1
1118 1118 if ymin == None: ymin = numpy.nanmin(zdB)
1119 1119 if ymax == None: ymax = numpy.nanmax(zdB)
1120 1120
1121 1121 self.isConfig = True
1122 1122
1123 1123 self.setWinTitle(title)
1124 1124
1125 1125 title = "Spectra Cuts: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1126 1126 axes = self.axesList[0]
1127 1127
1128 1128 legendlabels = ["Range = %dKm" %y[i] for i in hei_index]
1129 1129
1130 1130 axes.pmultilineyaxis( x, zdB,
1131 1131 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1132 1132 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
1133 1133 ytick_visible=True, nxticks=5,
1134 1134 grid='x')
1135 1135
1136 1136 self.draw()
1137 1137
1138 1138 self.save(figpath=figpath,
1139 1139 figfile=figfile,
1140 1140 save=save,
1141 1141 ftp=ftp,
1142 1142 wr_period=wr_period,
1143 1143 thisDatetime=thisDatetime)
1144 1144
1145 1145 return dataOut
1146 1146
1147 1147 @MPDecorator
1148 1148 class Noise(Figure):
1149 1149
1150 1150 isConfig = None
1151 1151 __nsubplots = None
1152 1152
1153 1153 PREFIX = 'noise'
1154 1154
1155 1155
1156 def __init__(self):#, **kwargs):
1157 Figure.__init__(self)#, **kwargs)
1156 def __init__(self):
1157 Figure.__init__(self)
1158 1158 self.timerange = 24*60*60
1159 1159 self.isConfig = False
1160 1160 self.__nsubplots = 1
1161 1161 self.counter_imagwr = 0
1162 1162 self.WIDTH = 800
1163 1163 self.HEIGHT = 400
1164 1164 self.WIDTHPROF = 120
1165 1165 self.HEIGHTPROF = 0
1166 1166 self.xdata = None
1167 1167 self.ydata = None
1168 1168
1169 1169 self.PLOT_CODE = NOISE_CODE
1170 1170
1171 1171 self.FTP_WEI = None
1172 1172 self.EXP_CODE = None
1173 1173 self.SUB_EXP_CODE = None
1174 1174 self.PLOT_POS = None
1175 1175 self.figfile = None
1176 1176
1177 1177 self.xmin = None
1178 1178 self.xmax = None
1179 1179
1180 1180 def getSubplots(self):
1181 1181
1182 1182 ncol = 1
1183 1183 nrow = 1
1184 1184
1185 1185 return nrow, ncol
1186 1186
1187 1187 def openfile(self, filename):
1188 1188 dirname = os.path.dirname(filename)
1189 1189
1190 1190 if not os.path.exists(dirname):
1191 1191 os.mkdir(dirname)
1192 1192
1193 1193 f = open(filename,'w+')
1194 1194 f.write('\n\n')
1195 1195 f.write('JICAMARCA RADIO OBSERVATORY - Noise \n')
1196 1196 f.write('DD MM YYYY HH MM SS Channel0 Channel1 Channel2 Channel3\n\n' )
1197 1197 f.close()
1198 1198
1199 1199 def save_data(self, filename_phase, data, data_datetime):
1200 1200
1201 1201 f=open(filename_phase,'a')
1202 1202
1203 1203 timetuple_data = data_datetime.timetuple()
1204 1204 day = str(timetuple_data.tm_mday)
1205 1205 month = str(timetuple_data.tm_mon)
1206 1206 year = str(timetuple_data.tm_year)
1207 1207 hour = str(timetuple_data.tm_hour)
1208 1208 minute = str(timetuple_data.tm_min)
1209 1209 second = str(timetuple_data.tm_sec)
1210 1210
1211 1211 data_msg = ''
1212 1212 for i in range(len(data)):
1213 1213 data_msg += str(data[i]) + ' '
1214 1214
1215 1215 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' ' + data_msg + '\n')
1216 1216 f.close()
1217 1217
1218 1218
1219 1219 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1220 1220
1221 1221 self.__showprofile = showprofile
1222 1222 self.nplots = nplots
1223 1223
1224 1224 ncolspan = 7
1225 1225 colspan = 6
1226 1226 self.__nsubplots = 2
1227 1227
1228 1228 self.createFigure(id = id,
1229 1229 wintitle = wintitle,
1230 1230 widthplot = self.WIDTH+self.WIDTHPROF,
1231 1231 heightplot = self.HEIGHT+self.HEIGHTPROF,
1232 1232 show=show)
1233 1233
1234 1234 nrow, ncol = self.getSubplots()
1235 1235
1236 1236 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1237 1237
1238 1238
1239 1239 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
1240 1240 xmin=None, xmax=None, ymin=None, ymax=None,
1241 1241 timerange=None,
1242 1242 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1243 1243 server=None, folder=None, username=None, password=None,
1244 1244 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1245 1245
1246 1246 if dataOut.flagNoData:
1247 1247 return dataOut
1248 1248
1249 1249 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1250 1250 return
1251 1251
1252 1252 if channelList == None:
1253 1253 channelIndexList = dataOut.channelIndexList
1254 1254 channelList = dataOut.channelList
1255 1255 else:
1256 1256 channelIndexList = []
1257 1257 for channel in channelList:
1258 1258 if channel not in dataOut.channelList:
1259 1259 raise ValueError("Channel %d is not in dataOut.channelList")
1260 1260 channelIndexList.append(dataOut.channelList.index(channel))
1261 1261
1262 1262 x = dataOut.getTimeRange()
1263 1263 #y = dataOut.getHeiRange()
1264 1264 factor = dataOut.normFactor
1265 1265 noise = dataOut.noise[channelIndexList]/factor
1266 1266 noisedB = 10*numpy.log10(noise)
1267 1267
1268 1268 thisDatetime = dataOut.datatime
1269 1269
1270 1270 title = wintitle + " Noise" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1271 1271 xlabel = ""
1272 1272 ylabel = "Intensity (dB)"
1273 1273 update_figfile = False
1274 1274
1275 1275 if not self.isConfig:
1276 1276
1277 1277 nplots = 1
1278 1278
1279 1279 self.setup(id=id,
1280 1280 nplots=nplots,
1281 1281 wintitle=wintitle,
1282 1282 showprofile=showprofile,
1283 1283 show=show)
1284 1284
1285 1285 if timerange != None:
1286 1286 self.timerange = timerange
1287 1287
1288 1288 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1289 1289
1290 1290 if ymin == None: ymin = numpy.floor(numpy.nanmin(noisedB)) - 10.0
1291 1291 if ymax == None: ymax = numpy.nanmax(noisedB) + 10.0
1292 1292
1293 1293 self.FTP_WEI = ftp_wei
1294 1294 self.EXP_CODE = exp_code
1295 1295 self.SUB_EXP_CODE = sub_exp_code
1296 1296 self.PLOT_POS = plot_pos
1297 1297
1298 1298
1299 1299 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1300 1300 self.isConfig = True
1301 1301 self.figfile = figfile
1302 1302 self.xdata = numpy.array([])
1303 1303 self.ydata = numpy.array([])
1304 1304
1305 1305 update_figfile = True
1306 1306
1307 1307 #open file beacon phase
1308 1308 path = '%s%03d' %(self.PREFIX, self.id)
1309 1309 noise_file = os.path.join(path,'%s.txt'%self.name)
1310 1310 self.filename_noise = os.path.join(figpath,noise_file)
1311 1311
1312 1312 self.setWinTitle(title)
1313 1313
1314 1314 title = "Noise %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1315 1315
1316 1316 legendlabels = ["channel %d"%(idchannel) for idchannel in channelList]
1317 1317 axes = self.axesList[0]
1318 1318
1319 1319 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1320 1320
1321 1321 if len(self.ydata)==0:
1322 1322 self.ydata = noisedB.reshape(-1,1)
1323 1323 else:
1324 1324 self.ydata = numpy.hstack((self.ydata, noisedB.reshape(-1,1)))
1325 1325
1326 1326
1327 1327 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1328 1328 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1329 1329 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1330 1330 XAxisAsTime=True, grid='both'
1331 1331 )
1332 1332
1333 1333 self.draw()
1334 1334
1335 1335 if dataOut.ltctime >= self.xmax:
1336 1336 self.counter_imagwr = wr_period
1337 1337 self.isConfig = False
1338 1338 update_figfile = True
1339 1339
1340 1340 self.save(figpath=figpath,
1341 1341 figfile=figfile,
1342 1342 save=save,
1343 1343 ftp=ftp,
1344 1344 wr_period=wr_period,
1345 1345 thisDatetime=thisDatetime,
1346 1346 update_figfile=update_figfile)
1347 1347
1348 1348 #store data beacon phase
1349 1349 if save:
1350 1350 self.save_data(self.filename_noise, noisedB, thisDatetime)
1351 1351
1352 1352 return dataOut
1353 1353
1354 1354 @MPDecorator
1355 1355 class BeaconPhase(Figure):
1356 1356
1357 1357 __isConfig = None
1358 1358 __nsubplots = None
1359 1359
1360 1360 PREFIX = 'beacon_phase'
1361 1361
1362 def __init__(self):#, **kwargs):
1363 Figure.__init__(self)#, **kwargs)
1362 def __init__(self):
1363 Figure.__init__(self)
1364 1364 self.timerange = 24*60*60
1365 1365 self.isConfig = False
1366 1366 self.__nsubplots = 1
1367 1367 self.counter_imagwr = 0
1368 1368 self.WIDTH = 800
1369 1369 self.HEIGHT = 400
1370 1370 self.WIDTHPROF = 120
1371 1371 self.HEIGHTPROF = 0
1372 1372 self.xdata = None
1373 1373 self.ydata = None
1374 1374
1375 1375 self.PLOT_CODE = BEACON_CODE
1376 1376
1377 1377 self.FTP_WEI = None
1378 1378 self.EXP_CODE = None
1379 1379 self.SUB_EXP_CODE = None
1380 1380 self.PLOT_POS = None
1381 1381
1382 1382 self.filename_phase = None
1383 1383
1384 1384 self.figfile = None
1385 1385
1386 1386 self.xmin = None
1387 1387 self.xmax = None
1388 1388
1389 1389 def getSubplots(self):
1390 1390
1391 1391 ncol = 1
1392 1392 nrow = 1
1393 1393
1394 1394 return nrow, ncol
1395 1395
1396 1396 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1397 1397
1398 1398 self.__showprofile = showprofile
1399 1399 self.nplots = nplots
1400 1400
1401 1401 ncolspan = 7
1402 1402 colspan = 6
1403 1403 self.__nsubplots = 2
1404 1404
1405 1405 self.createFigure(id = id,
1406 1406 wintitle = wintitle,
1407 1407 widthplot = self.WIDTH+self.WIDTHPROF,
1408 1408 heightplot = self.HEIGHT+self.HEIGHTPROF,
1409 1409 show=show)
1410 1410
1411 1411 nrow, ncol = self.getSubplots()
1412 1412
1413 1413 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1414 1414
1415 1415 def save_phase(self, filename_phase):
1416 1416 f = open(filename_phase,'w+')
1417 1417 f.write('\n\n')
1418 1418 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
1419 1419 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
1420 1420 f.close()
1421 1421
1422 1422 def save_data(self, filename_phase, data, data_datetime):
1423 1423 f=open(filename_phase,'a')
1424 1424 timetuple_data = data_datetime.timetuple()
1425 1425 day = str(timetuple_data.tm_mday)
1426 1426 month = str(timetuple_data.tm_mon)
1427 1427 year = str(timetuple_data.tm_year)
1428 1428 hour = str(timetuple_data.tm_hour)
1429 1429 minute = str(timetuple_data.tm_min)
1430 1430 second = str(timetuple_data.tm_sec)
1431 1431 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
1432 1432 f.close()
1433 1433
1434 1434
1435 1435 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1436 1436 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
1437 1437 timerange=None,
1438 1438 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1439 1439 server=None, folder=None, username=None, password=None,
1440 1440 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1441 1441
1442 1442 if dataOut.flagNoData:
1443 1443 return dataOut
1444 1444
1445 1445 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1446 1446 return
1447 1447
1448 1448 if pairsList == None:
1449 1449 pairsIndexList = dataOut.pairsIndexList[:10]
1450 1450 else:
1451 1451 pairsIndexList = []
1452 1452 for pair in pairsList:
1453 1453 if pair not in dataOut.pairsList:
1454 1454 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
1455 1455 pairsIndexList.append(dataOut.pairsList.index(pair))
1456 1456
1457 1457 if pairsIndexList == []:
1458 1458 return
1459 1459
1460 1460 # if len(pairsIndexList) > 4:
1461 1461 # pairsIndexList = pairsIndexList[0:4]
1462 1462
1463 1463 hmin_index = None
1464 1464 hmax_index = None
1465 1465
1466 1466 if hmin != None and hmax != None:
1467 1467 indexes = numpy.arange(dataOut.nHeights)
1468 1468 hmin_list = indexes[dataOut.heightList >= hmin]
1469 1469 hmax_list = indexes[dataOut.heightList <= hmax]
1470 1470
1471 1471 if hmin_list.any():
1472 1472 hmin_index = hmin_list[0]
1473 1473
1474 1474 if hmax_list.any():
1475 1475 hmax_index = hmax_list[-1]+1
1476 1476
1477 1477 x = dataOut.getTimeRange()
1478 1478 #y = dataOut.getHeiRange()
1479 1479
1480 1480
1481 1481 thisDatetime = dataOut.datatime
1482 1482
1483 1483 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1484 1484 xlabel = "Local Time"
1485 1485 ylabel = "Phase (degrees)"
1486 1486
1487 1487 update_figfile = False
1488 1488
1489 1489 nplots = len(pairsIndexList)
1490 1490 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1491 1491 phase_beacon = numpy.zeros(len(pairsIndexList))
1492 1492 for i in range(nplots):
1493 1493 pair = dataOut.pairsList[pairsIndexList[i]]
1494 1494 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
1495 1495 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
1496 1496 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
1497 1497 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
1498 1498 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
1499 1499
1500 1500 #print "Phase %d%d" %(pair[0], pair[1])
1501 1501 #print phase[dataOut.beacon_heiIndexList]
1502 1502
1503 1503 if dataOut.beacon_heiIndexList:
1504 1504 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
1505 1505 else:
1506 1506 phase_beacon[i] = numpy.average(phase)
1507 1507
1508 1508 if not self.isConfig:
1509 1509
1510 1510 nplots = len(pairsIndexList)
1511 1511
1512 1512 self.setup(id=id,
1513 1513 nplots=nplots,
1514 1514 wintitle=wintitle,
1515 1515 showprofile=showprofile,
1516 1516 show=show)
1517 1517
1518 1518 if timerange != None:
1519 1519 self.timerange = timerange
1520 1520
1521 1521 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1522 1522
1523 1523 if ymin == None: ymin = 0
1524 1524 if ymax == None: ymax = 360
1525 1525
1526 1526 self.FTP_WEI = ftp_wei
1527 1527 self.EXP_CODE = exp_code
1528 1528 self.SUB_EXP_CODE = sub_exp_code
1529 1529 self.PLOT_POS = plot_pos
1530 1530
1531 1531 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1532 1532 self.isConfig = True
1533 1533 self.figfile = figfile
1534 1534 self.xdata = numpy.array([])
1535 1535 self.ydata = numpy.array([])
1536 1536
1537 1537 update_figfile = True
1538 1538
1539 1539 #open file beacon phase
1540 1540 path = '%s%03d' %(self.PREFIX, self.id)
1541 1541 beacon_file = os.path.join(path,'%s.txt'%self.name)
1542 1542 self.filename_phase = os.path.join(figpath,beacon_file)
1543 1543 #self.save_phase(self.filename_phase)
1544 1544
1545 1545
1546 1546 #store data beacon phase
1547 1547 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1548 1548
1549 1549 self.setWinTitle(title)
1550 1550
1551 1551
1552 1552 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1553 1553
1554 1554 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
1555 1555
1556 1556 axes = self.axesList[0]
1557 1557
1558 1558 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1559 1559
1560 1560 if len(self.ydata)==0:
1561 1561 self.ydata = phase_beacon.reshape(-1,1)
1562 1562 else:
1563 1563 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1564 1564
1565 1565
1566 1566 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1567 1567 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1568 1568 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1569 1569 XAxisAsTime=True, grid='both'
1570 1570 )
1571 1571
1572 1572 self.draw()
1573 1573
1574 1574 if dataOut.ltctime >= self.xmax:
1575 1575 self.counter_imagwr = wr_period
1576 1576 self.isConfig = False
1577 1577 update_figfile = True
1578 1578
1579 1579 self.save(figpath=figpath,
1580 1580 figfile=figfile,
1581 1581 save=save,
1582 1582 ftp=ftp,
1583 1583 wr_period=wr_period,
1584 1584 thisDatetime=thisDatetime,
1585 1585 update_figfile=update_figfile)
1586 1586
1587 return dataOut #Yong No newline at end of file
1587 return dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,232 +1,232
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator #YONG
10 10 from schainpy.utils import log
11 11 from .figure import Figure
12 12
13 13
14 14 @MPDecorator
15 15 class Scope(Figure):
16 16
17 17 isConfig = None
18 18
19 19 def __init__(self):#, **kwargs): #YONG
20 20 Figure.__init__(self)#, **kwargs)
21 21 self.isConfig = False
22 22 self.WIDTH = 300
23 23 self.HEIGHT = 200
24 24 self.counter_imagwr = 0
25 25
26 26 def getSubplots(self):
27 27
28 28 nrow = self.nplots
29 29 ncol = 3
30 30 return nrow, ncol
31 31
32 32 def setup(self, id, nplots, wintitle, show):
33 33
34 34 self.nplots = nplots
35 35
36 36 self.createFigure(id=id,
37 37 wintitle=wintitle,
38 38 show=show)
39 39
40 40 nrow,ncol = self.getSubplots()
41 41 colspan = 3
42 42 rowspan = 1
43 43
44 44 for i in range(nplots):
45 45 self.addAxes(nrow, ncol, i, 0, colspan, rowspan)
46 46
47 47 def plot_iq(self, x, y, id, channelIndexList, thisDatetime, wintitle, show, xmin, xmax, ymin, ymax):
48 48 yreal = y[channelIndexList,:].real
49 49 yimag = y[channelIndexList,:].imag
50 50
51 51 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
52 52 xlabel = "Range (Km)"
53 53 ylabel = "Intensity - IQ"
54 54
55 55 if not self.isConfig:
56 56 nplots = len(channelIndexList)
57 57
58 58 self.setup(id=id,
59 59 nplots=nplots,
60 60 wintitle='',
61 61 show=show)
62 62
63 63 if xmin == None: xmin = numpy.nanmin(x)
64 64 if xmax == None: xmax = numpy.nanmax(x)
65 65 if ymin == None: ymin = min(numpy.nanmin(yreal),numpy.nanmin(yimag))
66 66 if ymax == None: ymax = max(numpy.nanmax(yreal),numpy.nanmax(yimag))
67 67
68 68 self.isConfig = True
69 69
70 70 self.setWinTitle(title)
71 71
72 72 for i in range(len(self.axesList)):
73 73 title = "Channel %d" %(i)
74 74 axes = self.axesList[i]
75 75
76 76 axes.pline(x, yreal[i,:],
77 77 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
78 78 xlabel=xlabel, ylabel=ylabel, title=title)
79 79
80 80 axes.addpline(x, yimag[i,:], idline=1, color="red", linestyle="solid", lw=2)
81 81
82 82 def plot_power(self, x, y, id, channelIndexList, thisDatetime, wintitle, show, xmin, xmax, ymin, ymax):
83 83 y = y[channelIndexList,:] * numpy.conjugate(y[channelIndexList,:])
84 84 yreal = y.real
85 85
86 86 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
87 87 xlabel = "Range (Km)"
88 88 ylabel = "Intensity"
89 89
90 90 if not self.isConfig:
91 91 nplots = len(channelIndexList)
92 92
93 93 self.setup(id=id,
94 94 nplots=nplots,
95 95 wintitle='',
96 96 show=show)
97 97
98 98 if xmin == None: xmin = numpy.nanmin(x)
99 99 if xmax == None: xmax = numpy.nanmax(x)
100 100 if ymin == None: ymin = numpy.nanmin(yreal)
101 101 if ymax == None: ymax = numpy.nanmax(yreal)
102 102
103 103 self.isConfig = True
104 104
105 105 self.setWinTitle(title)
106 106
107 107 for i in range(len(self.axesList)):
108 108 title = "Channel %d" %(i)
109 109 axes = self.axesList[i]
110 110 ychannel = yreal[i,:]
111 111 axes.pline(x, ychannel,
112 112 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
113 113 xlabel=xlabel, ylabel=ylabel, title=title)
114 114
115 115
116 116 def run(self, dataOut, id, wintitle="", channelList=None,
117 117 xmin=None, xmax=None, ymin=None, ymax=None, save=False,
118 118 figpath='./', figfile=None, show=True, wr_period=1,
119 119 ftp=False, server=None, folder=None, username=None, password=None, type='power', **kwargs):
120 120
121 121 """
122 122
123 123 Input:
124 124 dataOut :
125 125 id :
126 126 wintitle :
127 127 channelList :
128 128 xmin : None,
129 129 xmax : None,
130 130 ymin : None,
131 131 ymax : None,
132 132 """
133 133 if dataOut.flagNoData:
134 134 return dataOut
135 135
136 136 if channelList == None:
137 137 channelIndexList = dataOut.channelIndexList
138 138 else:
139 139 channelIndexList = []
140 140 for channel in channelList:
141 141 if channel not in dataOut.channelList:
142 142 raise ValueError("Channel %d is not in dataOut.channelList")
143 143 channelIndexList.append(dataOut.channelList.index(channel))
144 144
145 145 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
146 146
147 147 if dataOut.flagDataAsBlock:
148 148
149 149 for i in range(dataOut.nProfiles):
150 150
151 151 wintitle1 = wintitle + " [Profile = %d] " %i
152 152
153 153 if type == "power":
154 154 self.plot_power(dataOut.heightList,
155 155 dataOut.data[:,i,:],
156 156 id,
157 157 channelIndexList,
158 158 thisDatetime,
159 159 wintitle1,
160 160 show,
161 161 xmin,
162 162 xmax,
163 163 ymin,
164 164 ymax)
165 165
166 166 if type == "iq":
167 167 self.plot_iq(dataOut.heightList,
168 168 dataOut.data[:,i,:],
169 169 id,
170 170 channelIndexList,
171 171 thisDatetime,
172 172 wintitle1,
173 173 show,
174 174 xmin,
175 175 xmax,
176 176 ymin,
177 177 ymax)
178 178
179 179 self.draw()
180 180
181 181 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
182 182 figfile = self.getFilename(name = str_datetime) + "_" + str(i)
183 183
184 184 self.save(figpath=figpath,
185 185 figfile=figfile,
186 186 save=save,
187 187 ftp=ftp,
188 188 wr_period=wr_period,
189 189 thisDatetime=thisDatetime)
190 190
191 191 else:
192 192 wintitle += " [Profile = %d] " %dataOut.profileIndex
193 193
194 194 if type == "power":
195 195 self.plot_power(dataOut.heightList,
196 196 dataOut.data,
197 197 id,
198 198 channelIndexList,
199 199 thisDatetime,
200 200 wintitle,
201 201 show,
202 202 xmin,
203 203 xmax,
204 204 ymin,
205 205 ymax)
206 206
207 207 if type == "iq":
208 208 self.plot_iq(dataOut.heightList,
209 209 dataOut.data,
210 210 id,
211 211 channelIndexList,
212 212 thisDatetime,
213 213 wintitle,
214 214 show,
215 215 xmin,
216 216 xmax,
217 217 ymin,
218 218 ymax)
219 219
220 220 self.draw()
221 221
222 222 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") + "_" + str(dataOut.profileIndex)
223 223 figfile = self.getFilename(name = str_datetime)
224 224
225 225 self.save(figpath=figpath,
226 226 figfile=figfile,
227 227 save=save,
228 228 ftp=ftp,
229 229 wr_period=wr_period,
230 230 thisDatetime=thisDatetime)
231 231
232 return dataOut No newline at end of file
232 return dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1826 +1,1828
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time
14 14 import datetime
15 15 import traceback
16 16 import zmq
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25 25 from schainpy.utils import log
26 26 import schainpy.admin
27 27
28 28 LOCALTIME = True
29 29
30 30
31 31 def isNumber(cad):
32 32 """
33 33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
34 34
35 35 Excepciones:
36 36 Si un determinado string no puede ser convertido a numero
37 37 Input:
38 38 str, string al cual se le analiza para determinar si convertible a un numero o no
39 39
40 40 Return:
41 41 True : si el string es uno numerico
42 42 False : no es un string numerico
43 43 """
44 44 try:
45 45 float(cad)
46 46 return True
47 47 except:
48 48 return False
49 49
50 50
51 51 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
52 52 """
53 53 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
54 54
55 55 Inputs:
56 56 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
57 57
58 58 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
59 59 segundos contados desde 01/01/1970.
60 60 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
61 61 segundos contados desde 01/01/1970.
62 62
63 63 Return:
64 64 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
65 65 fecha especificado, de lo contrario retorna False.
66 66
67 67 Excepciones:
68 68 Si el archivo no existe o no puede ser abierto
69 69 Si la cabecera no puede ser leida.
70 70
71 71 """
72 72 basicHeaderObj = BasicHeader(LOCALTIME)
73 73
74 74 try:
75 75 fp = open(filename, 'rb')
76 76 except IOError:
77 77 print("The file %s can't be opened" % (filename))
78 78 return 0
79 79
80 80 sts = basicHeaderObj.read(fp)
81 81 fp.close()
82 82
83 83 if not(sts):
84 84 print("Skipping the file %s because it has not a valid header" % (filename))
85 85 return 0
86 86
87 87 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
88 88 return 0
89 89
90 90 return 1
91 91
92 92
93 93 def isTimeInRange(thisTime, startTime, endTime):
94 94 if endTime >= startTime:
95 95 if (thisTime < startTime) or (thisTime > endTime):
96 96 return 0
97 97 return 1
98 98 else:
99 99 if (thisTime < startTime) and (thisTime > endTime):
100 100 return 0
101 101 return 1
102 102
103 103
104 104 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
105 105 """
106 106 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
107 107
108 108 Inputs:
109 109 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
110 110
111 111 startDate : fecha inicial del rango seleccionado en formato datetime.date
112 112
113 113 endDate : fecha final del rango seleccionado en formato datetime.date
114 114
115 115 startTime : tiempo inicial del rango seleccionado en formato datetime.time
116 116
117 117 endTime : tiempo final del rango seleccionado en formato datetime.time
118 118
119 119 Return:
120 120 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
121 121 fecha especificado, de lo contrario retorna False.
122 122
123 123 Excepciones:
124 124 Si el archivo no existe o no puede ser abierto
125 125 Si la cabecera no puede ser leida.
126 126
127 127 """
128 128
129 129 try:
130 130 fp = open(filename, 'rb')
131 131 except IOError:
132 132 print("The file %s can't be opened" % (filename))
133 133 return None
134 134
135 135 firstBasicHeaderObj = BasicHeader(LOCALTIME)
136 136 systemHeaderObj = SystemHeader()
137 137 radarControllerHeaderObj = RadarControllerHeader()
138 138 processingHeaderObj = ProcessingHeader()
139 139
140 140 lastBasicHeaderObj = BasicHeader(LOCALTIME)
141 141
142 142 sts = firstBasicHeaderObj.read(fp)
143 143
144 144 if not(sts):
145 145 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
146 146 return None
147 147
148 148 if not systemHeaderObj.read(fp):
149 149 return None
150 150
151 151 if not radarControllerHeaderObj.read(fp):
152 152 return None
153 153
154 154 if not processingHeaderObj.read(fp):
155 155 return None
156 156
157 157 filesize = os.path.getsize(filename)
158 158
159 159 offset = processingHeaderObj.blockSize + 24 # header size
160 160
161 161 if filesize <= offset:
162 162 print("[Reading] %s: This file has not enough data" % filename)
163 163 return None
164 164
165 165 fp.seek(-offset, 2)
166 166
167 167 sts = lastBasicHeaderObj.read(fp)
168 168
169 169 fp.close()
170 170
171 171 thisDatetime = lastBasicHeaderObj.datatime
172 172 thisTime_last_block = thisDatetime.time()
173 173
174 174 thisDatetime = firstBasicHeaderObj.datatime
175 175 thisDate = thisDatetime.date()
176 176 thisTime_first_block = thisDatetime.time()
177 177
178 178 # General case
179 179 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
180 180 #-----------o----------------------------o-----------
181 181 # startTime endTime
182 182
183 183 if endTime >= startTime:
184 184 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
185 185 return None
186 186
187 187 return thisDatetime
188 188
189 189 # If endTime < startTime then endTime belongs to the next day
190 190
191 191 #<<<<<<<<<<<o o>>>>>>>>>>>
192 192 #-----------o----------------------------o-----------
193 193 # endTime startTime
194 194
195 195 if (thisDate == startDate) and (thisTime_last_block < startTime):
196 196 return None
197 197
198 198 if (thisDate == endDate) and (thisTime_first_block > endTime):
199 199 return None
200 200
201 201 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
202 202 return None
203 203
204 204 return thisDatetime
205 205
206 206
207 207 def isFolderInDateRange(folder, startDate=None, endDate=None):
208 208 """
209 209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
210 210
211 211 Inputs:
212 212 folder : nombre completo del directorio.
213 213 Su formato deberia ser "/path_root/?YYYYDDD"
214 214
215 215 siendo:
216 216 YYYY : Anio (ejemplo 2015)
217 217 DDD : Dia del anio (ejemplo 305)
218 218
219 219 startDate : fecha inicial del rango seleccionado en formato datetime.date
220 220
221 221 endDate : fecha final del rango seleccionado en formato datetime.date
222 222
223 223 Return:
224 224 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
225 225 fecha especificado, de lo contrario retorna False.
226 226 Excepciones:
227 227 Si el directorio no tiene el formato adecuado
228 228 """
229 229
230 230 basename = os.path.basename(folder)
231 231
232 232 if not isRadarFolder(basename):
233 233 print("The folder %s has not the rigth format" % folder)
234 234 return 0
235 235
236 236 if startDate and endDate:
237 237 thisDate = getDateFromRadarFolder(basename)
238 238
239 239 if thisDate < startDate:
240 240 return 0
241 241
242 242 if thisDate > endDate:
243 243 return 0
244 244
245 245 return 1
246 246
247 247
248 248 def isFileInDateRange(filename, startDate=None, endDate=None):
249 249 """
250 250 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
251 251
252 252 Inputs:
253 253 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
254 254
255 255 Su formato deberia ser "?YYYYDDDsss"
256 256
257 257 siendo:
258 258 YYYY : Anio (ejemplo 2015)
259 259 DDD : Dia del anio (ejemplo 305)
260 260 sss : set
261 261
262 262 startDate : fecha inicial del rango seleccionado en formato datetime.date
263 263
264 264 endDate : fecha final del rango seleccionado en formato datetime.date
265 265
266 266 Return:
267 267 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
268 268 fecha especificado, de lo contrario retorna False.
269 269 Excepciones:
270 270 Si el archivo no tiene el formato adecuado
271 271 """
272 272
273 273 basename = os.path.basename(filename)
274 274
275 275 if not isRadarFile(basename):
276 276 print("The filename %s has not the rigth format" % filename)
277 277 return 0
278 278
279 279 if startDate and endDate:
280 280 thisDate = getDateFromRadarFile(basename)
281 281
282 282 if thisDate < startDate:
283 283 return 0
284 284
285 285 if thisDate > endDate:
286 286 return 0
287 287
288 288 return 1
289 289
290 290
291 291 def getFileFromSet(path, ext, set):
292 292 validFilelist = []
293 293 fileList = os.listdir(path)
294 294
295 295 # 0 1234 567 89A BCDE
296 296 # H YYYY DDD SSS .ext
297 297
298 298 for thisFile in fileList:
299 299 try:
300 300 year = int(thisFile[1:5])
301 301 doy = int(thisFile[5:8])
302 302 except:
303 303 continue
304 304
305 305 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
306 306 continue
307 307
308 308 validFilelist.append(thisFile)
309 309
310 310 myfile = fnmatch.filter(
311 311 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
312 312
313 313 if len(myfile) != 0:
314 314 return myfile[0]
315 315 else:
316 316 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
317 317 print('the filename %s does not exist' % filename)
318 318 print('...going to the last file: ')
319 319
320 320 if validFilelist:
321 321 validFilelist = sorted(validFilelist, key=str.lower)
322 322 return validFilelist[-1]
323 323
324 324 return None
325 325
326 326
327 327 def getlastFileFromPath(path, ext):
328 328 """
329 329 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
330 330 al final de la depuracion devuelve el ultimo file de la lista que quedo.
331 331
332 332 Input:
333 333 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
334 334 ext : extension de los files contenidos en una carpeta
335 335
336 336 Return:
337 337 El ultimo file de una determinada carpeta, no se considera el path.
338 338 """
339 339 validFilelist = []
340 340 fileList = os.listdir(path)
341 341
342 342 # 0 1234 567 89A BCDE
343 343 # H YYYY DDD SSS .ext
344 344
345 345 for thisFile in fileList:
346 346
347 347 year = thisFile[1:5]
348 348 if not isNumber(year):
349 349 continue
350 350
351 351 doy = thisFile[5:8]
352 352 if not isNumber(doy):
353 353 continue
354 354
355 355 year = int(year)
356 356 doy = int(doy)
357 357
358 358 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
359 359 continue
360 360
361 361 validFilelist.append(thisFile)
362 362
363 363 if validFilelist:
364 364 validFilelist = sorted(validFilelist, key=str.lower)
365 365 return validFilelist[-1]
366 366
367 367 return None
368 368
369 369
370 370 def checkForRealPath(path, foldercounter, year, doy, set, ext):
371 371 """
372 372 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
373 373 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
374 374 el path exacto de un determinado file.
375 375
376 376 Example :
377 377 nombre correcto del file es .../.../D2009307/P2009307367.ext
378 378
379 379 Entonces la funcion prueba con las siguientes combinaciones
380 380 .../.../y2009307367.ext
381 381 .../.../Y2009307367.ext
382 382 .../.../x2009307/y2009307367.ext
383 383 .../.../x2009307/Y2009307367.ext
384 384 .../.../X2009307/y2009307367.ext
385 385 .../.../X2009307/Y2009307367.ext
386 386 siendo para este caso, la ultima combinacion de letras, identica al file buscado
387 387
388 388 Return:
389 389 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
390 390 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
391 391 para el filename
392 392 """
393 393 fullfilename = None
394 394 find_flag = False
395 395 filename = None
396 396
397 397 prefixDirList = [None, 'd', 'D']
398 398 if ext.lower() == ".r": # voltage
399 399 prefixFileList = ['d', 'D']
400 400 elif ext.lower() == ".pdata": # spectra
401 401 prefixFileList = ['p', 'P']
402 402 else:
403 403 return None, filename
404 404
405 405 # barrido por las combinaciones posibles
406 406 for prefixDir in prefixDirList:
407 407 thispath = path
408 408 if prefixDir != None:
409 409 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
410 410 if foldercounter == 0:
411 411 thispath = os.path.join(path, "%s%04d%03d" %
412 412 (prefixDir, year, doy))
413 413 else:
414 414 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
415 415 prefixDir, year, doy, foldercounter))
416 416 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
417 417 # formo el nombre del file xYYYYDDDSSS.ext
418 418 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
419 419 fullfilename = os.path.join(
420 420 thispath, filename) # formo el path completo
421 421
422 422 if os.path.exists(fullfilename): # verifico que exista
423 423 find_flag = True
424 424 break
425 425 if find_flag:
426 426 break
427 427
428 428 if not(find_flag):
429 429 return None, filename
430 430
431 431 return fullfilename, filename
432 432
433 433
434 434 def isRadarFolder(folder):
435 435 try:
436 436 year = int(folder[1:5])
437 437 doy = int(folder[5:8])
438 438 except:
439 439 return 0
440 440
441 441 return 1
442 442
443 443
444 444 def isRadarFile(file):
445 445 try:
446 446 year = int(file[1:5])
447 447 doy = int(file[5:8])
448 448 set = int(file[8:11])
449 449 except:
450 450 return 0
451 451
452 452 return 1
453 453
454 454
455 455 def getDateFromRadarFile(file):
456 456 try:
457 457 year = int(file[1:5])
458 458 doy = int(file[5:8])
459 459 set = int(file[8:11])
460 460 except:
461 461 return None
462 462
463 463 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
464 464 return thisDate
465 465
466 466
467 467 def getDateFromRadarFolder(folder):
468 468 try:
469 469 year = int(folder[1:5])
470 470 doy = int(folder[5:8])
471 471 except:
472 472 return None
473 473
474 474 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
475 475 return thisDate
476 476
477 477
478 478 class JRODataIO:
479 479
480 480 c = 3E8
481 481
482 482 isConfig = False
483 483
484 484 basicHeaderObj = None
485 485
486 486 systemHeaderObj = None
487 487
488 488 radarControllerHeaderObj = None
489 489
490 490 processingHeaderObj = None
491 491
492 492 dtype = None
493 493
494 494 pathList = []
495 495
496 496 filenameList = []
497 497
498 498 filename = None
499 499
500 500 ext = None
501 501
502 502 flagIsNewFile = 1
503 503
504 504 flagDiscontinuousBlock = 0
505 505
506 506 flagIsNewBlock = 0
507 507
508 508 fp = None
509 509
510 510 firstHeaderSize = 0
511 511
512 512 basicHeaderSize = 24
513 513
514 514 versionFile = 1103
515 515
516 516 fileSize = None
517 517
518 518 # ippSeconds = None
519 519
520 520 fileSizeByHeader = None
521 521
522 522 fileIndex = None
523 523
524 524 profileIndex = None
525 525
526 526 blockIndex = None
527 527
528 528 nTotalBlocks = None
529 529
530 530 maxTimeStep = 30
531 531
532 532 lastUTTime = None
533 533
534 534 datablock = None
535 535
536 536 dataOut = None
537 537
538 538 blocksize = None
539 539
540 540 getByBlock = False
541 541
542 542 def __init__(self):
543 543
544 544 raise NotImplementedError
545 545
546 546 def run(self):
547 547
548 548 raise NotImplementedError
549 549
550 550 def getDtypeWidth(self):
551 551
552 552 dtype_index = get_dtype_index(self.dtype)
553 553 dtype_width = get_dtype_width(dtype_index)
554 554
555 555 return dtype_width
556 556
557 557 def getAllowedArgs(self):
558 558 if hasattr(self, '__attrs__'):
559 559 return self.__attrs__
560 560 else:
561 561 return inspect.getargspec(self.run).args
562 562
563 563
564 564 class JRODataReader(JRODataIO):
565 565
566 566 online = 0
567 567
568 568 realtime = 0
569 569
570 570 nReadBlocks = 0
571 571
572 572 delay = 10 # number of seconds waiting a new file
573 573
574 574 nTries = 3 # quantity tries
575 575
576 576 nFiles = 3 # number of files for searching
577 577
578 578 path = None
579 579
580 580 foldercounter = 0
581 581
582 582 flagNoMoreFiles = 0
583 583
584 584 datetimeList = []
585 585
586 586 __isFirstTimeOnline = 1
587 587
588 588 __printInfo = True
589 589
590 590 profileIndex = None
591 591
592 592 nTxs = 1
593 593
594 594 txIndex = None
595 595
596 596 # Added--------------------
597 597
598 598 selBlocksize = None
599 599
600 600 selBlocktime = None
601 601
602 602 def __init__(self):
603 603 """
604 604 This class is used to find data files
605 605
606 606 Example:
607 607 reader = JRODataReader()
608 608 fileList = reader.findDataFiles()
609 609
610 610 """
611 611 pass
612 612
613 613 def createObjByDefault(self):
614 614 """
615 615
616 616 """
617 617 raise NotImplementedError
618 618
619 619 def getBlockDimension(self):
620 620
621 621 raise NotImplementedError
622 622
623 623 def searchFilesOffLine(self,
624 624 path,
625 625 startDate=None,
626 626 endDate=None,
627 627 startTime=datetime.time(0, 0, 0),
628 628 endTime=datetime.time(23, 59, 59),
629 629 set=None,
630 630 expLabel='',
631 631 ext='.r',
632 632 cursor=None,
633 633 skip=None,
634 634 walk=True):
635 635
636 636 self.filenameList = []
637 637 self.datetimeList = []
638 638
639 639 pathList = []
640 640
641 641 dateList, pathList = self.findDatafiles(
642 642 path, startDate, endDate, expLabel, ext, walk, include_path=True)
643 643
644 644 if dateList == []:
645 645 return [], []
646 646
647 647 if len(dateList) > 1:
648 648 print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
649 649 else:
650 650 print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
651 651
652 652 filenameList = []
653 653 datetimeList = []
654 654
655 655 for thisPath in pathList:
656 656
657 657 fileList = glob.glob1(thisPath, "*%s" % ext)
658 658 fileList.sort()
659 659
660 660 for file in fileList:
661 661
662 662 filename = os.path.join(thisPath, file)
663 663
664 664 if not isFileInDateRange(filename, startDate, endDate):
665 665 continue
666 666
667 667 thisDatetime = isFileInTimeRange(
668 668 filename, startDate, endDate, startTime, endTime)
669 669
670 670 if not(thisDatetime):
671 671 continue
672 672
673 673 filenameList.append(filename)
674 674 datetimeList.append(thisDatetime)
675 675
676 676 if cursor is not None and skip is not None:
677 677 filenameList = filenameList[cursor * skip:cursor * skip + skip]
678 678 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
679 679
680 680 if not(filenameList):
681 681 print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
682 682 return [], []
683 683
684 684 print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
685 685
686 686 # for i in range(len(filenameList)):
687 687 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
688 688
689 689 self.filenameList = filenameList
690 690 self.datetimeList = datetimeList
691 691
692 692 return pathList, filenameList
693 693
694 694 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
695 695 """
696 696 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
697 697 devuelve el archivo encontrado ademas de otros datos.
698 698
699 699 Input:
700 700 path : carpeta donde estan contenidos los files que contiene data
701 701
702 702 expLabel : Nombre del subexperimento (subfolder)
703 703
704 704 ext : extension de los files
705 705
706 706 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
707 707
708 708 Return:
709 709 directory : eL directorio donde esta el file encontrado
710 710 filename : el ultimo file de una determinada carpeta
711 711 year : el anho
712 712 doy : el numero de dia del anho
713 713 set : el set del archivo
714 714
715 715
716 716 """
717 717 if not os.path.isdir(path):
718 718 return None, None, None, None, None, None
719 719
720 720 dirList = []
721 721
722 722 if not walk:
723 723 fullpath = path
724 724 foldercounter = 0
725 725 else:
726 726 # Filtra solo los directorios
727 727 for thisPath in os.listdir(path):
728 728 if not os.path.isdir(os.path.join(path, thisPath)):
729 729 continue
730 730 if not isRadarFolder(thisPath):
731 731 continue
732 732
733 733 dirList.append(thisPath)
734 734
735 735 if not(dirList):
736 736 return None, None, None, None, None, None
737 737
738 738 dirList = sorted(dirList, key=str.lower)
739 739
740 740 doypath = dirList[-1]
741 741 foldercounter = int(doypath.split('_')[1]) if len(
742 742 doypath.split('_')) > 1 else 0
743 743 fullpath = os.path.join(path, doypath, expLabel)
744 744
745 745 print("[Reading] %s folder was found: " % (fullpath))
746 746
747 747 if set == None:
748 748 filename = getlastFileFromPath(fullpath, ext)
749 749 else:
750 750 filename = getFileFromSet(fullpath, ext, set)
751 751
752 752 if not(filename):
753 753 return None, None, None, None, None, None
754 754
755 755 print("[Reading] %s file was found" % (filename))
756 756
757 757 if not(self.__verifyFile(os.path.join(fullpath, filename))):
758 758 return None, None, None, None, None, None
759 759
760 760 year = int(filename[1:5])
761 761 doy = int(filename[5:8])
762 762 set = int(filename[8:11])
763 763
764 764 return fullpath, foldercounter, filename, year, doy, set
765 765
766 766 def __setNextFileOffline(self):
767 767
768 768 idFile = self.fileIndex
769 769
770 770 while (True):
771 771 idFile += 1
772 772 if not(idFile < len(self.filenameList)):
773 773 self.flagNoMoreFiles = 1
774 774 # print "[Reading] No more Files"
775 775 return 0
776 776
777 777 filename = self.filenameList[idFile]
778 778
779 779 if not(self.__verifyFile(filename)):
780 780 continue
781 781
782 782 fileSize = os.path.getsize(filename)
783 783 fp = open(filename, 'rb')
784 784 break
785 785
786 786 self.flagIsNewFile = 1
787 787 self.fileIndex = idFile
788 788 self.filename = filename
789 789 self.fileSize = fileSize
790 790 self.fp = fp
791 791
792 792 # print "[Reading] Setting the file: %s"%self.filename
793 793
794 794 return 1
795 795
796 796 def __setNextFileOnline(self):
797 797 """
798 798 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
799 799 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
800 800 siguientes.
801 801
802 802 Affected:
803 803 self.flagIsNewFile
804 804 self.filename
805 805 self.fileSize
806 806 self.fp
807 807 self.set
808 808 self.flagNoMoreFiles
809 809
810 810 Return:
811 811 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
812 812 1 : si el file fue abierto con exito y esta listo a ser leido
813 813
814 814 Excepciones:
815 815 Si un determinado file no puede ser abierto
816 816 """
817 817 nFiles = 0
818 818 fileOk_flag = False
819 819 firstTime_flag = True
820 820
821 821 self.set += 1
822 822
823 823 if self.set > 999:
824 824 self.set = 0
825 825 self.foldercounter += 1
826 826
827 827 # busca el 1er file disponible
828 828 fullfilename, filename = checkForRealPath(
829 829 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
830 830 if fullfilename:
831 831 if self.__verifyFile(fullfilename, False):
832 832 fileOk_flag = True
833 833
834 834 # si no encuentra un file entonces espera y vuelve a buscar
835 835 if not(fileOk_flag):
836 836 # busco en los siguientes self.nFiles+1 files posibles
837 837 for nFiles in range(self.nFiles + 1):
838 838
839 839 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
840 840 tries = self.nTries
841 841 else:
842 842 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
843 843
844 844 for nTries in range(tries):
845 845 if firstTime_flag:
846 846 print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1))
847 847 sleep(self.delay)
848 848 else:
849 849 print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext))
850 850
851 851 fullfilename, filename = checkForRealPath(
852 852 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
853 853 if fullfilename:
854 854 if self.__verifyFile(fullfilename):
855 855 fileOk_flag = True
856 856 break
857 857
858 858 if fileOk_flag:
859 859 break
860 860
861 861 firstTime_flag = False
862 862
863 863 log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
864 864 self.set += 1
865 865
866 866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
867 867 if nFiles == (self.nFiles - 1):
868 868 self.set = 0
869 869 self.doy += 1
870 870 self.foldercounter = 0
871 871
872 872 if fileOk_flag:
873 873 self.fileSize = os.path.getsize(fullfilename)
874 874 self.filename = fullfilename
875 875 self.flagIsNewFile = 1
876 876 if self.fp != None:
877 877 self.fp.close()
878 878 self.fp = open(fullfilename, 'rb')
879 879 self.flagNoMoreFiles = 0
880 880 # print '[Reading] Setting the file: %s' % fullfilename
881 881 else:
882 882 self.fileSize = 0
883 883 self.filename = None
884 884 self.flagIsNewFile = 0
885 885 self.fp = None
886 886 self.flagNoMoreFiles = 1
887 887 # print '[Reading] No more files to read'
888 888
889 889 return fileOk_flag
890 890
891 891 def setNextFile(self):
892 892 if self.fp != None:
893 893 self.fp.close()
894 894
895 895 if self.online:
896 896 newFile = self.__setNextFileOnline()
897 897 else:
898 898 newFile = self.__setNextFileOffline()
899 899
900 900 if not(newFile):
901 901 self.dataOut.error = (-1, 'No more files to read')
902 902 return 0
903 903
904 904 if self.verbose:
905 905 print('[Reading] Setting the file: %s' % self.filename)
906 906
907 907 self.__readFirstHeader()
908 908 self.nReadBlocks = 0
909 909 return 1
910 910
911 911 def __waitNewBlock(self):
912 912 """
913 913 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
914 914
915 915 Si el modo de lectura es OffLine siempre retorn 0
916 916 """
917 917 if not self.online:
918 918 return 0
919 919
920 920 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
921 921 return 0
922 922
923 923 currentPointer = self.fp.tell()
924 924
925 925 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
926 926
927 927 for nTries in range(self.nTries):
928 928
929 929 self.fp.close()
930 930 self.fp = open(self.filename, 'rb')
931 931 self.fp.seek(currentPointer)
932 932
933 933 self.fileSize = os.path.getsize(self.filename)
934 934 currentSize = self.fileSize - currentPointer
935 935
936 936 if (currentSize >= neededSize):
937 937 self.basicHeaderObj.read(self.fp)
938 938 return 1
939 939
940 940 if self.fileSize == self.fileSizeByHeader:
941 941 # self.flagEoF = True
942 942 return 0
943 943
944 944 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
945 945 sleep(self.delay)
946 946
947 947 return 0
948 948
949 949 def waitDataBlock(self, pointer_location):
950 950
951 951 currentPointer = pointer_location
952 952
953 953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
954 954
955 955 for nTries in range(self.nTries):
956 956 self.fp.close()
957 957 self.fp = open(self.filename, 'rb')
958 958 self.fp.seek(currentPointer)
959 959
960 960 self.fileSize = os.path.getsize(self.filename)
961 961 currentSize = self.fileSize - currentPointer
962 962
963 963 if (currentSize >= neededSize):
964 964 return 1
965 965
966 966 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
967 967 sleep(self.delay)
968 968
969 969 return 0
970 970
971 971 def __jumpToLastBlock(self):
972 972
973 973 if not(self.__isFirstTimeOnline):
974 974 return
975 975
976 976 csize = self.fileSize - self.fp.tell()
977 977 blocksize = self.processingHeaderObj.blockSize
978 978
979 979 # salta el primer bloque de datos
980 980 if csize > self.processingHeaderObj.blockSize:
981 981 self.fp.seek(self.fp.tell() + blocksize)
982 982 else:
983 983 return
984 984
985 985 csize = self.fileSize - self.fp.tell()
986 986 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
987 987 while True:
988 988
989 989 if self.fp.tell() < self.fileSize:
990 990 self.fp.seek(self.fp.tell() + neededsize)
991 991 else:
992 992 self.fp.seek(self.fp.tell() - neededsize)
993 993 break
994 994
995 995 # csize = self.fileSize - self.fp.tell()
996 996 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
997 997 # factor = int(csize/neededsize)
998 998 # if factor > 0:
999 999 # self.fp.seek(self.fp.tell() + factor*neededsize)
1000 1000
1001 1001 self.flagIsNewFile = 0
1002 1002 self.__isFirstTimeOnline = 0
1003 1003
1004 1004 def __setNewBlock(self):
1005 1005 # if self.server is None:
1006 1006 if self.fp == None:
1007 1007 return 0
1008 1008
1009 1009 # if self.online:
1010 1010 # self.__jumpToLastBlock()
1011 1011
1012 1012 if self.flagIsNewFile:
1013 1013 self.lastUTTime = self.basicHeaderObj.utc
1014 1014 return 1
1015 1015
1016 1016 if self.realtime:
1017 1017 self.flagDiscontinuousBlock = 1
1018 1018 if not(self.setNextFile()):
1019 1019 return 0
1020 1020 else:
1021 1021 return 1
1022 1022 # if self.server is None:
1023 1023 currentSize = self.fileSize - self.fp.tell()
1024 1024 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1025 1025 if (currentSize >= neededSize):
1026 1026 self.basicHeaderObj.read(self.fp)
1027 1027 self.lastUTTime = self.basicHeaderObj.utc
1028 1028 return 1
1029 1029 # else:
1030 1030 # self.basicHeaderObj.read(self.zHeader)
1031 1031 # self.lastUTTime = self.basicHeaderObj.utc
1032 1032 # return 1
1033 1033 if self.__waitNewBlock():
1034 1034 self.lastUTTime = self.basicHeaderObj.utc
1035 1035 return 1
1036 1036 # if self.server is None:
1037 1037 if not(self.setNextFile()):
1038 1038 return 0
1039 1039
1040 1040 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1041 1041 self.lastUTTime = self.basicHeaderObj.utc
1042 1042
1043 1043 self.flagDiscontinuousBlock = 0
1044 1044
1045 1045 if deltaTime > self.maxTimeStep:
1046 1046 self.flagDiscontinuousBlock = 1
1047 1047
1048 1048 return 1
1049 1049
1050 1050 def readNextBlock(self):
1051 1051
1052 1052 # Skip block out of startTime and endTime
1053 1053 while True:
1054 1054 if not(self.__setNewBlock()):
1055 1055 self.dataOut.error = (-1, 'No more files to read')
1056 1056 return 0
1057 1057
1058 1058 if not(self.readBlock()):
1059 1059 return 0
1060 1060
1061 1061 self.getBasicHeader()
1062 1062 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1063 1063 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1064 1064 self.processingHeaderObj.dataBlocksPerFile,
1065 1065 self.dataOut.datatime.ctime()))
1066 1066 continue
1067 1067
1068 1068 break
1069 1069
1070 1070 if self.verbose:
1071 1071 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1072 1072 self.processingHeaderObj.dataBlocksPerFile,
1073 1073 self.dataOut.datatime.ctime()))
1074 1074 return 1
1075 1075
1076 1076 def __readFirstHeader(self):
1077 1077
1078 1078 self.basicHeaderObj.read(self.fp)
1079 1079 self.systemHeaderObj.read(self.fp)
1080 1080 self.radarControllerHeaderObj.read(self.fp)
1081 1081 self.processingHeaderObj.read(self.fp)
1082 1082
1083 1083 self.firstHeaderSize = self.basicHeaderObj.size
1084 1084
1085 1085 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1086 1086 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1087 1087 if datatype == 0:
1088 1088 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1089 1089 elif datatype == 1:
1090 1090 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1091 1091 elif datatype == 2:
1092 1092 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1093 1093 elif datatype == 3:
1094 1094 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1095 1095 elif datatype == 4:
1096 1096 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1097 1097 elif datatype == 5:
1098 1098 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1099 1099 else:
1100 1100 raise ValueError('Data type was not defined')
1101 1101
1102 1102 self.dtype = datatype_str
1103 1103 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1104 1104 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1105 1105 self.firstHeaderSize + self.basicHeaderSize * \
1106 1106 (self.processingHeaderObj.dataBlocksPerFile - 1)
1107 1107 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1108 1108 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1109 1109 self.getBlockDimension()
1110 1110
1111 1111 def __verifyFile(self, filename, msgFlag=True):
1112 1112
1113 1113 msg = None
1114 1114
1115 1115 try:
1116 1116 fp = open(filename, 'rb')
1117 1117 except IOError:
1118 1118
1119 1119 if msgFlag:
1120 1120 print("[Reading] File %s can't be opened" % (filename))
1121 1121
1122 1122 return False
1123 1123
1124 1124 currentPosition = fp.tell()
1125 1125 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1126 1126
1127 1127 if neededSize == 0:
1128 1128 basicHeaderObj = BasicHeader(LOCALTIME)
1129 1129 systemHeaderObj = SystemHeader()
1130 1130 radarControllerHeaderObj = RadarControllerHeader()
1131 1131 processingHeaderObj = ProcessingHeader()
1132 1132
1133 1133 if not(basicHeaderObj.read(fp)):
1134 1134 fp.close()
1135 1135 return False
1136 1136
1137 1137 if not(systemHeaderObj.read(fp)):
1138 1138 fp.close()
1139 1139 return False
1140 1140
1141 1141 if not(radarControllerHeaderObj.read(fp)):
1142 1142 fp.close()
1143 1143 return False
1144 1144
1145 1145 if not(processingHeaderObj.read(fp)):
1146 1146 fp.close()
1147 1147 return False
1148 1148
1149 1149 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1150 1150 else:
1151 1151 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1152 1152
1153 1153 fp.close()
1154 1154
1155 1155 fileSize = os.path.getsize(filename)
1156 1156 currentSize = fileSize - currentPosition
1157 1157
1158 1158 if currentSize < neededSize:
1159 1159 if msgFlag and (msg != None):
1160 1160 print(msg)
1161 1161 return False
1162 1162
1163 1163 return True
1164 1164
1165 1165 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1166 1166
1167 1167 path_empty = True
1168 1168
1169 1169 dateList = []
1170 1170 pathList = []
1171 1171
1172 1172 multi_path = path.split(',')
1173 1173
1174 1174 if not walk:
1175 1175
1176 1176 for single_path in multi_path:
1177 1177
1178 1178 if not os.path.isdir(single_path):
1179 1179 continue
1180 1180
1181 1181 fileList = glob.glob1(single_path, "*" + ext)
1182 1182
1183 1183 if not fileList:
1184 1184 continue
1185 1185
1186 1186 path_empty = False
1187 1187
1188 1188 fileList.sort()
1189 1189
1190 1190 for thisFile in fileList:
1191 1191
1192 1192 if not os.path.isfile(os.path.join(single_path, thisFile)):
1193 1193 continue
1194 1194
1195 1195 if not isRadarFile(thisFile):
1196 1196 continue
1197 1197
1198 1198 if not isFileInDateRange(thisFile, startDate, endDate):
1199 1199 continue
1200 1200
1201 1201 thisDate = getDateFromRadarFile(thisFile)
1202 1202
1203 1203 if thisDate in dateList:
1204 1204 continue
1205 1205
1206 1206 dateList.append(thisDate)
1207 1207 pathList.append(single_path)
1208 1208
1209 1209 else:
1210 1210 for single_path in multi_path:
1211 1211
1212 1212 if not os.path.isdir(single_path):
1213 1213 continue
1214 1214
1215 1215 dirList = []
1216 1216
1217 1217 for thisPath in os.listdir(single_path):
1218 1218
1219 1219 if not os.path.isdir(os.path.join(single_path, thisPath)):
1220 1220 continue
1221 1221
1222 1222 if not isRadarFolder(thisPath):
1223 1223 continue
1224 1224
1225 1225 if not isFolderInDateRange(thisPath, startDate, endDate):
1226 1226 continue
1227 1227
1228 1228 dirList.append(thisPath)
1229 1229
1230 1230 if not dirList:
1231 1231 continue
1232 1232
1233 1233 dirList.sort()
1234 1234
1235 1235 for thisDir in dirList:
1236 1236
1237 1237 datapath = os.path.join(single_path, thisDir, expLabel)
1238 1238 fileList = glob.glob1(datapath, "*" + ext)
1239 1239
1240 1240 if not fileList:
1241 1241 continue
1242 1242
1243 1243 path_empty = False
1244 1244
1245 1245 thisDate = getDateFromRadarFolder(thisDir)
1246 1246
1247 1247 pathList.append(datapath)
1248 1248 dateList.append(thisDate)
1249 1249
1250 1250 dateList.sort()
1251 1251
1252 1252 if walk:
1253 1253 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1254 1254 else:
1255 1255 pattern_path = multi_path[0]
1256 1256
1257 1257 if path_empty:
1258 1258 print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1259 1259 else:
1260 1260 if not dateList:
1261 1261 print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1262 1262
1263 1263 if include_path:
1264 1264 return dateList, pathList
1265 1265
1266 1266 return dateList
1267 1267
1268 1268 def setup(self,
1269 1269 path=None,
1270 1270 startDate=None,
1271 1271 endDate=None,
1272 1272 startTime=datetime.time(0, 0, 0),
1273 1273 endTime=datetime.time(23, 59, 59),
1274 1274 set=None,
1275 1275 expLabel="",
1276 1276 ext=None,
1277 1277 online=False,
1278 1278 delay=60,
1279 1279 walk=True,
1280 1280 getblock=False,
1281 1281 nTxs=1,
1282 1282 realtime=False,
1283 1283 blocksize=None,
1284 1284 blocktime=None,
1285 1285 skip=None,
1286 1286 cursor=None,
1287 1287 warnings=True,
1288 1288 verbose=True,
1289 1289 server=None,
1290 1290 format=None,
1291 1291 oneDDict=None,
1292 1292 twoDDict=None,
1293 1293 ind2DList=None):
1294 1294 if server is not None:
1295 1295 if 'tcp://' in server:
1296 1296 address = server
1297 1297 else:
1298 1298 address = 'ipc:///tmp/%s' % server
1299 1299 self.server = address
1300 1300 self.context = zmq.Context()
1301 1301 self.receiver = self.context.socket(zmq.PULL)
1302 1302 self.receiver.connect(self.server)
1303 1303 time.sleep(0.5)
1304 1304 print('[Starting] ReceiverData from {}'.format(self.server))
1305 1305 else:
1306 1306 self.server = None
1307 1307 if path == None:
1308 1308 raise ValueError("[Reading] The path is not valid")
1309 1309
1310 1310 if ext == None:
1311 1311 ext = self.ext
1312 1312
1313 1313 if online:
1314 1314 print("[Reading] Searching files in online mode...")
1315 1315
1316 1316 for nTries in range(self.nTries):
1317 1317 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1318 1318 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1319 1319
1320 1320 if fullpath:
1321 1321 break
1322 1322
1323 1323 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
1324 1324 sleep(self.delay)
1325 1325
1326 1326 if not(fullpath):
1327 1327 self.dataOut.error = (-1, 'There isn\'t any valid file in {}'.format(path))
1328 1328 return
1329 1329
1330 1330 self.year = year
1331 1331 self.doy = doy
1332 1332 self.set = set - 1
1333 1333 self.path = path
1334 1334 self.foldercounter = foldercounter
1335 1335 last_set = None
1336 1336 else:
1337 1337 print("[Reading] Searching files in offline mode ...")
1338 1338 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1339 1339 startTime=startTime, endTime=endTime,
1340 1340 set=set, expLabel=expLabel, ext=ext,
1341 1341 walk=walk, cursor=cursor,
1342 1342 skip=skip)
1343 1343
1344 1344 if not(pathList):
1345 1345 self.fileIndex = -1
1346 1346 self.pathList = []
1347 1347 self.filenameList = []
1348 1348 return
1349 1349
1350 1350 self.fileIndex = -1
1351 1351 self.pathList = pathList
1352 1352 self.filenameList = filenameList
1353 1353 file_name = os.path.basename(filenameList[-1])
1354 1354 basename, ext = os.path.splitext(file_name)
1355 1355 last_set = int(basename[-3:])
1356 1356
1357 1357 self.online = online
1358 1358 self.realtime = realtime
1359 1359 self.delay = delay
1360 1360 ext = ext.lower()
1361 1361 self.ext = ext
1362 1362 self.getByBlock = getblock
1363 1363 self.nTxs = nTxs
1364 1364 self.startTime = startTime
1365 1365 self.endTime = endTime
1366 1366 self.endDate = endDate
1367 1367 self.startDate = startDate
1368 1368 # Added-----------------
1369 1369 self.selBlocksize = blocksize
1370 1370 self.selBlocktime = blocktime
1371 1371
1372 1372 # Verbose-----------
1373 1373 self.verbose = verbose
1374 1374 self.warnings = warnings
1375 1375
1376 1376 if not(self.setNextFile()):
1377 1377 if (startDate != None) and (endDate != None):
1378 1378 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
1379 1379 elif startDate != None:
1380 1380 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
1381 1381 else:
1382 1382 print("[Reading] No files")
1383 1383
1384 1384 self.fileIndex = -1
1385 1385 self.pathList = []
1386 1386 self.filenameList = []
1387 1387 return
1388 1388
1389 1389 # self.getBasicHeader()
1390 1390
1391 1391 if last_set != None:
1392 1392 self.dataOut.last_block = last_set * \
1393 1393 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1394 1394 return
1395 1395
1396 1396 def getBasicHeader(self):
1397 1397
1398 1398 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1399 1399 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1400 1400
1401 1401 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1402 1402
1403 1403 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1404 1404
1405 1405 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1406 1406
1407 1407 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1408 1408
1409 1409 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1410 1410
1411 1411 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1412 1412
1413 1413 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1414 1414
1415 1415 def getFirstHeader(self):
1416 1416
1417 1417 raise NotImplementedError
1418 1418
1419 1419 def getData(self):
1420 1420
1421 1421 raise NotImplementedError
1422 1422
1423 1423 def hasNotDataInBuffer(self):
1424 1424
1425 1425 raise NotImplementedError
1426 1426
1427 1427 def readBlock(self):
1428 1428
1429 1429 raise NotImplementedError
1430 1430
1431 1431 def isEndProcess(self):
1432 1432
1433 1433 return self.flagNoMoreFiles
1434 1434
1435 1435 def printReadBlocks(self):
1436 1436
1437 1437 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1438 1438
1439 1439 def printTotalBlocks(self):
1440 1440
1441 1441 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1442 1442
1443 1443 def printNumberOfBlock(self):
1444 1444 'SPAM!'
1445 1445
1446 1446 # if self.flagIsNewBlock:
1447 1447 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1448 1448 # self.processingHeaderObj.dataBlocksPerFile,
1449 1449 # self.dataOut.datatime.ctime())
1450 1450
1451 1451 def printInfo(self):
1452 1452
1453 1453 if self.__printInfo == False:
1454 1454 return
1455 1455
1456 1456 self.basicHeaderObj.printInfo()
1457 1457 self.systemHeaderObj.printInfo()
1458 1458 self.radarControllerHeaderObj.printInfo()
1459 1459 self.processingHeaderObj.printInfo()
1460 1460
1461 1461 self.__printInfo = False
1462 1462
1463 1463 def run(self,
1464 1464 path=None,
1465 1465 startDate=None,
1466 1466 endDate=None,
1467 1467 startTime=datetime.time(0, 0, 0),
1468 1468 endTime=datetime.time(23, 59, 59),
1469 1469 set=None,
1470 1470 expLabel="",
1471 1471 ext=None,
1472 1472 online=False,
1473 1473 delay=60,
1474 1474 walk=True,
1475 1475 getblock=False,
1476 1476 nTxs=1,
1477 1477 realtime=False,
1478 1478 blocksize=None,
1479 1479 blocktime=None,
1480 1480 skip=None,
1481 1481 cursor=None,
1482 1482 warnings=True,
1483 1483 server=None,
1484 1484 verbose=True,
1485 1485 format=None,
1486 1486 oneDDict=None,
1487 1487 twoDDict=None,
1488 1488 ind2DList=None, **kwargs):
1489 1489
1490 1490 if not(self.isConfig):
1491 1491 self.setup(path=path,
1492 1492 startDate=startDate,
1493 1493 endDate=endDate,
1494 1494 startTime=startTime,
1495 1495 endTime=endTime,
1496 1496 set=set,
1497 1497 expLabel=expLabel,
1498 1498 ext=ext,
1499 1499 online=online,
1500 1500 delay=delay,
1501 1501 walk=walk,
1502 1502 getblock=getblock,
1503 1503 nTxs=nTxs,
1504 1504 realtime=realtime,
1505 1505 blocksize=blocksize,
1506 1506 blocktime=blocktime,
1507 1507 skip=skip,
1508 1508 cursor=cursor,
1509 1509 warnings=warnings,
1510 1510 server=server,
1511 1511 verbose=verbose,
1512 1512 format=format,
1513 1513 oneDDict=oneDDict,
1514 1514 twoDDict=twoDDict,
1515 1515 ind2DList=ind2DList)
1516 1516 self.isConfig = True
1517 1517 if server is None:
1518 1518 self.getData()
1519 1519 else:
1520 1520 self.getFromServer()
1521 1521
1522 1522
1523 1523 class JRODataWriter(JRODataIO):
1524 1524
1525 1525 """
1526 1526 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1527 1527 de los datos siempre se realiza por bloques.
1528 1528 """
1529 1529
1530 1530 blockIndex = 0
1531 1531
1532 1532 path = None
1533 1533
1534 1534 setFile = None
1535 1535
1536 1536 profilesPerBlock = None
1537 1537
1538 1538 blocksPerFile = None
1539 1539
1540 1540 nWriteBlocks = 0
1541 1541
1542 1542 fileDate = None
1543 1543
1544 1544 def __init__(self, dataOut=None):
1545 1545 raise NotImplementedError
1546 1546
1547 1547 def hasAllDataInBuffer(self):
1548 1548 raise NotImplementedError
1549 1549
1550 1550 def setBlockDimension(self):
1551 1551 raise NotImplementedError
1552 1552
1553 1553 def writeBlock(self):
1554 1554 raise NotImplementedError
1555 1555
1556 1556 def putData(self):
1557 1557 raise NotImplementedError
1558 1558
1559 1559 def getProcessFlags(self):
1560 1560
1561 1561 processFlags = 0
1562 1562
1563 1563 dtype_index = get_dtype_index(self.dtype)
1564 1564 procflag_dtype = get_procflag_dtype(dtype_index)
1565 1565
1566 1566 processFlags += procflag_dtype
1567 1567
1568 1568 if self.dataOut.flagDecodeData:
1569 1569 processFlags += PROCFLAG.DECODE_DATA
1570 1570
1571 1571 if self.dataOut.flagDeflipData:
1572 1572 processFlags += PROCFLAG.DEFLIP_DATA
1573 1573
1574 1574 if self.dataOut.code is not None:
1575 1575 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1576 1576
1577 1577 if self.dataOut.nCohInt > 1:
1578 1578 processFlags += PROCFLAG.COHERENT_INTEGRATION
1579 1579
1580 1580 if self.dataOut.type == "Spectra":
1581 1581 if self.dataOut.nIncohInt > 1:
1582 1582 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1583 1583
1584 1584 if self.dataOut.data_dc is not None:
1585 1585 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1586 1586
1587 1587 if self.dataOut.flagShiftFFT:
1588 1588 processFlags += PROCFLAG.SHIFT_FFT_DATA
1589 1589
1590 1590 return processFlags
1591 1591
1592 1592 def setBasicHeader(self):
1593 1593
1594 1594 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1595 1595 self.basicHeaderObj.version = self.versionFile
1596 1596 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1597 1597
1598 1598 utc = numpy.floor(self.dataOut.utctime)
1599 1599 milisecond = (self.dataOut.utctime - utc) * 1000.0
1600 1600
1601 1601 self.basicHeaderObj.utc = utc
1602 1602 self.basicHeaderObj.miliSecond = milisecond
1603 1603 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1604 1604 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1605 1605 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1606 1606
1607 1607 def setFirstHeader(self):
1608 1608 """
1609 1609 Obtiene una copia del First Header
1610 1610
1611 1611 Affected:
1612 1612
1613 1613 self.basicHeaderObj
1614 1614 self.systemHeaderObj
1615 1615 self.radarControllerHeaderObj
1616 1616 self.processingHeaderObj self.
1617 1617
1618 1618 Return:
1619 1619 None
1620 1620 """
1621 1621
1622 1622 raise NotImplementedError
1623 1623
1624 1624 def __writeFirstHeader(self):
1625 1625 """
1626 1626 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1627 1627
1628 1628 Affected:
1629 1629 __dataType
1630 1630
1631 1631 Return:
1632 1632 None
1633 1633 """
1634 1634
1635 1635 # CALCULAR PARAMETROS
1636 1636
1637 1637 sizeLongHeader = self.systemHeaderObj.size + \
1638 1638 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1639 1639 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1640 1640
1641 1641 self.basicHeaderObj.write(self.fp)
1642 1642 self.systemHeaderObj.write(self.fp)
1643 1643 self.radarControllerHeaderObj.write(self.fp)
1644 1644 self.processingHeaderObj.write(self.fp)
1645 1645
1646 1646 def __setNewBlock(self):
1647 1647 """
1648 1648 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1649 1649
1650 1650 Return:
1651 1651 0 : si no pudo escribir nada
1652 1652 1 : Si escribio el Basic el First Header
1653 1653 """
1654 1654 if self.fp == None:
1655 1655 self.setNextFile()
1656 1656
1657 1657 if self.flagIsNewFile:
1658 1658 return 1
1659 1659
1660 1660 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1661 1661 self.basicHeaderObj.write(self.fp)
1662 1662 return 1
1663 1663
1664 1664 if not(self.setNextFile()):
1665 1665 return 0
1666 1666
1667 1667 return 1
1668 1668
1669 1669 def writeNextBlock(self):
1670 1670 """
1671 1671 Selecciona el bloque siguiente de datos y los escribe en un file
1672 1672
1673 1673 Return:
1674 1674 0 : Si no hizo pudo escribir el bloque de datos
1675 1675 1 : Si no pudo escribir el bloque de datos
1676 1676 """
1677 1677 if not(self.__setNewBlock()):
1678 1678 return 0
1679 1679
1680 1680 self.writeBlock()
1681 1681
1682 1682 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1683 1683 self.processingHeaderObj.dataBlocksPerFile))
1684 1684
1685 1685 return 1
1686 1686
1687 1687 def setNextFile(self):
1688 1688 """
1689 1689 Determina el siguiente file que sera escrito
1690 1690
1691 1691 Affected:
1692 1692 self.filename
1693 1693 self.subfolder
1694 1694 self.fp
1695 1695 self.setFile
1696 1696 self.flagIsNewFile
1697 1697
1698 1698 Return:
1699 1699 0 : Si el archivo no puede ser escrito
1700 1700 1 : Si el archivo esta listo para ser escrito
1701 1701 """
1702 1702 ext = self.ext
1703 1703 path = self.path
1704 1704
1705 1705 if self.fp != None:
1706 1706 self.fp.close()
1707 1707
1708 1708 timeTuple = time.localtime(self.dataOut.utctime)
1709 1709 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1710 1710
1711 1711 fullpath = os.path.join(path, subfolder)
1712 1712 setFile = self.setFile
1713 1713
1714 1714 if not(os.path.exists(fullpath)):
1715 1715 os.mkdir(fullpath)
1716 1716 setFile = -1 # inicializo mi contador de seteo
1717 1717 else:
1718 1718 filesList = os.listdir(fullpath)
1719 1719 if len(filesList) > 0:
1720 1720 filesList = sorted(filesList, key=str.lower)
1721 1721 filen = filesList[-1]
1722 1722 # el filename debera tener el siguiente formato
1723 1723 # 0 1234 567 89A BCDE (hex)
1724 1724 # x YYYY DDD SSS .ext
1725 1725 if isNumber(filen[8:11]):
1726 1726 # inicializo mi contador de seteo al seteo del ultimo file
1727 1727 setFile = int(filen[8:11])
1728 1728 else:
1729 1729 setFile = -1
1730 1730 else:
1731 1731 setFile = -1 # inicializo mi contador de seteo
1732 1732
1733 1733 setFile += 1
1734 1734
1735 1735 # If this is a new day it resets some values
1736 1736 if self.dataOut.datatime.date() > self.fileDate:
1737 1737 setFile = 0
1738 1738 self.nTotalBlocks = 0
1739 1739
1740 1740 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1741 1741 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1742 1742
1743 1743 filename = os.path.join(path, subfolder, filen)
1744 1744
1745 1745 fp = open(filename, 'wb')
1746 1746
1747 1747 self.blockIndex = 0
1748 1748
1749 1749 # guardando atributos
1750 1750 self.filename = filename
1751 1751 self.subfolder = subfolder
1752 1752 self.fp = fp
1753 1753 self.setFile = setFile
1754 1754 self.flagIsNewFile = 1
1755 1755 self.fileDate = self.dataOut.datatime.date()
1756 1756
1757 1757 self.setFirstHeader()
1758 1758
1759 1759 print('[Writing] Opening file: %s' % self.filename)
1760 1760
1761 1761 self.__writeFirstHeader()
1762 1762
1763 1763 return 1
1764 1764
1765 1765 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1766 1766 """
1767 1767 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1768 1768
1769 1769 Inputs:
1770 1770 path : directory where data will be saved
1771 1771 profilesPerBlock : number of profiles per block
1772 1772 set : initial file set
1773 1773 datatype : An integer number that defines data type:
1774 1774 0 : int8 (1 byte)
1775 1775 1 : int16 (2 bytes)
1776 1776 2 : int32 (4 bytes)
1777 1777 3 : int64 (8 bytes)
1778 1778 4 : float32 (4 bytes)
1779 1779 5 : double64 (8 bytes)
1780 1780
1781 1781 Return:
1782 1782 0 : Si no realizo un buen seteo
1783 1783 1 : Si realizo un buen seteo
1784 1784 """
1785 1785
1786 1786 if ext == None:
1787 1787 ext = self.ext
1788 1788
1789 1789 self.ext = ext.lower()
1790 1790
1791 1791 self.path = path
1792 1792
1793 1793 if set is None:
1794 1794 self.setFile = -1
1795 1795 else:
1796 1796 self.setFile = set - 1
1797 1797
1798 1798 self.blocksPerFile = blocksPerFile
1799 1799
1800 1800 self.profilesPerBlock = profilesPerBlock
1801 1801
1802 1802 self.dataOut = dataOut
1803 1803 self.fileDate = self.dataOut.datatime.date()
1804 1804 # By default
1805 1805 self.dtype = self.dataOut.dtype
1806 1806
1807 1807 if datatype is not None:
1808 1808 self.dtype = get_numpy_dtype(datatype)
1809 1809
1810 1810 if not(self.setNextFile()):
1811 1811 print("[Writing] There isn't a next file")
1812 1812 return 0
1813 1813
1814 1814 self.setBlockDimension()
1815 1815
1816 1816 return 1
1817 1817
1818 1818 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1819 1819
1820 1820 if not(self.isConfig):
1821 1821
1822 1822 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1823 1823 set=set, ext=ext, datatype=datatype, **kwargs)
1824 1824 self.isConfig = True
1825 1825
1826 self.putData() No newline at end of file
1826 self.dataOut = dataOut
1827 self.putData()
1828 return self.dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1095 +1,1103
1 1 import numpy
2 2 import time
3 3 import os
4 4 import h5py
5 5 import re
6 6 import datetime
7 7
8 8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 10 # from .jroIO_base import *
11 11 from schainpy.model.io.jroIO_base import *
12 12 import schainpy
13 from schainpy.utils import log
13 14
14
15 class ParamReader(ProcessingUnit):
15 @MPDecorator
16 class ParamReader(JRODataReader,ProcessingUnit):
16 17 '''
17 18 Reads HDF5 format files
18 19
19 20 path
20 21
21 22 startDate
22 23
23 24 endDate
24 25
25 26 startTime
26 27
27 28 endTime
28 29 '''
29 30
30 31 ext = ".hdf5"
31 32
32 33 optchar = "D"
33 34
34 35 timezone = None
35 36
36 37 startTime = None
37 38
38 39 endTime = None
39 40
40 41 fileIndex = None
41 42
42 43 utcList = None #To select data in the utctime list
43 44
44 45 blockList = None #List to blocks to be read from the file
45 46
46 47 blocksPerFile = None #Number of blocks to be read
47 48
48 49 blockIndex = None
49 50
50 51 path = None
51 52
52 53 #List of Files
53 54
54 55 filenameList = None
55 56
56 57 datetimeList = None
57 58
58 59 #Hdf5 File
59 60
60 61 listMetaname = None
61 62
62 63 listMeta = None
63 64
64 65 listDataname = None
65 66
66 67 listData = None
67 68
68 69 listShapes = None
69 70
70 71 fp = None
71 72
72 73 #dataOut reconstruction
73 74
74 75 dataOut = None
75 76
76 77
77 def __init__(self, **kwargs):
78 ProcessingUnit.__init__(self, **kwargs)
78 def __init__(self):#, **kwargs):
79 ProcessingUnit.__init__(self) #, **kwargs)
79 80 self.dataOut = Parameters()
80 81 return
81 82
82 83 def setup(self, **kwargs):
83 84
84 85 path = kwargs['path']
85 86 startDate = kwargs['startDate']
86 87 endDate = kwargs['endDate']
87 88 startTime = kwargs['startTime']
88 89 endTime = kwargs['endTime']
89 90 walk = kwargs['walk']
90 91 if 'ext' in kwargs:
91 92 ext = kwargs['ext']
92 93 else:
93 94 ext = '.hdf5'
94 95 if 'timezone' in kwargs:
95 96 self.timezone = kwargs['timezone']
96 97 else:
97 98 self.timezone = 'lt'
98 99
99 100 print("[Reading] Searching files in offline mode ...")
100 101 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 102 startTime=startTime, endTime=endTime,
102 103 ext=ext, walk=walk)
103 104
104 105 if not(filenameList):
105 106 print("There is no files into the folder: %s"%(path))
106 107 sys.exit(-1)
107 108
108 109 self.fileIndex = -1
109 110 self.startTime = startTime
110 111 self.endTime = endTime
111 112
112 113 self.__readMetadata()
113 114
114 115 self.__setNextFileOffline()
115 116
116 117 return
117 118
118 119 def searchFilesOffLine(self,
119 120 path,
120 121 startDate=None,
121 122 endDate=None,
122 123 startTime=datetime.time(0,0,0),
123 124 endTime=datetime.time(23,59,59),
124 125 ext='.hdf5',
125 126 walk=True):
126 127
127 128 expLabel = ''
128 129 self.filenameList = []
129 130 self.datetimeList = []
130 131
131 132 pathList = []
132 133
133 134 JRODataObj = JRODataReader()
134 135 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
135 136
136 137 if dateList == []:
137 138 print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
138 139 datetime.datetime.combine(startDate,startTime).ctime(),
139 140 datetime.datetime.combine(endDate,endTime).ctime()))
140 141
141 142 return None, None
142 143
143 144 if len(dateList) > 1:
144 145 print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
145 146 else:
146 147 print("[Reading] data was found for the date %s" %(dateList[0]))
147 148
148 149 filenameList = []
149 150 datetimeList = []
150 151
151 152 #----------------------------------------------------------------------------------
152 153
153 154 for thisPath in pathList:
154 155 # thisPath = pathList[pathDict[file]]
155 156
156 157 fileList = glob.glob1(thisPath, "*%s" %ext)
157 158 fileList.sort()
158 159
159 160 for file in fileList:
160 161
161 162 filename = os.path.join(thisPath,file)
162 163
163 164 if not isFileInDateRange(filename, startDate, endDate):
164 165 continue
165 166
166 167 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
167 168
168 169 if not(thisDatetime):
169 170 continue
170 171
171 172 filenameList.append(filename)
172 173 datetimeList.append(thisDatetime)
173 174
174 175 if not(filenameList):
175 176 print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
176 177 return None, None
177 178
178 179 print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
179 180 print()
180 181
181 182 # for i in range(len(filenameList)):
182 183 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
183 184
184 185 self.filenameList = filenameList
185 186 self.datetimeList = datetimeList
186 187
187 188 return pathList, filenameList
188 189
189 190 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
190 191
191 192 """
192 193 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
193 194
194 195 Inputs:
195 196 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
196 197
197 198 startDate : fecha inicial del rango seleccionado en formato datetime.date
198 199
199 200 endDate : fecha final del rango seleccionado en formato datetime.date
200 201
201 202 startTime : tiempo inicial del rango seleccionado en formato datetime.time
202 203
203 204 endTime : tiempo final del rango seleccionado en formato datetime.time
204 205
205 206 Return:
206 207 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
207 208 fecha especificado, de lo contrario retorna False.
208 209
209 210 Excepciones:
210 211 Si el archivo no existe o no puede ser abierto
211 212 Si la cabecera no puede ser leida.
212 213
213 214 """
214 215
215 216 try:
216 217 fp = h5py.File(filename,'r')
217 218 grp1 = fp['Data']
218 219
219 220 except IOError:
220 221 traceback.print_exc()
221 222 raise IOError("The file %s can't be opened" %(filename))
222 223 #chino rata
223 224 #In case has utctime attribute
224 225 grp2 = grp1['utctime']
225 226 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
226 227 thisUtcTime = grp2.value[0]
227 228
228 229 fp.close()
229 230
230 231 if self.timezone == 'lt':
231 232 thisUtcTime -= 5*3600
232 233
233 234 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
234 235 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
235 236 thisDate = thisDatetime.date()
236 237 thisTime = thisDatetime.time()
237 238
238 239 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 240 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
240 241
241 242 #General case
242 243 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
243 244 #-----------o----------------------------o-----------
244 245 # startTime endTime
245 246
246 247 if endTime >= startTime:
247 248 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
248 249 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
249 250 return thisDatetime
250 251 return None
251 252
252 253 #If endTime < startTime then endTime belongs to the next day
253 254 #<<<<<<<<<<<o o>>>>>>>>>>>
254 255 #-----------o----------------------------o-----------
255 256 # endTime startTime
256 257
257 258 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
258 259 return None
259 260
260 261 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
261 262 return None
262 263
263 264 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
264 265 return None
265 266
266 267 return thisDatetime
267 268
268 269 def __setNextFileOffline(self):
269 270
270 271 self.fileIndex += 1
271 272 idFile = self.fileIndex
272 273
273 274 if not(idFile < len(self.filenameList)):
274 275 print("No more Files")
275 276 return 0
276 277
277 278 filename = self.filenameList[idFile]
278 279
279 280 filePointer = h5py.File(filename,'r')
280 281
281 282 self.filename = filename
282 283
283 284 self.fp = filePointer
284 285
285 286 print("Setting the file: %s"%self.filename)
286 287
287 288 # self.__readMetadata()
288 289 self.__setBlockList()
289 290 self.__readData()
290 291 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
291 292 # self.nRecords = self.fp['Data'].attrs['nRecords']
292 293 self.blockIndex = 0
293 294 return 1
294 295
295 296 def __setBlockList(self):
296 297 '''
297 298 Selects the data within the times defined
298 299
299 300 self.fp
300 301 self.startTime
301 302 self.endTime
302 303
303 304 self.blockList
304 305 self.blocksPerFile
305 306
306 307 '''
307 308 fp = self.fp
308 309 startTime = self.startTime
309 310 endTime = self.endTime
310 311
311 312 grp = fp['Data']
312 313 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
313 314
314 315 #ERROOOOR
315 316 if self.timezone == 'lt':
316 317 thisUtcTime -= 5*3600
317 318
318 319 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
319 320
320 321 thisDate = thisDatetime.date()
321 322 thisTime = thisDatetime.time()
322 323
323 324 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 325 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
325 326
326 327 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
327 328
328 329 self.blockList = ind
329 330 self.blocksPerFile = len(ind)
330 331
331 332 return
332 333
333 334 def __readMetadata(self):
334 335 '''
335 336 Reads Metadata
336 337
337 338 self.pathMeta
338 339
339 340 self.listShapes
340 341 self.listMetaname
341 342 self.listMeta
342 343
343 344 '''
344 345
345 346 # grp = self.fp['Data']
346 347 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
347 348 #
348 349 # if pathMeta == self.pathMeta:
349 350 # return
350 351 # else:
351 352 # self.pathMeta = pathMeta
352 353 #
353 354 # filePointer = h5py.File(self.pathMeta,'r')
354 355 # groupPointer = filePointer['Metadata']
355 356
356 357 filename = self.filenameList[0]
357 358
358 359 fp = h5py.File(filename,'r')
359 360
360 361 gp = fp['Metadata']
361 362
362 363 listMetaname = []
363 364 listMetadata = []
364 365 for item in list(gp.items()):
365 366 name = item[0]
366 367
367 368 if name=='array dimensions':
368 369 table = gp[name][:]
369 370 listShapes = {}
370 371 for shapes in table:
371 372 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
372 373 else:
373 374 data = gp[name].value
374 375 listMetaname.append(name)
375 376 listMetadata.append(data)
376 377
377 378 # if name=='type':
378 379 # self.__initDataOut(data)
379 380
380 381 self.listShapes = listShapes
381 382 self.listMetaname = listMetaname
382 383 self.listMeta = listMetadata
383 384
384 385 fp.close()
385 386 return
386 387
387 388 def __readData(self):
388 389 grp = self.fp['Data']
389 390 listdataname = []
390 391 listdata = []
391 392
392 393 for item in list(grp.items()):
393 394 name = item[0]
394 395 listdataname.append(name)
395 396
396 397 array = self.__setDataArray(grp[name],self.listShapes[name])
397 398 listdata.append(array)
398 399
399 400 self.listDataname = listdataname
400 401 self.listData = listdata
401 402 return
402 403
403 404 def __setDataArray(self, dataset, shapes):
404 405
405 406 nDims = shapes[0]
406 407
407 408 nDim2 = shapes[1] #Dimension 0
408 409
409 410 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
410 411
411 412 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
412 413
413 414 mode = shapes[4] #Mode of storing
414 415
415 416 blockList = self.blockList
416 417
417 418 blocksPerFile = self.blocksPerFile
418 419
419 420 #Depending on what mode the data was stored
420 421 if mode == 0: #Divided in channels
421 422 arrayData = dataset.value.astype(numpy.float)[0][blockList]
422 423 if mode == 1: #Divided in parameter
423 424 strds = 'table'
424 425 nDatas = nDim1
425 426 newShapes = (blocksPerFile,nDim2,nDim0)
426 427 elif mode==2: #Concatenated in a table
427 428 strds = 'table0'
428 429 arrayData = dataset[strds].value
429 430 #Selecting part of the dataset
430 431 utctime = arrayData[:,0]
431 432 u, indices = numpy.unique(utctime, return_index=True)
432 433
433 434 if blockList.size != indices.size:
434 435 indMin = indices[blockList[0]]
435 436 if blockList[1] + 1 >= indices.size:
436 437 arrayData = arrayData[indMin:,:]
437 438 else:
438 439 indMax = indices[blockList[1] + 1]
439 440 arrayData = arrayData[indMin:indMax,:]
440 441 return arrayData
441 442
442 443 # One dimension
443 444 if nDims == 0:
444 445 arrayData = dataset.value.astype(numpy.float)[0][blockList]
445 446
446 447 # Two dimensions
447 448 elif nDims == 2:
448 449 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
449 450 newShapes = (blocksPerFile,nDim0)
450 451 nDatas = nDim1
451 452
452 453 for i in range(nDatas):
453 454 data = dataset[strds + str(i)].value
454 455 arrayData[:,i,:] = data[blockList,:]
455 456
456 457 # Three dimensions
457 458 else:
458 459 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
459 460 for i in range(nDatas):
460 461
461 462 data = dataset[strds + str(i)].value
462 463
463 464 for b in range(blockList.size):
464 465 arrayData[b,:,i,:] = data[:,:,blockList[b]]
465 466
466 467 return arrayData
467 468
468 469 def __setDataOut(self):
469 470 listMeta = self.listMeta
470 471 listMetaname = self.listMetaname
471 472 listDataname = self.listDataname
472 473 listData = self.listData
473 474 listShapes = self.listShapes
474 475
475 476 blockIndex = self.blockIndex
476 477 # blockList = self.blockList
477 478
478 479 for i in range(len(listMeta)):
479 480 setattr(self.dataOut,listMetaname[i],listMeta[i])
480 481
481 482 for j in range(len(listData)):
482 483 nShapes = listShapes[listDataname[j]][0]
483 484 mode = listShapes[listDataname[j]][4]
484 485 if nShapes == 1:
485 486 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
486 487 elif nShapes > 1:
487 488 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
488 489 elif mode==0:
489 490 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
490 491 #Mode Meteors
491 492 elif mode ==2:
492 493 selectedData = self.__selectDataMode2(listData[j], blockIndex)
493 494 setattr(self.dataOut, listDataname[j], selectedData)
494 495 return
495 496
496 497 def __selectDataMode2(self, data, blockIndex):
497 498 utctime = data[:,0]
498 499 aux, indices = numpy.unique(utctime, return_inverse=True)
499 500 selInd = numpy.where(indices == blockIndex)[0]
500 501 selData = data[selInd,:]
501 502
502 503 return selData
503 504
504 505 def getData(self):
505 506
506 507 # if self.flagNoMoreFiles:
507 508 # self.dataOut.flagNoData = True
508 509 # print 'Process finished'
509 510 # return 0
510 511 #
511 512 if self.blockIndex==self.blocksPerFile:
512 513 if not( self.__setNextFileOffline() ):
513 514 self.dataOut.flagNoData = True
514 515 return 0
515 516
516 517 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
517 518 # self.dataOut.flagNoData = True
518 519 # return 0
519 520 # self.__readData()
520 521 self.__setDataOut()
521 522 self.dataOut.flagNoData = False
522 523
523 524 self.blockIndex += 1
524 525
525 526 return
526 527
527 528 def run(self, **kwargs):
528 529
529 530 if not(self.isConfig):
530 531 self.setup(**kwargs)
531 532 # self.setObjProperties()
532 533 self.isConfig = True
533 534
534 535 self.getData()
535 536
536 537 return
537
538 @MPDecorator
538 539 class ParamWriter(Operation):
539 540 '''
540 541 HDF5 Writer, stores parameters data in HDF5 format files
541 542
542 543 path: path where the files will be stored
543 544
544 545 blocksPerFile: number of blocks that will be saved in per HDF5 format file
545 546
546 547 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
547 548
548 549 metadataList: list of attributes that will be stored as metadata
549 550
550 551 dataList: list of attributes that will be stores as data
551 552
552 553 '''
553 554
554 555
555 556 ext = ".hdf5"
556 557
557 558 optchar = "D"
558 559
559 560 metaoptchar = "M"
560 561
561 562 metaFile = None
562 563
563 564 filename = None
564 565
565 566 path = None
566 567
567 568 setFile = None
568 569
569 570 fp = None
570 571
571 572 grp = None
572 573
573 574 ds = None
574 575
575 576 firsttime = True
576 577
577 578 #Configurations
578 579
579 580 blocksPerFile = None
580 581
581 582 blockIndex = None
582 583
583 584 dataOut = None
584 585
585 586 #Data Arrays
586 587
587 588 dataList = None
588 589
589 590 metadataList = None
590 591
591 592 # arrayDim = None
592 593
593 594 dsList = None #List of dictionaries with dataset properties
594 595
595 596 tableDim = None
596 597
597 598 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
598 599
599 600 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
600 601
601 602 currentDay = None
602 603
603 604 lastTime = None
604 605
605 def __init__(self, **kwargs):
606 Operation.__init__(self, **kwargs)
607 self.isConfig = False
606 def __init__(self):#, **kwargs):
607 Operation.__init__(self)#, **kwargs)
608 #self.isConfig = False
608 609 return
609 610
610 611 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
611 612 self.path = path
612 613 self.blocksPerFile = blocksPerFile
613 614 self.metadataList = metadataList
614 615 self.dataList = dataList
615 616 self.dataOut = dataOut
616 617 self.mode = mode
617
618 618 if self.mode is not None:
619 619 self.mode = numpy.zeros(len(self.dataList)) + mode
620 620 else:
621 #self.mode = numpy.ones(len(self.dataList),int)
621 622 self.mode = numpy.ones(len(self.dataList))
623 log.error(self.mode)#yong
622 624
623 625 arrayDim = numpy.zeros((len(self.dataList),5))
624 626
625 627 #Table dimensions
626 628 dtype0 = self.dtype
627 629 tableList = []
628 630
629 631 #Dictionary and list of tables
630 632 dsList = []
631 633
632 634 for i in range(len(self.dataList)):
633 635 dsDict = {}
634 636 dataAux = getattr(self.dataOut, self.dataList[i])
635 637 dsDict['variable'] = self.dataList[i]
636 638 #--------------------- Conditionals ------------------------
637 639 #There is no data
638 640 if dataAux is None:
639 641 return 0
640 642
641 643 #Not array, just a number
642 644 #Mode 0
645 #log.error(mode)#yong
646 #log.error(len(mode))#yong
647 #log.error(type(mode))#yong
643 648 if type(dataAux)==float or type(dataAux)==int:
644 649 dsDict['mode'] = 0
645 650 dsDict['nDim'] = 0
646 651 arrayDim[i,0] = 0
647 652 dsList.append(dsDict)
648 653
649 654 #Mode 2: meteors
650 elif mode[i] == 2:
655 elif self.mode[i] == 2:
651 656 # dsDict['nDim'] = 0
652 657 dsDict['dsName'] = 'table0'
653 658 dsDict['mode'] = 2 # Mode meteors
654 659 dsDict['shape'] = dataAux.shape[-1]
655 660 dsDict['nDim'] = 0
656 661 dsDict['dsNumber'] = 1
657 662
658 663 arrayDim[i,3] = dataAux.shape[-1]
659 arrayDim[i,4] = mode[i] #Mode the data was stored
664 arrayDim[i,4] = self.mode[i] #Mode the data was stored
660 665
661 666 dsList.append(dsDict)
662 667
663 668 #Mode 1
664 669 else:
665 670 arrayDim0 = dataAux.shape #Data dimensions
666 671 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
667 arrayDim[i,4] = mode[i] #Mode the data was stored
672 arrayDim[i,4] = self.mode[i] #Mode the data was stored
668 673
669 674 strtable = 'table'
670 675 dsDict['mode'] = 1 # Mode parameters
671 676
672 677 # Three-dimension arrays
673 678 if len(arrayDim0) == 3:
674 679 arrayDim[i,1:-1] = numpy.array(arrayDim0)
675 680 nTables = int(arrayDim[i,2])
676 681 dsDict['dsNumber'] = nTables
677 682 dsDict['shape'] = arrayDim[i,2:4]
678 683 dsDict['nDim'] = 3
679 684
680 685 for j in range(nTables):
681 686 dsDict = dsDict.copy()
682 687 dsDict['dsName'] = strtable + str(j)
683 688 dsList.append(dsDict)
684 689
685 690 # Two-dimension arrays
686 691 elif len(arrayDim0) == 2:
687 692 arrayDim[i,2:-1] = numpy.array(arrayDim0)
688 693 nTables = int(arrayDim[i,2])
689 694 dsDict['dsNumber'] = nTables
690 695 dsDict['shape'] = arrayDim[i,3]
691 696 dsDict['nDim'] = 2
692 697
693 698 for j in range(nTables):
694 699 dsDict = dsDict.copy()
695 700 dsDict['dsName'] = strtable + str(j)
696 701 dsList.append(dsDict)
697 702
698 703 # One-dimension arrays
699 704 elif len(arrayDim0) == 1:
700 705 arrayDim[i,3] = arrayDim0[0]
701 706 dsDict['shape'] = arrayDim0[0]
702 707 dsDict['dsNumber'] = 1
703 708 dsDict['dsName'] = strtable + str(0)
704 709 dsDict['nDim'] = 1
705 710 dsList.append(dsDict)
706 711
707 712 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
708 713 tableList.append(table)
709 714
710 715 # self.arrayDim = arrayDim
711 716 self.dsList = dsList
712 717 self.tableDim = numpy.array(tableList, dtype = dtype0)
713 718 self.blockIndex = 0
714 719
715 720 timeTuple = time.localtime(dataOut.utctime)
716 721 self.currentDay = timeTuple.tm_yday
717 722 return 1
718 723
719 724 def putMetadata(self):
720 725
721 726 fp = self.createMetadataFile()
722 727 self.writeMetadata(fp)
723 728 fp.close()
724 729 return
725 730
726 731 def createMetadataFile(self):
727 732 ext = self.ext
728 733 path = self.path
729 734 setFile = self.setFile
730 735
731 736 timeTuple = time.localtime(self.dataOut.utctime)
732 737
733 738 subfolder = ''
734 739 fullpath = os.path.join( path, subfolder )
735 740
736 741 if not( os.path.exists(fullpath) ):
737 742 os.mkdir(fullpath)
738 743 setFile = -1 #inicializo mi contador de seteo
739 744
740 745 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
741 746 fullpath = os.path.join( path, subfolder )
742 747
743 748 if not( os.path.exists(fullpath) ):
744 749 os.mkdir(fullpath)
745 750 setFile = -1 #inicializo mi contador de seteo
746 751
747 752 else:
748 753 filesList = os.listdir( fullpath )
749 754 filesList = sorted( filesList, key=str.lower )
750 755 if len( filesList ) > 0:
751 756 filesList = [k for k in filesList if 'M' in k]
752 757 filen = filesList[-1]
753 758 # el filename debera tener el siguiente formato
754 759 # 0 1234 567 89A BCDE (hex)
755 760 # x YYYY DDD SSS .ext
756 761 if isNumber( filen[8:11] ):
757 762 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
758 763 else:
759 764 setFile = -1
760 765 else:
761 766 setFile = -1 #inicializo mi contador de seteo
762 767
763 768 if self.setType is None:
764 769 setFile += 1
765 770 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
766 771 timeTuple.tm_year,
767 772 timeTuple.tm_yday,
768 773 setFile,
769 774 ext )
770 775 else:
771 776 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
772 777 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
773 778 timeTuple.tm_year,
774 779 timeTuple.tm_yday,
775 780 setFile,
776 781 ext )
777 782
778 783 filename = os.path.join( path, subfolder, file )
779 784 self.metaFile = file
780 785 #Setting HDF5 File
781 786 fp = h5py.File(filename,'w')
782 787
783 788 return fp
784 789
785 790 def writeMetadata(self, fp):
786 791
787 792 grp = fp.create_group("Metadata")
788 793 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
789 794
790 795 for i in range(len(self.metadataList)):
791 796 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
792 797 return
793 798
794 799 def timeFlag(self):
795 800 currentTime = self.dataOut.utctime
796 801
797 802 if self.lastTime is None:
798 803 self.lastTime = currentTime
799 804
800 805 #Day
801 806 timeTuple = time.localtime(currentTime)
802 807 dataDay = timeTuple.tm_yday
803 808
804 809 #Time
805 810 timeDiff = currentTime - self.lastTime
806 811
807 812 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
808 813 if dataDay != self.currentDay:
809 814 self.currentDay = dataDay
810 815 return True
811 816 elif timeDiff > 3*60*60:
812 817 self.lastTime = currentTime
813 818 return True
814 819 else:
815 820 self.lastTime = currentTime
816 821 return False
817 822
818 823 def setNextFile(self):
819 824
820 825 ext = self.ext
821 826 path = self.path
822 827 setFile = self.setFile
823 828 mode = self.mode
824 829
825 830 timeTuple = time.localtime(self.dataOut.utctime)
826 831 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
827 832
828 833 fullpath = os.path.join( path, subfolder )
829 834
830 835 if os.path.exists(fullpath):
831 836 filesList = os.listdir( fullpath )
832 837 filesList = [k for k in filesList if 'D' in k]
833 838 if len( filesList ) > 0:
834 839 filesList = sorted( filesList, key=str.lower )
835 840 filen = filesList[-1]
836 841 # el filename debera tener el siguiente formato
837 842 # 0 1234 567 89A BCDE (hex)
838 843 # x YYYY DDD SSS .ext
839 844 if isNumber( filen[8:11] ):
840 845 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
841 846 else:
842 847 setFile = -1
843 848 else:
844 849 setFile = -1 #inicializo mi contador de seteo
845 850 else:
846 851 os.makedirs(fullpath)
847 852 setFile = -1 #inicializo mi contador de seteo
848 853
849 if self.setType is None:
854 if None is None:
850 855 setFile += 1
851 856 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
852 857 timeTuple.tm_year,
853 858 timeTuple.tm_yday,
854 859 setFile,
855 860 ext )
856 861 else:
857 862 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
858 863 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
859 864 timeTuple.tm_year,
860 865 timeTuple.tm_yday,
861 866 setFile,
862 867 ext )
863 868
864 869 filename = os.path.join( path, subfolder, file )
865 870
866 871 #Setting HDF5 File
867 872 fp = h5py.File(filename,'w')
868 873 #write metadata
869 874 self.writeMetadata(fp)
870 875 #Write data
871 876 grp = fp.create_group("Data")
872 877 # grp.attrs['metadata'] = self.metaFile
873 878
874 879 # grp.attrs['blocksPerFile'] = 0
875 880 ds = []
876 881 data = []
877 882 dsList = self.dsList
878 883 i = 0
879 884 while i < len(dsList):
880 885 dsInfo = dsList[i]
881 886 #One-dimension data
882 887 if dsInfo['mode'] == 0:
883 888 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
884 889 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
885 890 ds.append(ds0)
886 891 data.append([])
887 892 i += 1
888 893 continue
889 894 # nDimsForDs.append(nDims[i])
890 895
891 896 elif dsInfo['mode'] == 2:
892 897 grp0 = grp.create_group(dsInfo['variable'])
893 898 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
894 899 ds.append(ds0)
895 900 data.append([])
896 901 i += 1
897 902 continue
898 903
899 904 elif dsInfo['mode'] == 1:
900 905 grp0 = grp.create_group(dsInfo['variable'])
901 906
902 907 for j in range(dsInfo['dsNumber']):
903 908 dsInfo = dsList[i]
904 909 tableName = dsInfo['dsName']
905 shape = int(dsInfo['shape'])
910
906 911
907 912 if dsInfo['nDim'] == 3:
913 shape = dsInfo['shape'].astype(int)
908 914 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
909 915 else:
916 shape = int(dsInfo['shape'])
910 917 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
911 918
912 919 ds.append(ds0)
913 920 data.append([])
914 921 i += 1
915 922 # nDimsForDs.append(nDims[i])
916 923
917 924 fp.flush()
918 925 fp.close()
919 926
920 927 # self.nDatas = nDatas
921 928 # self.nDims = nDims
922 929 # self.nDimsForDs = nDimsForDs
923 930 #Saving variables
924 931 print('Writing the file: %s'%filename)
925 932 self.filename = filename
926 933 # self.fp = fp
927 934 # self.grp = grp
928 935 # self.grp.attrs.modify('nRecords', 1)
929 936 self.ds = ds
930 937 self.data = data
931 938 # self.setFile = setFile
932 939 self.firsttime = True
933 940 self.blockIndex = 0
934 941 return
935 942
936 943 def putData(self):
937 944
938 945 if self.blockIndex == self.blocksPerFile or self.timeFlag():
939 946 self.setNextFile()
940 947
941 948 # if not self.firsttime:
942 949 self.readBlock()
943 950 self.setBlock() #Prepare data to be written
944 951 self.writeBlock() #Write data
945 952
946 953 return
947 954
948 955 def readBlock(self):
949 956
950 957 '''
951 958 data Array configured
952 959
953 960
954 961 self.data
955 962 '''
956 963 dsList = self.dsList
957 964 ds = self.ds
958 965 #Setting HDF5 File
959 966 fp = h5py.File(self.filename,'r+')
960 967 grp = fp["Data"]
961 968 ind = 0
962 969
963 970 # grp.attrs['blocksPerFile'] = 0
964 971 while ind < len(dsList):
965 972 dsInfo = dsList[ind]
966 973
967 974 if dsInfo['mode'] == 0:
968 975 ds0 = grp[dsInfo['variable']]
969 976 ds[ind] = ds0
970 977 ind += 1
971 978 else:
972 979
973 980 grp0 = grp[dsInfo['variable']]
974 981
975 982 for j in range(dsInfo['dsNumber']):
976 983 dsInfo = dsList[ind]
977 984 ds0 = grp0[dsInfo['dsName']]
978 985 ds[ind] = ds0
979 986 ind += 1
980 987
981 988 self.fp = fp
982 989 self.grp = grp
983 990 self.ds = ds
984 991
985 992 return
986 993
987 994 def setBlock(self):
988 995 '''
989 996 data Array configured
990 997
991 998
992 999 self.data
993 1000 '''
994 1001 #Creating Arrays
995 1002 dsList = self.dsList
996 1003 data = self.data
997 1004 ind = 0
998 1005
999 1006 while ind < len(dsList):
1000 1007 dsInfo = dsList[ind]
1001 1008 dataAux = getattr(self.dataOut, dsInfo['variable'])
1002 1009
1003 1010 mode = dsInfo['mode']
1004 1011 nDim = dsInfo['nDim']
1005 1012
1006 1013 if mode == 0 or mode == 2 or nDim == 1:
1007 1014 data[ind] = dataAux
1008 1015 ind += 1
1009 1016 # elif nDim == 1:
1010 1017 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1011 1018 # ind += 1
1012 1019 elif nDim == 2:
1013 1020 for j in range(dsInfo['dsNumber']):
1014 1021 data[ind] = dataAux[j,:]
1015 1022 ind += 1
1016 1023 elif nDim == 3:
1017 1024 for j in range(dsInfo['dsNumber']):
1018 1025 data[ind] = dataAux[:,j,:]
1019 1026 ind += 1
1020 1027
1021 1028 self.data = data
1022 1029 return
1023 1030
1024 1031 def writeBlock(self):
1025 1032 '''
1026 1033 Saves the block in the HDF5 file
1027 1034 '''
1028 1035 dsList = self.dsList
1029 1036
1030 1037 for i in range(len(self.ds)):
1031 1038 dsInfo = dsList[i]
1032 1039 nDim = dsInfo['nDim']
1033 1040 mode = dsInfo['mode']
1034 1041
1035 1042 # First time
1036 1043 if self.firsttime:
1037 1044 # self.ds[i].resize(self.data[i].shape)
1038 1045 # self.ds[i][self.blockIndex,:] = self.data[i]
1039 1046 if type(self.data[i]) == numpy.ndarray:
1040 1047
1041 1048 if nDim == 3:
1042 1049 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1043 1050 self.ds[i].resize(self.data[i].shape)
1044 1051 if mode == 2:
1045 1052 self.ds[i].resize(self.data[i].shape)
1046 1053 self.ds[i][:] = self.data[i]
1047 1054 else:
1048 1055
1049 1056 # From second time
1050 1057 # Meteors!
1051 1058 if mode == 2:
1052 1059 dataShape = self.data[i].shape
1053 1060 dsShape = self.ds[i].shape
1054 1061 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1055 1062 self.ds[i][dsShape[0]:,:] = self.data[i]
1056 1063 # No dimension
1057 1064 elif mode == 0:
1058 1065 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1059 1066 self.ds[i][0,-1] = self.data[i]
1060 1067 # One dimension
1061 1068 elif nDim == 1:
1062 1069 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1063 1070 self.ds[i][-1,:] = self.data[i]
1064 1071 # Two dimension
1065 1072 elif nDim == 2:
1066 1073 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1067 1074 self.ds[i][self.blockIndex,:] = self.data[i]
1068 1075 # Three dimensions
1069 1076 elif nDim == 3:
1070 1077 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1071 1078 self.ds[i][:,:,-1] = self.data[i]
1072 1079
1073 1080 self.firsttime = False
1074 1081 self.blockIndex += 1
1075 1082
1076 1083 #Close to save changes
1077 1084 self.fp.flush()
1078 1085 self.fp.close()
1079 1086 return
1080 1087
1081 def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1088 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1082 1089
1083 1090 if not(self.isConfig):
1084 1091 flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1085 1092 metadataList=metadataList, dataList=dataList, mode=mode, **kwargs)
1086 1093
1087 1094 if not(flagdata):
1088 1095 return
1089 1096
1090 1097 self.isConfig = True
1091 1098 # self.putMetadata()
1092 1099 self.setNextFile()
1093 1100
1094 1101 self.putData()
1095 return No newline at end of file
1102 return
1103 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,680 +1,679
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import numpy
7 7
8 8 from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 10 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 11 from schainpy.model.data.jrodata import Spectra
12 from schainpy.utils import log
12 13
13 14 @MPDecorator
14 15 class SpectraReader(JRODataReader, ProcessingUnit):
15 16 """
16 17 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
17 18 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
18 19 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
19 20
20 21 paresCanalesIguales * alturas * perfiles (Self Spectra)
21 22 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
22 23 canales * alturas (DC Channels)
23 24
24 25 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
25 26 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
26 27 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
27 28 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
28 29
29 30 Example:
30 31 dpath = "/home/myuser/data"
31 32
32 33 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
33 34
34 35 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
35 36
36 37 readerObj = SpectraReader()
37 38
38 39 readerObj.setup(dpath, startTime, endTime)
39 40
40 41 while(True):
41 42
42 43 readerObj.getData()
43 44
44 45 print readerObj.data_spc
45 46
46 47 print readerObj.data_cspc
47 48
48 49 print readerObj.data_dc
49 50
50 51 if readerObj.flagNoMoreFiles:
51 52 break
52 53
53 54 """
54 55
55 56 pts2read_SelfSpectra = 0
56 57
57 58 pts2read_CrossSpectra = 0
58 59
59 60 pts2read_DCchannels = 0
60 61
61 62 ext = ".pdata"
62 63
63 64 optchar = "P"
64 65
65 66 dataOut = None
66 67
67 68 nRdChannels = None
68 69
69 70 nRdPairs = None
70 71
71 72 rdPairList = []
72 73
73 74 def __init__(self):#, **kwargs):
74 75 """
75 76 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
76 77
77 78 Inputs:
78 79 dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para
79 80 almacenar un perfil de datos cada vez que se haga un requerimiento
80 81 (getData). El perfil sera obtenido a partir del buffer de datos,
81 82 si el buffer esta vacio se hara un nuevo proceso de lectura de un
82 83 bloque de datos.
83 84 Si este parametro no es pasado se creara uno internamente.
84 85
85 86 Affected:
86 87 self.dataOut
87 88
88 89 Return : None
89 90 """
90 91
91 92 #Eliminar de la base la herencia
92 93 ProcessingUnit.__init__(self)#, **kwargs)
93 94
94 95 # self.isConfig = False
95 96
96 97 self.pts2read_SelfSpectra = 0
97 98
98 99 self.pts2read_CrossSpectra = 0
99 100
100 101 self.pts2read_DCchannels = 0
101 102
102 103 self.datablock = None
103 104
104 105 self.utc = None
105 106
106 107 self.ext = ".pdata"
107 108
108 109 self.optchar = "P"
109 110
110 111 self.basicHeaderObj = BasicHeader(LOCALTIME)
111 112
112 113 self.systemHeaderObj = SystemHeader()
113 114
114 115 self.radarControllerHeaderObj = RadarControllerHeader()
115 116
116 117 self.processingHeaderObj = ProcessingHeader()
117 118
118 119 self.online = 0
119 120
120 121 self.fp = None
121 122
122 123 self.idFile = None
123 124
124 125 self.dtype = None
125 126
126 127 self.fileSizeByHeader = None
127 128
128 129 self.filenameList = []
129 130
130 131 self.filename = None
131 132
132 133 self.fileSize = None
133 134
134 135 self.firstHeaderSize = 0
135 136
136 137 self.basicHeaderSize = 24
137 138
138 139 self.pathList = []
139 140
140 141 self.lastUTTime = 0
141 142
142 143 self.maxTimeStep = 30
143 144
144 145 self.flagNoMoreFiles = 0
145 146
146 147 self.set = 0
147 148
148 149 self.path = None
149 150
150 151 self.delay = 60 #seconds
151 152
152 153 self.nTries = 3 #quantity tries
153 154
154 155 self.nFiles = 3 #number of files for searching
155 156
156 157 self.nReadBlocks = 0
157 158
158 159 self.flagIsNewFile = 1
159 160
160 161 self.__isFirstTimeOnline = 1
161 162
162 163 # self.ippSeconds = 0
163 164
164 165 self.flagDiscontinuousBlock = 0
165 166
166 167 self.flagIsNewBlock = 0
167 168
168 169 self.nTotalBlocks = 0
169 170
170 171 self.blocksize = 0
171 172
172 173 self.dataOut = self.createObjByDefault()
173 174
174 175 self.profileIndex = 1 #Always
175 176
176 177
177 178 def createObjByDefault(self):
178 179
179 180 dataObj = Spectra()
180 181
181 182 return dataObj
182 183
183 184 def __hasNotDataInBuffer(self):
184 185 return 1
185 186
186 187
187 188 def getBlockDimension(self):
188 189 """
189 190 Obtiene la cantidad de puntos a leer por cada bloque de datos
190 191
191 192 Affected:
192 193 self.nRdChannels
193 194 self.nRdPairs
194 195 self.pts2read_SelfSpectra
195 196 self.pts2read_CrossSpectra
196 197 self.pts2read_DCchannels
197 198 self.blocksize
198 199 self.dataOut.nChannels
199 200 self.dataOut.nPairs
200 201
201 202 Return:
202 203 None
203 204 """
204 205 self.nRdChannels = 0
205 206 self.nRdPairs = 0
206 207 self.rdPairList = []
207 208
208 209 for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
209 210 if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
210 211 self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
211 212 else:
212 213 self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
213 214 self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
214 215
215 216 pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
216 217
217 218 self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
218 219 self.blocksize = self.pts2read_SelfSpectra
219 220
220 221 if self.processingHeaderObj.flag_cspc:
221 222 self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
222 223 self.blocksize += self.pts2read_CrossSpectra
223 224
224 225 if self.processingHeaderObj.flag_dc:
225 226 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
226 227 self.blocksize += self.pts2read_DCchannels
227 228
228 229 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
229 230
230 231
231 232 def readBlock(self):
232 233 """
233 234 Lee el bloque de datos desde la posicion actual del puntero del archivo
234 235 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
235 236 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
236 237 es seteado a 0
237 238
238 239 Return: None
239 240
240 241 Variables afectadas:
241 242
242 243 self.flagIsNewFile
243 244 self.flagIsNewBlock
244 245 self.nTotalBlocks
245 246 self.data_spc
246 247 self.data_cspc
247 248 self.data_dc
248 249
249 250 Exceptions:
250 251 Si un bloque leido no es un bloque valido
251 252 """
252 253 blockOk_flag = False
253 254 fpointer = self.fp.tell()
254 255
255 256 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
256 257 spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
257 258
258 259 if self.processingHeaderObj.flag_cspc:
259 260 cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
260 261 cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
261 262
262 263 if self.processingHeaderObj.flag_dc:
263 264 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
264 265 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
265 266
266 267
267 268 if not self.processingHeaderObj.shif_fft:
268 269 #desplaza a la derecha en el eje 2 determinadas posiciones
269 270 shift = int(self.processingHeaderObj.profilesPerBlock/2)
270 271 spc = numpy.roll( spc, shift , axis=2 )
271 272
272 273 if self.processingHeaderObj.flag_cspc:
273 274 #desplaza a la derecha en el eje 2 determinadas posiciones
274 275 cspc = numpy.roll( cspc, shift, axis=2 )
275 276
276 277 #Dimensions : nChannels, nProfiles, nSamples
277 278 spc = numpy.transpose( spc, (0,2,1) )
278 279 self.data_spc = spc
279 280
280 281 if self.processingHeaderObj.flag_cspc:
281 282 cspc = numpy.transpose( cspc, (0,2,1) )
282 283 self.data_cspc = cspc['real'] + cspc['imag']*1j
283 284 else:
284 285 self.data_cspc = None
285 286
286 287 if self.processingHeaderObj.flag_dc:
287 288 self.data_dc = dc['real'] + dc['imag']*1j
288 289 else:
289 290 self.data_dc = None
290 291
291 292 self.flagIsNewFile = 0
292 293 self.flagIsNewBlock = 1
293 294
294 295 self.nTotalBlocks += 1
295 296 self.nReadBlocks += 1
296 297
297 298 return 1
298 299
299 300 def getFirstHeader(self):
300 301
301 302 self.getBasicHeader()
302 303
303 304 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
304 305
305 306 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
306 307
307 308 # self.dataOut.ippSeconds = self.ippSeconds
308 309
309 310 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.processingHeaderObj.profilesPerBlock
310 311
311 312 self.dataOut.dtype = self.dtype
312 313
313 314 # self.dataOut.nPairs = self.nPairs
314 315
315 316 self.dataOut.pairsList = self.rdPairList
316 317
317 318 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
318 319
319 320 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
320 321
321 322 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
322 323
323 324 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
324 325
325 326 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
326 327
327 328 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
328 329
329 330 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
330 331
331 332 self.dataOut.flagShiftFFT = True #Data is always shifted
332 333
333 334 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
334 335
335 336 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
336 337
337 338 def getData(self):
338 339 """
339 340 First method to execute before "RUN" is called.
340 341
341 342 Copia el buffer de lectura a la clase "Spectra",
342 343 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
343 344 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
344 345
345 346 Return:
346 347 0 : Si no hay mas archivos disponibles
347 348 1 : Si hizo una buena copia del buffer
348 349
349 350 Affected:
350 351 self.dataOut
351 352
352 353 self.flagDiscontinuousBlock
353 354 self.flagIsNewBlock
354 355 """
355 356
356 357 if self.flagNoMoreFiles:
357 358 self.dataOut.flagNoData = True
358 359 print('Process finished')
359 360 return 0
360 361
361 362 self.flagDiscontinuousBlock = 0
362 363 self.flagIsNewBlock = 0
363 364
364 365 if self.__hasNotDataInBuffer():
365 366
366 367 if not( self.readNextBlock() ):
367 368 self.dataOut.flagNoData = True
368 369 return 0
369 370
370 371 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
371 372
372 373 if self.data_spc is None:
373 374 self.dataOut.flagNoData = True
374 375 return 0
375 376
376 377 self.getBasicHeader()
377 378
378 379 self.getFirstHeader()
379 380
380 381 self.dataOut.data_spc = self.data_spc
381 382
382 383 self.dataOut.data_cspc = self.data_cspc
383 384
384 385 self.dataOut.data_dc = self.data_dc
385 386
386 387 self.dataOut.flagNoData = False
387 388
388 389 self.dataOut.realtime = self.online
389 390
390 391 return self.dataOut.data_spc
391
392 @MPDecorator
392 393 class SpectraWriter(JRODataWriter, Operation):
393 394
394 395 """
395 396 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
396 397 de los datos siempre se realiza por bloques.
397 398 """
398 399
399 400 ext = ".pdata"
400 401
401 402 optchar = "P"
402 403
403 404 shape_spc_Buffer = None
404 405
405 406 shape_cspc_Buffer = None
406 407
407 408 shape_dc_Buffer = None
408 409
409 410 data_spc = None
410 411
411 412 data_cspc = None
412 413
413 414 data_dc = None
414 415
415 416 # dataOut = None
416 417
417 def __init__(self, **kwargs):
418 def __init__(self):#, **kwargs):
418 419 """
419 420 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
420 421
421 422 Affected:
422 423 self.dataOut
423 424 self.basicHeaderObj
424 425 self.systemHeaderObj
425 426 self.radarControllerHeaderObj
426 427 self.processingHeaderObj
427 428
428 429 Return: None
429 430 """
430 431
431 Operation.__init__(self, **kwargs)
432 Operation.__init__(self)#, **kwargs)
432 433
433 self.isConfig = False
434 #self.isConfig = False
434 435
435 436 self.nTotalBlocks = 0
436 437
437 438 self.data_spc = None
438 439
439 440 self.data_cspc = None
440 441
441 442 self.data_dc = None
442 443
443 444 self.fp = None
444 445
445 446 self.flagIsNewFile = 1
446 447
447 448 self.nTotalBlocks = 0
448 449
449 450 self.flagIsNewBlock = 0
450 451
451 452 self.setFile = None
452 453
453 454 self.dtype = None
454 455
455 456 self.path = None
456 457
457 458 self.noMoreFiles = 0
458 459
459 460 self.filename = None
460 461
461 462 self.basicHeaderObj = BasicHeader(LOCALTIME)
462 463
463 464 self.systemHeaderObj = SystemHeader()
464 465
465 466 self.radarControllerHeaderObj = RadarControllerHeader()
466 467
467 468 self.processingHeaderObj = ProcessingHeader()
468 469
469 470
470 471 def hasAllDataInBuffer(self):
471 472 return 1
472 473
473 474
474 475 def setBlockDimension(self):
475 476 """
476 477 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
477 478
478 479 Affected:
479 480 self.shape_spc_Buffer
480 481 self.shape_cspc_Buffer
481 482 self.shape_dc_Buffer
482 483
483 484 Return: None
484 485 """
485 486 self.shape_spc_Buffer = (self.dataOut.nChannels,
486 487 self.processingHeaderObj.nHeights,
487 488 self.processingHeaderObj.profilesPerBlock)
488 489
489 490 self.shape_cspc_Buffer = (self.dataOut.nPairs,
490 491 self.processingHeaderObj.nHeights,
491 492 self.processingHeaderObj.profilesPerBlock)
492 493
493 494 self.shape_dc_Buffer = (self.dataOut.nChannels,
494 495 self.processingHeaderObj.nHeights)
495 496
496 497
497 498 def writeBlock(self):
498 499 """
499 500 Escribe el buffer en el file designado
500 501
501 502 Affected:
502 503 self.data_spc
503 504 self.data_cspc
504 505 self.data_dc
505 506 self.flagIsNewFile
506 507 self.flagIsNewBlock
507 508 self.nTotalBlocks
508 509 self.nWriteBlocks
509 510
510 511 Return: None
511 512 """
512 513
513 514 spc = numpy.transpose( self.data_spc, (0,2,1) )
514 515 if not self.processingHeaderObj.shif_fft:
515 516 spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
516 517 data = spc.reshape((-1))
517 518 data = data.astype(self.dtype[0])
518 519 data.tofile(self.fp)
519 520
520 521 if self.data_cspc is not None:
521 522 data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
522 523 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
523 524 if not self.processingHeaderObj.shif_fft:
524 525 cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
525 526 data['real'] = cspc.real
526 527 data['imag'] = cspc.imag
527 528 data = data.reshape((-1))
528 529 data.tofile(self.fp)
529 530
530 531 if self.data_dc is not None:
531 532 data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
532 533 dc = self.data_dc
533 534 data['real'] = dc.real
534 535 data['imag'] = dc.imag
535 536 data = data.reshape((-1))
536 537 data.tofile(self.fp)
537 538
538 539 # self.data_spc.fill(0)
539 540 #
540 541 # if self.data_dc is not None:
541 542 # self.data_dc.fill(0)
542 543 #
543 544 # if self.data_cspc is not None:
544 545 # self.data_cspc.fill(0)
545 546
546 547 self.flagIsNewFile = 0
547 548 self.flagIsNewBlock = 1
548 549 self.nTotalBlocks += 1
549 550 self.nWriteBlocks += 1
550 551 self.blockIndex += 1
551 552
552 553 # print "[Writing] Block = %d04" %self.blockIndex
553 554
554 555 def putData(self):
555 556 """
556 557 Setea un bloque de datos y luego los escribe en un file
557 558
558 559 Affected:
559 560 self.data_spc
560 561 self.data_cspc
561 562 self.data_dc
562 563
563 564 Return:
564 565 0 : Si no hay data o no hay mas files que puedan escribirse
565 566 1 : Si se escribio la data de un bloque en un file
566 567 """
567 568
568 569 if self.dataOut.flagNoData:
569 570 return 0
570 571
571 572 self.flagIsNewBlock = 0
572 573
573 574 if self.dataOut.flagDiscontinuousBlock:
574 575 self.data_spc.fill(0)
575 576 if self.dataOut.data_cspc is not None:
576 577 self.data_cspc.fill(0)
577 578 if self.dataOut.data_dc is not None:
578 579 self.data_dc.fill(0)
579 580 self.setNextFile()
580 581
581 582 if self.flagIsNewFile == 0:
582 583 self.setBasicHeader()
583 584
584 585 self.data_spc = self.dataOut.data_spc.copy()
585 586
586 587 if self.dataOut.data_cspc is not None:
587 588 self.data_cspc = self.dataOut.data_cspc.copy()
588 589
589 590 if self.dataOut.data_dc is not None:
590 591 self.data_dc = self.dataOut.data_dc.copy()
591 592
592 593 # #self.processingHeaderObj.dataBlocksPerFile)
593 594 if self.hasAllDataInBuffer():
594 595 # self.setFirstHeader()
595 596 self.writeNextBlock()
596 597
597 return 1
598
599 598 def __getBlockSize(self):
600 599 '''
601 600 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
602 601 '''
603 602
604 603 dtype_width = self.getDtypeWidth()
605 604
606 605 pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
607 606
608 607 pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
609 608 blocksize = (pts2write_SelfSpectra*dtype_width)
610 609
611 610 if self.dataOut.data_cspc is not None:
612 611 pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
613 612 blocksize += (pts2write_CrossSpectra*dtype_width*2)
614 613
615 614 if self.dataOut.data_dc is not None:
616 615 pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
617 616 blocksize += (pts2write_DCchannels*dtype_width*2)
618 617
619 618 # blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
620 619
621 620 return blocksize
622 621
623 622 def setFirstHeader(self):
624 623
625 624 """
626 625 Obtiene una copia del First Header
627 626
628 627 Affected:
629 628 self.systemHeaderObj
630 629 self.radarControllerHeaderObj
631 630 self.dtype
632 631
633 632 Return:
634 633 None
635 634 """
636 635
637 636 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
638 637 self.systemHeaderObj.nChannels = self.dataOut.nChannels
639 638 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
640 639
641 640 self.processingHeaderObj.dtype = 1 # Spectra
642 641 self.processingHeaderObj.blockSize = self.__getBlockSize()
643 642 self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
644 643 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
645 644 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
646 645 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
647 646 self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
648 647 self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
649 648 self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
650 649
651 650 if self.processingHeaderObj.totalSpectra > 0:
652 651 channelList = []
653 652 for channel in range(self.dataOut.nChannels):
654 653 channelList.append(channel)
655 654 channelList.append(channel)
656 655
657 656 pairsList = []
658 657 if self.dataOut.nPairs > 0:
659 658 for pair in self.dataOut.pairsList:
660 659 pairsList.append(pair[0])
661 660 pairsList.append(pair[1])
662 661
663 662 spectraComb = channelList + pairsList
664 663 spectraComb = numpy.array(spectraComb, dtype="u1")
665 664 self.processingHeaderObj.spectraComb = spectraComb
666 665
667 666 if self.dataOut.code is not None:
668 667 self.processingHeaderObj.code = self.dataOut.code
669 668 self.processingHeaderObj.nCode = self.dataOut.nCode
670 669 self.processingHeaderObj.nBaud = self.dataOut.nBaud
671 670
672 671 if self.processingHeaderObj.nWindows != 0:
673 672 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
674 673 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
675 674 self.processingHeaderObj.nHeights = self.dataOut.nHeights
676 675 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
677 676
678 677 self.processingHeaderObj.processFlags = self.getProcessFlags()
679 678
680 679 self.setBasicHeader() No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,765 +1,765
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6
7 7 import numpy
8 8
9 9 from .jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 12 from schainpy.model.data.jrodata import Voltage
13 13 import zmq
14 14 import tempfile
15 15 from io import StringIO
16 16 # from _sha import blocksize
17 17
18 18 @MPDecorator
19 19 class VoltageReader(JRODataReader, ProcessingUnit):
20 20 """
21 21 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
22 22 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
23 23 perfiles*alturas*canales) son almacenados en la variable "buffer".
24 24
25 25 perfiles * alturas * canales
26 26
27 27 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
28 28 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
29 29 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
30 30 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
31 31
32 32 Example:
33 33
34 34 dpath = "/home/myuser/data"
35 35
36 36 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
37 37
38 38 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
39 39
40 40 readerObj = VoltageReader()
41 41
42 42 readerObj.setup(dpath, startTime, endTime)
43 43
44 44 while(True):
45 45
46 46 #to get one profile
47 47 profile = readerObj.getData()
48 48
49 49 #print the profile
50 50 print profile
51 51
52 52 #If you want to see all datablock
53 53 print readerObj.datablock
54 54
55 55 if readerObj.flagNoMoreFiles:
56 56 break
57 57
58 58 """
59 59
60 60 ext = ".r"
61 61
62 62 optchar = "D"
63 63 dataOut = None
64 64
65 65 def __init__(self):#, **kwargs):
66 66 """
67 67 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
68 68
69 69 Input:
70 70 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
71 71 almacenar un perfil de datos cada vez que se haga un requerimiento
72 72 (getData). El perfil sera obtenido a partir del buffer de datos,
73 73 si el buffer esta vacio se hara un nuevo proceso de lectura de un
74 74 bloque de datos.
75 75 Si este parametro no es pasado se creara uno internamente.
76 76
77 77 Variables afectadas:
78 78 self.dataOut
79 79
80 80 Return:
81 81 None
82 82 """
83 83
84 84 ProcessingUnit.__init__(self)#, **kwargs)
85 85
86 86 self.isConfig = False
87 87
88 88 self.datablock = None
89 89
90 90 self.utc = 0
91 91
92 92 self.ext = ".r"
93 93
94 94 self.optchar = "D"
95 95
96 96 self.basicHeaderObj = BasicHeader(LOCALTIME)
97 97
98 98 self.systemHeaderObj = SystemHeader()
99 99
100 100 self.radarControllerHeaderObj = RadarControllerHeader()
101 101
102 102 self.processingHeaderObj = ProcessingHeader()
103 103
104 104 self.online = 0
105 105
106 106 self.fp = None
107 107
108 108 self.idFile = None
109 109
110 110 self.dtype = None
111 111
112 112 self.fileSizeByHeader = None
113 113
114 114 self.filenameList = []
115 115
116 116 self.filename = None
117 117
118 118 self.fileSize = None
119 119
120 120 self.firstHeaderSize = 0
121 121
122 122 self.basicHeaderSize = 24
123 123
124 124 self.pathList = []
125 125
126 126 self.filenameList = []
127 127
128 128 self.lastUTTime = 0
129 129
130 130 self.maxTimeStep = 30
131 131
132 132 self.flagNoMoreFiles = 0
133 133
134 134 self.set = 0
135 135
136 136 self.path = None
137 137
138 138 self.profileIndex = 2**32 - 1
139 139
140 140 self.delay = 3 # seconds
141 141
142 142 self.nTries = 3 # quantity tries
143 143
144 144 self.nFiles = 3 # number of files for searching
145 145
146 146 self.nReadBlocks = 0
147 147
148 148 self.flagIsNewFile = 1
149 149
150 150 self.__isFirstTimeOnline = 1
151 151
152 152 # self.ippSeconds = 0
153 153
154 154 self.flagDiscontinuousBlock = 0
155 155
156 156 self.flagIsNewBlock = 0
157 157
158 158 self.nTotalBlocks = 0
159 159
160 160 self.blocksize = 0
161 161
162 162 self.dataOut = self.createObjByDefault()
163 163
164 164 self.nTxs = 1
165 165
166 166 self.txIndex = 0
167 167
168 168 def createObjByDefault(self):
169 169
170 170 dataObj = Voltage()
171 171
172 172 return dataObj
173 173
174 174 def __hasNotDataInBuffer(self):
175 175
176 176 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock * self.nTxs:
177 177 return 1
178 178
179 179 return 0
180 180
181 181 def getBlockDimension(self):
182 182 """
183 183 Obtiene la cantidad de puntos a leer por cada bloque de datos
184 184
185 185 Affected:
186 186 self.blocksize
187 187
188 188 Return:
189 189 None
190 190 """
191 191 pts2read = self.processingHeaderObj.profilesPerBlock * \
192 192 self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
193 193 self.blocksize = pts2read
194 194
195 195 def readBlock(self):
196 196 """
197 197 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
198 198 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
199 199 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
200 200 es seteado a 0
201 201
202 202 Inputs:
203 203 None
204 204
205 205 Return:
206 206 None
207 207
208 208 Affected:
209 209 self.profileIndex
210 210 self.datablock
211 211 self.flagIsNewFile
212 212 self.flagIsNewBlock
213 213 self.nTotalBlocks
214 214
215 215 Exceptions:
216 216 Si un bloque leido no es un bloque valido
217 217 """
218 218
219 219 # if self.server is not None:
220 220 # self.zBlock = self.receiver.recv()
221 221 # self.zHeader = self.zBlock[:24]
222 222 # self.zDataBlock = self.zBlock[24:]
223 223 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
224 224 # self.processingHeaderObj.profilesPerBlock = 240
225 225 # self.processingHeaderObj.nHeights = 248
226 226 # self.systemHeaderObj.nChannels
227 227 # else:
228 228 current_pointer_location = self.fp.tell()
229 229 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
230 230
231 231 try:
232 232 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
233 233 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
234 234 except:
235 235 # print "The read block (%3d) has not enough data" %self.nReadBlocks
236 236
237 237 if self.waitDataBlock(pointer_location=current_pointer_location):
238 238 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
239 239 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
240 240 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
241 241 # return 0
242 242
243 243 # Dimensions : nChannels, nProfiles, nSamples
244 244
245 245 junk = numpy.transpose(junk, (2, 0, 1))
246 246 self.datablock = junk['real'] + junk['imag'] * 1j
247 247
248 248 self.profileIndex = 0
249 249
250 250 self.flagIsNewFile = 0
251 251 self.flagIsNewBlock = 1
252 252
253 253 self.nTotalBlocks += 1
254 254 self.nReadBlocks += 1
255 255
256 256 return 1
257 257
258 258 def getFirstHeader(self):
259 259
260 260 self.getBasicHeader()
261 261
262 262 self.dataOut.processingHeaderObj = self.processingHeaderObj.copy()
263 263
264 264 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
265 265
266 266 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
267 267
268 268 if self.nTxs > 1:
269 269 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
270 270 # Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
271 271
272 272 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
273 273 #
274 274 # if self.radarControllerHeaderObj.code is not None:
275 275 #
276 276 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
277 277 #
278 278 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
279 279 #
280 280 # self.dataOut.code = self.radarControllerHeaderObj.code
281 281
282 282 self.dataOut.dtype = self.dtype
283 283
284 284 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
285 285
286 286 self.dataOut.heightList = numpy.arange(
287 287 self.processingHeaderObj.nHeights) * self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
288 288
289 289 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
290 290
291 291 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
292 292
293 293 # asumo q la data no esta decodificada
294 294 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode
295 295
296 296 # asumo q la data no esta sin flip
297 297 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip
298 298
299 299 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
300 300
301 301 def reshapeData(self):
302 302
303 303 if self.nTxs < 0:
304 304 return
305 305
306 306 if self.nTxs == 1:
307 307 return
308 308
309 309 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1. / self.nTxs) != 0:
310 310 raise ValueError("1./nTxs (=%f), should be a multiple of nProfiles (=%d)" % (
311 311 1. / self.nTxs, self.processingHeaderObj.profilesPerBlock))
312 312
313 313 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
314 314 raise ValueError("nTxs (=%d), should be a multiple of nHeights (=%d)" % (
315 315 self.nTxs, self.processingHeaderObj.nHeights))
316 316
317 317 self.datablock = self.datablock.reshape(
318 318 (self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock * self.nTxs, self.processingHeaderObj.nHeights / self.nTxs))
319 319
320 320 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock * self.nTxs
321 321 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights / self.nTxs) * \
322 322 self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
323 323 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
324 324
325 325 return
326 326
327 327 def readFirstHeaderFromServer(self):
328 328
329 329 self.getFirstHeader()
330 330
331 331 self.firstHeaderSize = self.basicHeaderObj.size
332 332
333 333 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
334 334 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
335 335 if datatype == 0:
336 336 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
337 337 elif datatype == 1:
338 338 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
339 339 elif datatype == 2:
340 340 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
341 341 elif datatype == 3:
342 342 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
343 343 elif datatype == 4:
344 344 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
345 345 elif datatype == 5:
346 346 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
347 347 else:
348 348 raise ValueError('Data type was not defined')
349 349
350 350 self.dtype = datatype_str
351 351 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
352 352 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
353 353 self.firstHeaderSize + self.basicHeaderSize * \
354 354 (self.processingHeaderObj.dataBlocksPerFile - 1)
355 355 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
356 356 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
357 357 self.getBlockDimension()
358 358
359 359 def getFromServer(self):
360 360 self.flagDiscontinuousBlock = 0
361 361 self.profileIndex = 0
362 362 self.flagIsNewBlock = 1
363 363 self.dataOut.flagNoData = False
364 364 self.nTotalBlocks += 1
365 365 self.nReadBlocks += 1
366 366 self.blockPointer = 0
367 367
368 368 block = self.receiver.recv()
369 369
370 370 self.basicHeaderObj.read(block[self.blockPointer:])
371 371 self.blockPointer += self.basicHeaderObj.length
372 372 self.systemHeaderObj.read(block[self.blockPointer:])
373 373 self.blockPointer += self.systemHeaderObj.length
374 374 self.radarControllerHeaderObj.read(block[self.blockPointer:])
375 375 self.blockPointer += self.radarControllerHeaderObj.length
376 376 self.processingHeaderObj.read(block[self.blockPointer:])
377 377 self.blockPointer += self.processingHeaderObj.length
378 378 self.readFirstHeaderFromServer()
379 379
380 380 timestamp = self.basicHeaderObj.get_datatime()
381 381 print('[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp))
382 382 current_pointer_location = self.blockPointer
383 383 junk = numpy.fromstring(
384 384 block[self.blockPointer:], self.dtype, self.blocksize)
385 385
386 386 try:
387 387 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
388 388 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
389 389 except:
390 390 # print "The read block (%3d) has not enough data" %self.nReadBlocks
391 391 if self.waitDataBlock(pointer_location=current_pointer_location):
392 392 junk = numpy.fromstring(
393 393 block[self.blockPointer:], self.dtype, self.blocksize)
394 394 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
395 395 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
396 396 # return 0
397 397
398 398 # Dimensions : nChannels, nProfiles, nSamples
399 399
400 400 junk = numpy.transpose(junk, (2, 0, 1))
401 401 self.datablock = junk['real'] + junk['imag'] * 1j
402 402 self.profileIndex = 0
403 403 if self.selBlocksize == None:
404 404 self.selBlocksize = self.dataOut.nProfiles
405 405 if self.selBlocktime != None:
406 406 if self.dataOut.nCohInt is not None:
407 407 nCohInt = self.dataOut.nCohInt
408 408 else:
409 409 nCohInt = 1
410 410 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
411 411 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
412 412 self.dataOut.data = self.datablock[:,
413 413 self.profileIndex:self.profileIndex + self.selBlocksize, :]
414 414 datasize = self.dataOut.data.shape[1]
415 415 if datasize < self.selBlocksize:
416 416 buffer = numpy.zeros(
417 417 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
418 418 buffer[:, :datasize, :] = self.dataOut.data
419 419 self.dataOut.data = buffer
420 420 self.profileIndex = blockIndex
421 421
422 422 self.dataOut.flagDataAsBlock = True
423 423 self.flagIsNewBlock = 1
424 424 self.dataOut.realtime = self.online
425 425
426 426 return self.dataOut.data
427 427
428 428 def getData(self):
429 429 """
430 430 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
431 431 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
432 432 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
433 433 "readNextBlock"
434 434
435 435 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
436 436
437 437 Return:
438 438
439 439 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
440 440 es igual al total de perfiles leidos desde el archivo.
441 441
442 442 Si self.getByBlock == False:
443 443
444 444 self.dataOut.data = buffer[:, thisProfile, :]
445 445
446 446 shape = [nChannels, nHeis]
447 447
448 448 Si self.getByBlock == True:
449 449
450 450 self.dataOut.data = buffer[:, :, :]
451 451
452 452 shape = [nChannels, nProfiles, nHeis]
453 453
454 454 Variables afectadas:
455 455 self.dataOut
456 456 self.profileIndex
457 457
458 458 Affected:
459 459 self.dataOut
460 460 self.profileIndex
461 461 self.flagDiscontinuousBlock
462 462 self.flagIsNewBlock
463 463 """
464 464 if self.flagNoMoreFiles:
465 465 self.dataOut.flagNoData = True
466 466 print('Process finished')
467 467 return 0
468 468 self.flagDiscontinuousBlock = 0
469 469 self.flagIsNewBlock = 0
470 470 if self.__hasNotDataInBuffer():
471 471 if not(self.readNextBlock()):
472 472 return 0
473 473
474 474 self.getFirstHeader()
475 475
476 476 self.reshapeData()
477 477 if self.datablock is None:
478 478 self.dataOut.flagNoData = True
479 479 return 0
480 480
481 481 if not self.getByBlock:
482 482
483 483 """
484 484 Return profile by profile
485 485
486 486 If nTxs > 1 then one profile is divided by nTxs and number of total
487 487 blocks is increased by nTxs (nProfiles *= nTxs)
488 488 """
489 489 self.dataOut.flagDataAsBlock = False
490 490 self.dataOut.data = self.datablock[:, self.profileIndex, :]
491 491 self.dataOut.profileIndex = self.profileIndex
492 492
493 493 self.profileIndex += 1
494 494
495 495 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
496 496 # """
497 497 # Return all block
498 498 # """
499 499 # self.dataOut.flagDataAsBlock = True
500 500 # self.dataOut.data = self.datablock
501 501 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
502 502 #
503 503 # self.profileIndex = self.dataOut.nProfiles
504 504
505 505 else:
506 506 """
507 507 Return a block
508 508 """
509 509 if self.selBlocksize == None:
510 510 self.selBlocksize = self.dataOut.nProfiles
511 511 if self.selBlocktime != None:
512 512 if self.dataOut.nCohInt is not None:
513 513 nCohInt = self.dataOut.nCohInt
514 514 else:
515 515 nCohInt = 1
516 516 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
517 517 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
518 518
519 519 self.dataOut.data = self.datablock[:,
520 520 self.profileIndex:self.profileIndex + self.selBlocksize, :]
521 521 self.profileIndex += self.selBlocksize
522 522 datasize = self.dataOut.data.shape[1]
523 523
524 524 if datasize < self.selBlocksize:
525 525 buffer = numpy.zeros(
526 526 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
527 527 buffer[:, :datasize, :] = self.dataOut.data
528 528
529 529 while datasize < self.selBlocksize: # Not enough profiles to fill the block
530 530 if not(self.readNextBlock()):
531 531 return 0
532 532 self.getFirstHeader()
533 533 self.reshapeData()
534 534 if self.datablock is None:
535 535 self.dataOut.flagNoData = True
536 536 return 0
537 537 # stack data
538 538 blockIndex = self.selBlocksize - datasize
539 539 datablock1 = self.datablock[:, :blockIndex, :]
540 540
541 541 buffer[:, datasize:datasize +
542 542 datablock1.shape[1], :] = datablock1
543 543 datasize += datablock1.shape[1]
544 544
545 545 self.dataOut.data = buffer
546 546 self.profileIndex = blockIndex
547 547
548 548 self.dataOut.flagDataAsBlock = True
549 549 self.dataOut.nProfiles = self.dataOut.data.shape[1]
550 550
551 551 self.dataOut.flagNoData = False
552 552
553 553 self.getBasicHeader()
554 554
555 555 self.dataOut.realtime = self.online
556 556
557 557 return self.dataOut.data
558 558
559
559 @MPDecorator
560 560 class VoltageWriter(JRODataWriter, Operation):
561 561 """
562 562 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
563 563 de los datos siempre se realiza por bloques.
564 564 """
565 565
566 566 ext = ".r"
567 567
568 568 optchar = "D"
569 569
570 570 shapeBuffer = None
571 571
572 def __init__(self, **kwargs):
572 def __init__(self):#, **kwargs):
573 573 """
574 574 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
575 575
576 576 Affected:
577 577 self.dataOut
578 578
579 579 Return: None
580 580 """
581 Operation.__init__(self, **kwargs)
581 Operation.__init__(self)#, **kwargs)
582 582
583 583 self.nTotalBlocks = 0
584 584
585 585 self.profileIndex = 0
586 586
587 587 self.isConfig = False
588 588
589 589 self.fp = None
590 590
591 591 self.flagIsNewFile = 1
592 592
593 593 self.blockIndex = 0
594 594
595 595 self.flagIsNewBlock = 0
596 596
597 597 self.setFile = None
598 598
599 599 self.dtype = None
600 600
601 601 self.path = None
602 602
603 603 self.filename = None
604 604
605 605 self.basicHeaderObj = BasicHeader(LOCALTIME)
606 606
607 607 self.systemHeaderObj = SystemHeader()
608 608
609 609 self.radarControllerHeaderObj = RadarControllerHeader()
610 610
611 611 self.processingHeaderObj = ProcessingHeader()
612 612
613 613 def hasAllDataInBuffer(self):
614 614 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
615 615 return 1
616 616 return 0
617 617
618 618 def setBlockDimension(self):
619 619 """
620 620 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
621 621
622 622 Affected:
623 623 self.shape_spc_Buffer
624 624 self.shape_cspc_Buffer
625 625 self.shape_dc_Buffer
626 626
627 627 Return: None
628 628 """
629 629 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
630 630 self.processingHeaderObj.nHeights,
631 631 self.systemHeaderObj.nChannels)
632 632
633 633 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
634 634 self.processingHeaderObj.profilesPerBlock,
635 635 self.processingHeaderObj.nHeights),
636 636 dtype=numpy.dtype('complex64'))
637 637
638 638 def writeBlock(self):
639 639 """
640 640 Escribe el buffer en el file designado
641 641
642 642 Affected:
643 643 self.profileIndex
644 644 self.flagIsNewFile
645 645 self.flagIsNewBlock
646 646 self.nTotalBlocks
647 647 self.blockIndex
648 648
649 649 Return: None
650 650 """
651 651 data = numpy.zeros(self.shapeBuffer, self.dtype)
652 652
653 653 junk = numpy.transpose(self.datablock, (1, 2, 0))
654 654
655 655 data['real'] = junk.real
656 656 data['imag'] = junk.imag
657 657
658 658 data = data.reshape((-1))
659 659
660 660 data.tofile(self.fp)
661 661
662 662 self.datablock.fill(0)
663 663
664 664 self.profileIndex = 0
665 665 self.flagIsNewFile = 0
666 666 self.flagIsNewBlock = 1
667 667
668 668 self.blockIndex += 1
669 669 self.nTotalBlocks += 1
670 670
671 671 # print "[Writing] Block = %04d" %self.blockIndex
672 672
673 673 def putData(self):
674 674 """
675 675 Setea un bloque de datos y luego los escribe en un file
676 676
677 677 Affected:
678 678 self.flagIsNewBlock
679 679 self.profileIndex
680 680
681 681 Return:
682 682 0 : Si no hay data o no hay mas files que puedan escribirse
683 683 1 : Si se escribio la data de un bloque en un file
684 684 """
685 685 if self.dataOut.flagNoData:
686 686 return 0
687 687
688 688 self.flagIsNewBlock = 0
689 689
690 690 if self.dataOut.flagDiscontinuousBlock:
691 691 self.datablock.fill(0)
692 692 self.profileIndex = 0
693 693 self.setNextFile()
694 694
695 695 if self.profileIndex == 0:
696 696 self.setBasicHeader()
697 697
698 698 self.datablock[:, self.profileIndex, :] = self.dataOut.data
699 699
700 700 self.profileIndex += 1
701 701
702 702 if self.hasAllDataInBuffer():
703 703 # if self.flagIsNewFile:
704 704 self.writeNextBlock()
705 705 # self.setFirstHeader()
706 706
707 707 return 1
708 708
709 709 def __getBlockSize(self):
710 710 '''
711 711 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
712 712 '''
713 713
714 714 dtype_width = self.getDtypeWidth()
715 715
716 716 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels *
717 717 self.profilesPerBlock * dtype_width * 2)
718 718
719 719 return blocksize
720 720
721 721 def setFirstHeader(self):
722 722 """
723 723 Obtiene una copia del First Header
724 724
725 725 Affected:
726 726 self.systemHeaderObj
727 727 self.radarControllerHeaderObj
728 728 self.dtype
729 729
730 730 Return:
731 731 None
732 732 """
733 733
734 734 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
735 735 self.systemHeaderObj.nChannels = self.dataOut.nChannels
736 736 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
737 737
738 738 self.processingHeaderObj.dtype = 0 # Voltage
739 739 self.processingHeaderObj.blockSize = self.__getBlockSize()
740 740 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
741 741 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
742 742 # podria ser 1 o self.dataOut.processingHeaderObj.nWindows
743 743 self.processingHeaderObj.nWindows = 1
744 744 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
745 745 # Cuando la data de origen es de tipo Voltage
746 746 self.processingHeaderObj.nIncohInt = 1
747 747 # Cuando la data de origen es de tipo Voltage
748 748 self.processingHeaderObj.totalSpectra = 0
749 749
750 750 if self.dataOut.code is not None:
751 751 self.processingHeaderObj.code = self.dataOut.code
752 752 self.processingHeaderObj.nCode = self.dataOut.nCode
753 753 self.processingHeaderObj.nBaud = self.dataOut.nBaud
754 754
755 755 if self.processingHeaderObj.nWindows != 0:
756 756 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
757 757 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - \
758 758 self.dataOut.heightList[0]
759 759 self.processingHeaderObj.nHeights = self.dataOut.nHeights
760 760 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
761 761
762 762 self.processingHeaderObj.processFlags = self.getProcessFlags()
763 763
764 764 self.setBasicHeader()
765 765 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,367 +1,367
1 1 '''
2 2 Updated for multiprocessing
3 3 Author : Sergio Cortez
4 4 Jan 2018
5 5 Abstract:
6 6 Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
7 7 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
8 8 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
9 9
10 10 Based on:
11 11 $Author: murco $
12 12 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
13 13 '''
14 14 from platform import python_version
15 15 import inspect
16 16 import zmq
17 17 import time
18 18 import pickle
19 19 import os
20 20 from multiprocessing import Process
21 21 from schainpy.utils import log
22 22
23 23
24 24 class ProcessingUnit(object):
25 25
26 26 """
27 27 Update - Jan 2018 - MULTIPROCESSING
28 28 All the "call" methods present in the previous base were removed.
29 29 The majority of operations are independant processes, thus
30 30 the decorator is in charge of communicate the operation processes
31 31 with the proccessing unit via IPC.
32 32
33 33 The constructor does not receive any argument. The remaining methods
34 34 are related with the operations to execute.
35 35
36 36
37 37 """
38 38
39 39 METHODS = {}
40 40 dataIn = None
41 41 dataInList = []
42 42 id = None
43 43 inputId = None
44 44 dataOut = None
45 45 dictProcs = None
46 46 isConfig = False
47 47
48 48 def __init__(self):
49 49
50 50 self.dataIn = None
51 51 self.dataOut = None
52 52 self.isConfig = False
53 53 self.operations = []
54 54
55 55 def getAllowedArgs(self):
56 56 if hasattr(self, '__attrs__'):
57 57 return self.__attrs__
58 58 else:
59 59 return inspect.getargspec(self.run).args
60 60
61 61 def addOperation(self, conf, operation):
62 62
63 63 """
64 64 This method is used in the controller, and update the dictionary containing the operations to execute. The dict
65 65 posses the id of the operation process (IPC purposes)
66 66
67 67 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
68 68 identificador asociado a este objeto.
69 69
70 70 Input:
71 71
72 72 object : objeto de la clase "Operation"
73 73
74 74 Return:
75 75
76 76 objId : identificador del objeto, necesario para comunicar con master(procUnit)
77 77 """
78 78
79 79 self.operations.append((operation, conf.type, conf.id, conf.getKwargs()))
80 80
81 81 def getOperationObj(self, objId):
82 82
83 83 if objId not in list(self.operations.keys()):
84 84 return None
85 85
86 86 return self.operations[objId]
87 87
88 88 def operation(self, **kwargs):
89 89
90 90 """
91 91 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
92 92 atributos del objeto dataOut
93 93
94 94 Input:
95 95
96 96 **kwargs : Diccionario de argumentos de la funcion a ejecutar
97 97 """
98 98
99 99 raise NotImplementedError
100 100
101 101 def setup(self):
102 102
103 103 raise NotImplementedError
104 104
105 105 def run(self):
106 106
107 107 raise NotImplementedError
108 108
109 109 def close(self):
110 110 #Close every thread, queue or any other object here is it is neccesary.
111 111 return
112 112
113 113 class Operation(object):
114 114
115 115 """
116 116 Update - Jan 2018 - MULTIPROCESSING
117 117
118 118 Most of the methods remained the same. The decorator parse the arguments and executed the run() method for each process.
119 119 The constructor doe snot receive any argument, neither the baseclass.
120 120
121 121
122 122 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
123 123 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
124 124 acumulacion dentro de esta clase
125 125
126 126 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
127 127
128 128 """
129 129 id = None
130 130 __buffer = None
131 131 dest = None
132 132 isConfig = False
133 133 readyFlag = None
134 134
135 135 def __init__(self):
136 136
137 137 self.buffer = None
138 138 self.dest = None
139 139 self.isConfig = False
140 140 self.readyFlag = False
141 141
142 142 if not hasattr(self, 'name'):
143 143 self.name = self.__class__.__name__
144 144
145 145 def getAllowedArgs(self):
146 146 if hasattr(self, '__attrs__'):
147 147 return self.__attrs__
148 148 else:
149 149 return inspect.getargspec(self.run).args
150 150
151 151 def setup(self):
152 152
153 153 self.isConfig = True
154 154
155 155 raise NotImplementedError
156 156
157 157
158 158 def run(self, dataIn, **kwargs):
159 159
160 160 """
161 161 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
162 162 atributos del objeto dataIn.
163 163
164 164 Input:
165 165
166 166 dataIn : objeto del tipo JROData
167 167
168 168 Return:
169 169
170 170 None
171 171
172 172 Affected:
173 173 __buffer : buffer de recepcion de datos.
174 174
175 175 """
176 176 if not self.isConfig:
177 177 self.setup(**kwargs)
178 178
179 179 raise NotImplementedError
180 180
181 181 def close(self):
182 182
183 183 pass
184 184
185 185
186 186 def MPDecorator(BaseClass):
187 187
188 188 """
189 189 Multiprocessing class decorator
190 190
191 191 This function add multiprocessing features to a BaseClass. Also, it handle
192 192 the communication beetween processes (readers, procUnits and operations).
193 193 """
194 194
195 195 class MPClass(BaseClass, Process):
196 196
197 197 def __init__(self, *args, **kwargs):
198 198 super(MPClass, self).__init__()
199 199 Process.__init__(self)
200 200 self.operationKwargs = {}
201 201 self.args = args
202 202 self.kwargs = kwargs
203 203 self.sender = None
204 204 self.receiver = None
205 205 self.name = BaseClass.__name__
206 206
207 207 if len(self.args) is 3:
208 208 self.typeProc = "ProcUnit"
209 209 self.id = args[0]
210 210 self.inputId = args[1]
211 211 self.project_id = args[2]
212 212 else:
213 213 self.id = args[0]
214 214 self.inputId = args[0]
215 215 self.project_id = args[1]
216 216 self.typeProc = "Operation"
217 217
218 218 def getAllowedArgs(self):
219 219
220 220 if hasattr(self, '__attrs__'):
221 221 return self.__attrs__
222 222 else:
223 223 return inspect.getargspec(BaseClass.run).args
224 224
225 225 def subscribe(self):
226 226 '''
227 227 This function create a socket to receive objects from the
228 228 topic `inputId`.
229 229 '''
230 230
231 231 c = zmq.Context()
232 232 self.receiver = c.socket(zmq.SUB)
233 233 self.receiver.connect('ipc:///tmp/schain/{}_pub'.format(self.project_id))
234 234 self.receiver.setsockopt(zmq.SUBSCRIBE, self.inputId.encode())
235 235
236 236 def listen(self):
237 237 '''
238 238 This function waits for objects and deserialize using pickle
239 239 '''
240 240
241 241 data = pickle.loads(self.receiver.recv_multipart()[1])
242 242 return data
243 243
244 244 def set_publisher(self):
245 245 '''
246 246 This function create a socket for publishing purposes.
247 247 '''
248 248
249 249 time.sleep(1)
250 250 c = zmq.Context()
251 251 self.sender = c.socket(zmq.PUB)
252 252 self.sender.connect('ipc:///tmp/schain/{}_sub'.format(self.project_id))
253 253
254 254 def publish(self, data, id):
255 255 '''
256 256 This function publish an object, to a specific topic.
257 257 '''
258 258
259 259 self.sender.send_multipart([str(id).encode(), pickle.dumps(data)])
260 260
261 261 def runReader(self):
262 262 '''
263 263 Run fuction for read units
264 264 '''
265 265 while True:
266 266
267 267 BaseClass.run(self, **self.kwargs)
268 268
269 269 if self.dataOut.error[0] == -1:
270 270 log.error(self.dataOut.error[1])
271 271 self.publish('end', self.id)
272 272 #self.sender.send_multipart([str(self.project_id).encode(), 'end'.encode()])
273 273 break
274 274
275 275 for op, optype, id, kwargs in self.operations:
276 276 if optype=='self':
277 277 op(**kwargs)
278 278 elif optype=='other':
279 279 self.dataOut = op.run(self.dataOut, **self.kwargs)
280 280 elif optype=='external':
281 281 self.publish(self.dataOut, opId)
282 282
283 283 if self.dataOut.flagNoData:
284 284 continue
285 285
286 286 self.publish(self.dataOut, self.id)
287 287
288 288 def runProc(self):
289 289 '''
290 290 Run function for proccessing units
291 291 '''
292 292
293 293 while True:
294 294 self.dataIn = self.listen()
295 295
296 296 if self.dataIn == 'end':
297 297 self.publish('end', self.id)
298 298 for op, optype, opId, kwargs in self.operations:
299 299 if optype == 'external':
300 300 self.publish('end', opId)
301 301 break
302 302
303 303 if self.dataIn.flagNoData:
304 304 continue
305 305
306 306 BaseClass.run(self, **self.kwargs)
307 307
308 308 for op, optype, opId, kwargs in self.operations:
309 309 if optype=='self':
310 310 op(**kwargs)
311 311 elif optype=='other':
312 312 self.dataOut = op.run(self.dataOut, **kwargs)
313 313 elif optype=='external':
314 314 self.publish(self.dataOut, opId)
315 315
316 316 if self.dataOut.flagNoData:
317 317 continue
318 318
319 319 self.publish(self.dataOut, self.id)
320 320
321 321 def runOp(self):
322 322 '''
323 323 Run function for operations
324 324 '''
325 325
326 326 while True:
327 327
328 328 dataOut = self.listen()
329
329
330 330 if dataOut == 'end':
331 331 break
332 332
333 333 BaseClass.run(self, dataOut, **self.kwargs)
334 334
335 335 def run(self):
336 336
337 337 if self.typeProc is "ProcUnit":
338 338
339 339 if self.inputId is not None:
340 340 self.subscribe()
341 341 self.set_publisher()
342 342
343 343 if 'Reader' not in BaseClass.__name__:
344 344 self.runProc()
345 345 else:
346 346 self.runReader()
347 347
348 348 elif self.typeProc is "Operation":
349 349
350 350 self.subscribe()
351 351 self.runOp()
352 352
353 353 else:
354 354 raise ValueError("Unknown type")
355 355
356 356 print("%s done" % BaseClass.__name__)
357 357 self.close()
358 358
359 359 def close(self):
360 360
361 361 if self.sender:
362 362 self.sender.close()
363 363
364 364 if self.receiver:
365 365 self.receiver.close()
366 366
367 367 return MPClass No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,4042 +1,4047
1 1 import numpy
2 2 import math
3 3 from scipy import optimize, interpolate, signal, stats, ndimage
4 4 import scipy
5 5 import re
6 6 import datetime
7 7 import copy
8 8 import sys
9 9 import importlib
10 10 import itertools
11 11 from multiprocessing import Pool, TimeoutError
12 12 from multiprocessing.pool import ThreadPool
13 13 import types
14 14 from functools import partial
15 15 import time
16 16 #from sklearn.cluster import KMeans
17 17
18 18
19 19 from scipy.optimize import fmin_l_bfgs_b #optimize with bounds on state papameters
20 from .jroproc_base import ProcessingUnit, Operation
20 from .jroproc_base import ProcessingUnit, Operation, MPDecorator
21 21 from schainpy.model.data.jrodata import Parameters, hildebrand_sekhon
22 22 from scipy import asarray as ar,exp
23 23 from scipy.optimize import curve_fit
24
24 from schainpy.utils import log
25 25 import warnings
26 26 from numpy import NaN
27 27 from scipy.optimize.optimize import OptimizeWarning
28 28 warnings.filterwarnings('ignore')
29 29
30 30
31 31 SPEED_OF_LIGHT = 299792458
32 32
33 33
34 34 '''solving pickling issue'''
35 35
36 36 def _pickle_method(method):
37 37 func_name = method.__func__.__name__
38 38 obj = method.__self__
39 39 cls = method.__self__.__class__
40 40 return _unpickle_method, (func_name, obj, cls)
41 41
42 42 def _unpickle_method(func_name, obj, cls):
43 43 for cls in cls.mro():
44 44 try:
45 45 func = cls.__dict__[func_name]
46 46 except KeyError:
47 47 pass
48 48 else:
49 49 break
50 50 return func.__get__(obj, cls)
51 51
52 @MPDecorator
52 53 class ParametersProc(ProcessingUnit):
53 54
55 METHODS = {}
54 56 nSeconds = None
55 57
56 58 def __init__(self):
57 59 ProcessingUnit.__init__(self)
58 60
59 61 # self.objectDict = {}
60 62 self.buffer = None
61 63 self.firstdatatime = None
62 64 self.profIndex = 0
63 65 self.dataOut = Parameters()
64
66 self.setupReq = False #Agregar a todas las unidades de proc
67
65 68 def __updateObjFromInput(self):
66 69
67 70 self.dataOut.inputUnit = self.dataIn.type
68 71
69 72 self.dataOut.timeZone = self.dataIn.timeZone
70 73 self.dataOut.dstFlag = self.dataIn.dstFlag
71 74 self.dataOut.errorCount = self.dataIn.errorCount
72 75 self.dataOut.useLocalTime = self.dataIn.useLocalTime
73 76
74 77 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
75 78 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
76 79 self.dataOut.channelList = self.dataIn.channelList
77 80 self.dataOut.heightList = self.dataIn.heightList
78 81 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
79 82 # self.dataOut.nHeights = self.dataIn.nHeights
80 83 # self.dataOut.nChannels = self.dataIn.nChannels
81 84 self.dataOut.nBaud = self.dataIn.nBaud
82 85 self.dataOut.nCode = self.dataIn.nCode
83 86 self.dataOut.code = self.dataIn.code
84 87 # self.dataOut.nProfiles = self.dataOut.nFFTPoints
85 88 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
86 89 # self.dataOut.utctime = self.firstdatatime
87 90 self.dataOut.utctime = self.dataIn.utctime
88 91 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
89 92 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
90 93 self.dataOut.nCohInt = self.dataIn.nCohInt
91 94 # self.dataOut.nIncohInt = 1
92 95 self.dataOut.ippSeconds = self.dataIn.ippSeconds
93 96 # self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
94 97 self.dataOut.timeInterval1 = self.dataIn.timeInterval
95 98 self.dataOut.heightList = self.dataIn.getHeiRange()
96 99 self.dataOut.frequency = self.dataIn.frequency
97 100 # self.dataOut.noise = self.dataIn.noise
98 101
99 102 def run(self):
100
103
104
105
101 106 #---------------------- Voltage Data ---------------------------
102 107
103 108 if self.dataIn.type == "Voltage":
104 109
105 110 self.__updateObjFromInput()
106 111 self.dataOut.data_pre = self.dataIn.data.copy()
107 112 self.dataOut.flagNoData = False
108 113 self.dataOut.utctimeInit = self.dataIn.utctime
109 114 self.dataOut.paramInterval = self.dataIn.nProfiles*self.dataIn.nCohInt*self.dataIn.ippSeconds
110 115 return
111 116
112 117 #---------------------- Spectra Data ---------------------------
113 118
114 119 if self.dataIn.type == "Spectra":
115 120
116 121 self.dataOut.data_pre = (self.dataIn.data_spc, self.dataIn.data_cspc)
117 122 self.dataOut.data_spc = self.dataIn.data_spc
118 123 self.dataOut.data_cspc = self.dataIn.data_cspc
119 124 self.dataOut.nProfiles = self.dataIn.nProfiles
120 125 self.dataOut.nIncohInt = self.dataIn.nIncohInt
121 126 self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
122 127 self.dataOut.ippFactor = self.dataIn.ippFactor
123 128 self.dataOut.abscissaList = self.dataIn.getVelRange(1)
124 129 self.dataOut.spc_noise = self.dataIn.getNoise()
125 130 self.dataOut.spc_range = (self.dataIn.getFreqRange(1)/1000. , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1))
126 131 self.dataOut.pairsList = self.dataIn.pairsList
127 132 self.dataOut.groupList = self.dataIn.pairsList
128 133 self.dataOut.flagNoData = False
129 134
130 135 if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels
131 136 self.dataOut.ChanDist = self.dataIn.ChanDist
132 137 else: self.dataOut.ChanDist = None
133 138
134 139 if hasattr(self.dataIn, 'VelRange'): #Velocities range
135 140 self.dataOut.VelRange = self.dataIn.VelRange
136 141 else: self.dataOut.VelRange = None
137 142
138 143 if hasattr(self.dataIn, 'RadarConst'): #Radar Constant
139 144 self.dataOut.RadarConst = self.dataIn.RadarConst
140 145
141 146 if hasattr(self.dataIn, 'NPW'): #NPW
142 147 self.dataOut.NPW = self.dataIn.NPW
143 148
144 149 if hasattr(self.dataIn, 'COFA'): #COFA
145 150 self.dataOut.COFA = self.dataIn.COFA
146 151
147 152
148 153
149 154 #---------------------- Correlation Data ---------------------------
150 155
151 156 if self.dataIn.type == "Correlation":
152 157 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.dataIn.splitFunctions()
153 158
154 159 self.dataOut.data_pre = (self.dataIn.data_cf[acf_ind,:], self.dataIn.data_cf[ccf_ind,:,:])
155 160 self.dataOut.normFactor = (self.dataIn.normFactor[acf_ind,:], self.dataIn.normFactor[ccf_ind,:])
156 161 self.dataOut.groupList = (acf_pairs, ccf_pairs)
157 162
158 163 self.dataOut.abscissaList = self.dataIn.lagRange
159 164 self.dataOut.noise = self.dataIn.noise
160 165 self.dataOut.data_SNR = self.dataIn.SNR
161 166 self.dataOut.flagNoData = False
162 167 self.dataOut.nAvg = self.dataIn.nAvg
163 168
164 169 #---------------------- Parameters Data ---------------------------
165 170
166 171 if self.dataIn.type == "Parameters":
167 172 self.dataOut.copy(self.dataIn)
168 173 self.dataOut.flagNoData = False
169 174
170 175 return True
171 176
172 177 self.__updateObjFromInput()
173 178 self.dataOut.utctimeInit = self.dataIn.utctime
174 179 self.dataOut.paramInterval = self.dataIn.timeInterval
175 180
176 181 return
177 182
178 183
179 184 def target(tups):
180 185
181 186 obj, args = tups
182 187 #print 'TARGETTT', obj, args
183 188 return obj.FitGau(args)
184 189
185 190 class GaussianFit(Operation):
186 191
187 192 '''
188 193 Function that fit of one and two generalized gaussians (gg) based
189 194 on the PSD shape across an "power band" identified from a cumsum of
190 195 the measured spectrum - noise.
191 196
192 197 Input:
193 198 self.dataOut.data_pre : SelfSpectra
194 199
195 200 Output:
196 201 self.dataOut.GauSPC : SPC_ch1, SPC_ch2
197 202
198 203 '''
199 204 def __init__(self, **kwargs):
200 205 Operation.__init__(self, **kwargs)
201 206 self.i=0
202 207
203 208
204 209 def run(self, dataOut, num_intg=7, pnoise=1., vel_arr=None, SNRlimit=-9): #num_intg: Incoherent integrations, pnoise: Noise, vel_arr: range of velocities, similar to the ftt points
205 210 """This routine will find a couple of generalized Gaussians to a power spectrum
206 211 input: spc
207 212 output:
208 213 Amplitude0,shift0,width0,p0,Amplitude1,shift1,width1,p1,noise
209 214 """
210 215
211 216 self.spc = dataOut.data_pre[0].copy()
212 217
213 218
214 219 print('SelfSpectra Shape', numpy.asarray(self.spc).shape)
215 220
216 221
217 222 #plt.figure(50)
218 223 #plt.subplot(121)
219 224 #plt.plot(self.spc,'k',label='spc(66)')
220 225 #plt.plot(xFrec,ySamples[1],'g',label='Ch1')
221 226 #plt.plot(xFrec,ySamples[2],'r',label='Ch2')
222 227 #plt.plot(xFrec,FitGauss,'yo:',label='fit')
223 228 #plt.legend()
224 229 #plt.title('DATOS A ALTURA DE 7500 METROS')
225 230 #plt.show()
226 231
227 232 self.Num_Hei = self.spc.shape[2]
228 233 #self.Num_Bin = len(self.spc)
229 234 self.Num_Bin = self.spc.shape[1]
230 235 self.Num_Chn = self.spc.shape[0]
231 236
232 237 Vrange = dataOut.abscissaList
233 238
234 239 #print 'self.spc2', numpy.asarray(self.spc).shape
235 240
236 241 GauSPC = numpy.empty([2,self.Num_Bin,self.Num_Hei])
237 242 SPC_ch1 = numpy.empty([self.Num_Bin,self.Num_Hei])
238 243 SPC_ch2 = numpy.empty([self.Num_Bin,self.Num_Hei])
239 244 SPC_ch1[:] = numpy.NaN
240 245 SPC_ch2[:] = numpy.NaN
241 246
242 247
243 248 start_time = time.time()
244 249
245 250 noise_ = dataOut.spc_noise[0].copy()
246 251
247 252
248 253
249 254 pool = Pool(processes=self.Num_Chn)
250 255 args = [(Vrange, Ch, pnoise, noise_, num_intg, SNRlimit) for Ch in range(self.Num_Chn)]
251 256 objs = [self for __ in range(self.Num_Chn)]
252 257 attrs = list(zip(objs, args))
253 258 gauSPC = pool.map(target, attrs)
254 259 dataOut.GauSPC = numpy.asarray(gauSPC)
255 260 # ret = []
256 261 # for n in range(self.Num_Chn):
257 262 # self.FitGau(args[n])
258 263 # dataOut.GauSPC = ret
259 264
260 265
261 266
262 267 # for ch in range(self.Num_Chn):
263 268 #
264 269 # for ht in range(self.Num_Hei):
265 270 # #print (numpy.asarray(self.spc).shape)
266 271 # spc = numpy.asarray(self.spc)[ch,:,ht]
267 272 #
268 273 # #############################################
269 274 # # normalizing spc and noise
270 275 # # This part differs from gg1
271 276 # spc_norm_max = max(spc)
272 277 # spc = spc / spc_norm_max
273 278 # pnoise = pnoise / spc_norm_max
274 279 # #############################################
275 280 #
276 281 # if abs(vel_arr[0])<15.0: # this switch is for spectra collected with different length IPP's
277 282 # fatspectra=1.0
278 283 # else:
279 284 # fatspectra=0.5
280 285 #
281 286 # wnoise = noise_ / spc_norm_max
282 287 # #print 'wnoise', noise_, dataOut.spc_noise[0], wnoise
283 288 # #wnoise,stdv,i_max,index =enoise(spc,num_intg) #noise estimate using Hildebrand Sekhon, only wnoise is used
284 289 # #if wnoise>1.1*pnoise: # to be tested later
285 290 # # wnoise=pnoise
286 291 # noisebl=wnoise*0.9; noisebh=wnoise*1.1
287 292 # spc=spc-wnoise
288 293 #
289 294 # minx=numpy.argmin(spc)
290 295 # spcs=numpy.roll(spc,-minx)
291 296 # cum=numpy.cumsum(spcs)
292 297 # tot_noise=wnoise * self.Num_Bin #64;
293 298 # #tot_signal=sum(cum[-5:])/5.; ''' How does this line work? '''
294 299 # #snr=tot_signal/tot_noise
295 300 # #snr=cum[-1]/tot_noise
296 301 #
297 302 # #print 'spc' , spcs[5:8] , 'tot_noise', tot_noise
298 303 #
299 304 # snr = sum(spcs)/tot_noise
300 305 # snrdB=10.*numpy.log10(snr)
301 306 #
302 307 # #if snrdB < -9 :
303 308 # # snrdB = numpy.NaN
304 309 # # continue
305 310 #
306 311 # #print 'snr',snrdB # , sum(spcs) , tot_noise
307 312 #
308 313 #
309 314 # #if snrdB<-18 or numpy.isnan(snrdB) or num_intg<4:
310 315 # # return [None,]*4,[None,]*4,None,snrdB,None,None,[None,]*5,[None,]*9,None
311 316 #
312 317 # cummax=max(cum); epsi=0.08*fatspectra # cumsum to narrow down the energy region
313 318 # cumlo=cummax*epsi;
314 319 # cumhi=cummax*(1-epsi)
315 320 # powerindex=numpy.array(numpy.where(numpy.logical_and(cum>cumlo, cum<cumhi))[0])
316 321 #
317 322 # #if len(powerindex)==1:
318 323 # ##return [numpy.mod(powerindex[0]+minx,64),None,None,None,],[None,]*4,None,snrdB,None,None,[None,]*5,[None,]*9,None
319 324 # #return [numpy.mod(powerindex[0]+minx, self.Num_Bin ),None,None,None,],[None,]*4,None,snrdB,None,None,[None,]*5,[None,]*9,None
320 325 # #elif len(powerindex)<4*fatspectra:
321 326 # #return [None,]*4,[None,]*4,None,snrdB,None,None,[None,]*5,[None,]*9,None
322 327 #
323 328 # if len(powerindex) < 1:# case for powerindex 0
324 329 # continue
325 330 # powerlo=powerindex[0]
326 331 # powerhi=powerindex[-1]
327 332 # powerwidth=powerhi-powerlo
328 333 #
329 334 # firstpeak=powerlo+powerwidth/10.# first gaussian energy location
330 335 # secondpeak=powerhi-powerwidth/10.#second gaussian energy location
331 336 # midpeak=(firstpeak+secondpeak)/2.
332 337 # firstamp=spcs[int(firstpeak)]
333 338 # secondamp=spcs[int(secondpeak)]
334 339 # midamp=spcs[int(midpeak)]
335 340 # #x=numpy.spc.shape[1]
336 341 #
337 342 # #x=numpy.arange(64)
338 343 # x=numpy.arange( self.Num_Bin )
339 344 # y_data=spc+wnoise
340 345 #
341 346 # # single gaussian
342 347 # #shift0=numpy.mod(midpeak+minx,64)
343 348 # shift0=numpy.mod(midpeak+minx, self.Num_Bin )
344 349 # width0=powerwidth/4.#Initialization entire power of spectrum divided by 4
345 350 # power0=2.
346 351 # amplitude0=midamp
347 352 # state0=[shift0,width0,amplitude0,power0,wnoise]
348 353 # #bnds=((0,63),(1,powerwidth),(0,None),(0.5,3.),(noisebl,noisebh))
349 354 # bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth),(0,None),(0.5,3.),(noisebl,noisebh))
350 355 # #bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth),(0,None),(0.5,3.),(0.1,0.5))
351 356 # # bnds = range of fft, power width, amplitude, power, noise
352 357 # lsq1=fmin_l_bfgs_b(self.misfit1,state0,args=(y_data,x,num_intg),bounds=bnds,approx_grad=True)
353 358 #
354 359 # chiSq1=lsq1[1];
355 360 # jack1= self.y_jacobian1(x,lsq1[0])
356 361 #
357 362 #
358 363 # try:
359 364 # sigmas1=numpy.sqrt(chiSq1*numpy.diag(numpy.linalg.inv(numpy.dot(jack1.T,jack1))))
360 365 # except:
361 366 # std1=32.; sigmas1=numpy.ones(5)
362 367 # else:
363 368 # std1=sigmas1[0]
364 369 #
365 370 #
366 371 # if fatspectra<1.0 and powerwidth<4:
367 372 # choice=0
368 373 # Amplitude0=lsq1[0][2]
369 374 # shift0=lsq1[0][0]
370 375 # width0=lsq1[0][1]
371 376 # p0=lsq1[0][3]
372 377 # Amplitude1=0.
373 378 # shift1=0.
374 379 # width1=0.
375 380 # p1=0.
376 381 # noise=lsq1[0][4]
377 382 # #return (numpy.array([shift0,width0,Amplitude0,p0]),
378 383 # # numpy.array([shift1,width1,Amplitude1,p1]),noise,snrdB,chiSq1,6.,sigmas1,[None,]*9,choice)
379 384 #
380 385 # # two gaussians
381 386 # #shift0=numpy.mod(firstpeak+minx,64); shift1=numpy.mod(secondpeak+minx,64)
382 387 # shift0=numpy.mod(firstpeak+minx, self.Num_Bin );
383 388 # shift1=numpy.mod(secondpeak+minx, self.Num_Bin )
384 389 # width0=powerwidth/6.;
385 390 # width1=width0
386 391 # power0=2.;
387 392 # power1=power0
388 393 # amplitude0=firstamp;
389 394 # amplitude1=secondamp
390 395 # state0=[shift0,width0,amplitude0,power0,shift1,width1,amplitude1,power1,wnoise]
391 396 # #bnds=((0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
392 397 # bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth/2.),(0,None),(0.5,3.),( 0,(self.Num_Bin-1)),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
393 398 # #bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth/2.),(0,None),(0.5,3.),( 0,(self.Num_Bin-1)),(1,powerwidth/2.),(0,None),(0.5,3.),(0.1,0.5))
394 399 #
395 400 # lsq2=fmin_l_bfgs_b(self.misfit2,state0,args=(y_data,x,num_intg),bounds=bnds,approx_grad=True)
396 401 #
397 402 #
398 403 # chiSq2=lsq2[1]; jack2=self.y_jacobian2(x,lsq2[0])
399 404 #
400 405 #
401 406 # try:
402 407 # sigmas2=numpy.sqrt(chiSq2*numpy.diag(numpy.linalg.inv(numpy.dot(jack2.T,jack2))))
403 408 # except:
404 409 # std2a=32.; std2b=32.; sigmas2=numpy.ones(9)
405 410 # else:
406 411 # std2a=sigmas2[0]; std2b=sigmas2[4]
407 412 #
408 413 #
409 414 #
410 415 # oneG=(chiSq1<5 and chiSq1/chiSq2<2.0) and (abs(lsq2[0][0]-lsq2[0][4])<(lsq2[0][1]+lsq2[0][5])/3. or abs(lsq2[0][0]-lsq2[0][4])<10)
411 416 #
412 417 # if snrdB>-9: # when SNR is strong pick the peak with least shift (LOS velocity) error
413 418 # if oneG:
414 419 # choice=0
415 420 # else:
416 421 # w1=lsq2[0][1]; w2=lsq2[0][5]
417 422 # a1=lsq2[0][2]; a2=lsq2[0][6]
418 423 # p1=lsq2[0][3]; p2=lsq2[0][7]
419 424 # s1=(2**(1+1./p1))*scipy.special.gamma(1./p1)/p1; s2=(2**(1+1./p2))*scipy.special.gamma(1./p2)/p2;
420 425 # gp1=a1*w1*s1; gp2=a2*w2*s2 # power content of each ggaussian with proper p scaling
421 426 #
422 427 # if gp1>gp2:
423 428 # if a1>0.7*a2:
424 429 # choice=1
425 430 # else:
426 431 # choice=2
427 432 # elif gp2>gp1:
428 433 # if a2>0.7*a1:
429 434 # choice=2
430 435 # else:
431 436 # choice=1
432 437 # else:
433 438 # choice=numpy.argmax([a1,a2])+1
434 439 # #else:
435 440 # #choice=argmin([std2a,std2b])+1
436 441 #
437 442 # else: # with low SNR go to the most energetic peak
438 443 # choice=numpy.argmax([lsq1[0][2]*lsq1[0][1],lsq2[0][2]*lsq2[0][1],lsq2[0][6]*lsq2[0][5]])
439 444 #
440 445 # #print 'choice',choice
441 446 #
442 447 # if choice==0: # pick the single gaussian fit
443 448 # Amplitude0=lsq1[0][2]
444 449 # shift0=lsq1[0][0]
445 450 # width0=lsq1[0][1]
446 451 # p0=lsq1[0][3]
447 452 # Amplitude1=0.
448 453 # shift1=0.
449 454 # width1=0.
450 455 # p1=0.
451 456 # noise=lsq1[0][4]
452 457 # elif choice==1: # take the first one of the 2 gaussians fitted
453 458 # Amplitude0 = lsq2[0][2]
454 459 # shift0 = lsq2[0][0]
455 460 # width0 = lsq2[0][1]
456 461 # p0 = lsq2[0][3]
457 462 # Amplitude1 = lsq2[0][6] # This is 0 in gg1
458 463 # shift1 = lsq2[0][4] # This is 0 in gg1
459 464 # width1 = lsq2[0][5] # This is 0 in gg1
460 465 # p1 = lsq2[0][7] # This is 0 in gg1
461 466 # noise = lsq2[0][8]
462 467 # else: # the second one
463 468 # Amplitude0 = lsq2[0][6]
464 469 # shift0 = lsq2[0][4]
465 470 # width0 = lsq2[0][5]
466 471 # p0 = lsq2[0][7]
467 472 # Amplitude1 = lsq2[0][2] # This is 0 in gg1
468 473 # shift1 = lsq2[0][0] # This is 0 in gg1
469 474 # width1 = lsq2[0][1] # This is 0 in gg1
470 475 # p1 = lsq2[0][3] # This is 0 in gg1
471 476 # noise = lsq2[0][8]
472 477 #
473 478 # #print len(noise + Amplitude0*numpy.exp(-0.5*(abs(x-shift0))/width0)**p0)
474 479 # SPC_ch1[:,ht] = noise + Amplitude0*numpy.exp(-0.5*(abs(x-shift0))/width0)**p0
475 480 # SPC_ch2[:,ht] = noise + Amplitude1*numpy.exp(-0.5*(abs(x-shift1))/width1)**p1
476 481 # #print 'SPC_ch1.shape',SPC_ch1.shape
477 482 # #print 'SPC_ch2.shape',SPC_ch2.shape
478 483 # #dataOut.data_param = SPC_ch1
479 484 # GauSPC[0] = SPC_ch1
480 485 # GauSPC[1] = SPC_ch2
481 486
482 487 # #plt.gcf().clear()
483 488 # plt.figure(50+self.i)
484 489 # self.i=self.i+1
485 490 # #plt.subplot(121)
486 491 # plt.plot(self.spc,'k')#,label='spc(66)')
487 492 # plt.plot(SPC_ch1[ch,ht],'b')#,label='gg1')
488 493 # #plt.plot(SPC_ch2,'r')#,label='gg2')
489 494 # #plt.plot(xFrec,ySamples[1],'g',label='Ch1')
490 495 # #plt.plot(xFrec,ySamples[2],'r',label='Ch2')
491 496 # #plt.plot(xFrec,FitGauss,'yo:',label='fit')
492 497 # plt.legend()
493 498 # plt.title('DATOS A ALTURA DE 7500 METROS')
494 499 # plt.show()
495 500 # print 'shift0', shift0
496 501 # print 'Amplitude0', Amplitude0
497 502 # print 'width0', width0
498 503 # print 'p0', p0
499 504 # print '========================'
500 505 # print 'shift1', shift1
501 506 # print 'Amplitude1', Amplitude1
502 507 # print 'width1', width1
503 508 # print 'p1', p1
504 509 # print 'noise', noise
505 510 # print 's_noise', wnoise
506 511
507 512 print('========================================================')
508 513 print('total_time: ', time.time()-start_time)
509 514
510 515 # re-normalizing spc and noise
511 516 # This part differs from gg1
512 517
513 518
514 519
515 520 ''' Parameters:
516 521 1. Amplitude
517 522 2. Shift
518 523 3. Width
519 524 4. Power
520 525 '''
521 526
522 527
523 528 ###############################################################################
524 529 def FitGau(self, X):
525 530
526 531 Vrange, ch, pnoise, noise_, num_intg, SNRlimit = X
527 532 #print 'VARSSSS', ch, pnoise, noise, num_intg
528 533
529 534 #print 'HEIGHTS', self.Num_Hei
530 535
531 536 GauSPC = []
532 537 SPC_ch1 = numpy.empty([self.Num_Bin,self.Num_Hei])
533 538 SPC_ch2 = numpy.empty([self.Num_Bin,self.Num_Hei])
534 539 SPC_ch1[:] = 0#numpy.NaN
535 540 SPC_ch2[:] = 0#numpy.NaN
536 541
537 542
538 543
539 544 for ht in range(self.Num_Hei):
540 545 #print (numpy.asarray(self.spc).shape)
541 546
542 547 #print 'TTTTT', ch , ht
543 548 #print self.spc.shape
544 549
545 550
546 551 spc = numpy.asarray(self.spc)[ch,:,ht]
547 552
548 553 #############################################
549 554 # normalizing spc and noise
550 555 # This part differs from gg1
551 556 spc_norm_max = max(spc)
552 557 spc = spc / spc_norm_max
553 558 pnoise = pnoise / spc_norm_max
554 559 #############################################
555 560
556 561 fatspectra=1.0
557 562
558 563 wnoise = noise_ / spc_norm_max
559 564 #wnoise,stdv,i_max,index =enoise(spc,num_intg) #noise estimate using Hildebrand Sekhon, only wnoise is used
560 565 #if wnoise>1.1*pnoise: # to be tested later
561 566 # wnoise=pnoise
562 567 noisebl=wnoise*0.9; noisebh=wnoise*1.1
563 568 spc=spc-wnoise
564 569 # print 'wnoise', noise_[0], spc_norm_max, wnoise
565 570 minx=numpy.argmin(spc)
566 571 spcs=numpy.roll(spc,-minx)
567 572 cum=numpy.cumsum(spcs)
568 573 tot_noise=wnoise * self.Num_Bin #64;
569 574 #print 'spc' , spcs[5:8] , 'tot_noise', tot_noise
570 575 #tot_signal=sum(cum[-5:])/5.; ''' How does this line work? '''
571 576 #snr=tot_signal/tot_noise
572 577 #snr=cum[-1]/tot_noise
573 578 snr = sum(spcs)/tot_noise
574 579 snrdB=10.*numpy.log10(snr)
575 580
576 581 if snrdB < SNRlimit :
577 582 snr = numpy.NaN
578 583 SPC_ch1[:,ht] = 0#numpy.NaN
579 584 SPC_ch1[:,ht] = 0#numpy.NaN
580 585 GauSPC = (SPC_ch1,SPC_ch2)
581 586 continue
582 587 #print 'snr',snrdB #, sum(spcs) , tot_noise
583 588
584 589
585 590
586 591 #if snrdB<-18 or numpy.isnan(snrdB) or num_intg<4:
587 592 # return [None,]*4,[None,]*4,None,snrdB,None,None,[None,]*5,[None,]*9,None
588 593
589 594 cummax=max(cum); epsi=0.08*fatspectra # cumsum to narrow down the energy region
590 595 cumlo=cummax*epsi;
591 596 cumhi=cummax*(1-epsi)
592 597 powerindex=numpy.array(numpy.where(numpy.logical_and(cum>cumlo, cum<cumhi))[0])
593 598
594 599
595 600 if len(powerindex) < 1:# case for powerindex 0
596 601 continue
597 602 powerlo=powerindex[0]
598 603 powerhi=powerindex[-1]
599 604 powerwidth=powerhi-powerlo
600 605
601 606 firstpeak=powerlo+powerwidth/10.# first gaussian energy location
602 607 secondpeak=powerhi-powerwidth/10.#second gaussian energy location
603 608 midpeak=(firstpeak+secondpeak)/2.
604 609 firstamp=spcs[int(firstpeak)]
605 610 secondamp=spcs[int(secondpeak)]
606 611 midamp=spcs[int(midpeak)]
607 612
608 613 x=numpy.arange( self.Num_Bin )
609 614 y_data=spc+wnoise
610 615
611 616 # single gaussian
612 617 shift0=numpy.mod(midpeak+minx, self.Num_Bin )
613 618 width0=powerwidth/4.#Initialization entire power of spectrum divided by 4
614 619 power0=2.
615 620 amplitude0=midamp
616 621 state0=[shift0,width0,amplitude0,power0,wnoise]
617 622 bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth),(0,None),(0.5,3.),(noisebl,noisebh))
618 623 lsq1=fmin_l_bfgs_b(self.misfit1,state0,args=(y_data,x,num_intg),bounds=bnds,approx_grad=True)
619 624
620 625 chiSq1=lsq1[1];
621 626 jack1= self.y_jacobian1(x,lsq1[0])
622 627
623 628
624 629 try:
625 630 sigmas1=numpy.sqrt(chiSq1*numpy.diag(numpy.linalg.inv(numpy.dot(jack1.T,jack1))))
626 631 except:
627 632 std1=32.; sigmas1=numpy.ones(5)
628 633 else:
629 634 std1=sigmas1[0]
630 635
631 636
632 637 if fatspectra<1.0 and powerwidth<4:
633 638 choice=0
634 639 Amplitude0=lsq1[0][2]
635 640 shift0=lsq1[0][0]
636 641 width0=lsq1[0][1]
637 642 p0=lsq1[0][3]
638 643 Amplitude1=0.
639 644 shift1=0.
640 645 width1=0.
641 646 p1=0.
642 647 noise=lsq1[0][4]
643 648 #return (numpy.array([shift0,width0,Amplitude0,p0]),
644 649 # numpy.array([shift1,width1,Amplitude1,p1]),noise,snrdB,chiSq1,6.,sigmas1,[None,]*9,choice)
645 650
646 651 # two gaussians
647 652 #shift0=numpy.mod(firstpeak+minx,64); shift1=numpy.mod(secondpeak+minx,64)
648 653 shift0=numpy.mod(firstpeak+minx, self.Num_Bin );
649 654 shift1=numpy.mod(secondpeak+minx, self.Num_Bin )
650 655 width0=powerwidth/6.;
651 656 width1=width0
652 657 power0=2.;
653 658 power1=power0
654 659 amplitude0=firstamp;
655 660 amplitude1=secondamp
656 661 state0=[shift0,width0,amplitude0,power0,shift1,width1,amplitude1,power1,wnoise]
657 662 #bnds=((0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
658 663 bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth/2.),(0,None),(0.5,3.),( 0,(self.Num_Bin-1)),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
659 664 #bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth/2.),(0,None),(0.5,3.),( 0,(self.Num_Bin-1)),(1,powerwidth/2.),(0,None),(0.5,3.),(0.1,0.5))
660 665
661 666 lsq2=fmin_l_bfgs_b(self.misfit2,state0,args=(y_data,x,num_intg),bounds=bnds,approx_grad=True)
662 667
663 668
664 669 chiSq2=lsq2[1]; jack2=self.y_jacobian2(x,lsq2[0])
665 670
666 671
667 672 try:
668 673 sigmas2=numpy.sqrt(chiSq2*numpy.diag(numpy.linalg.inv(numpy.dot(jack2.T,jack2))))
669 674 except:
670 675 std2a=32.; std2b=32.; sigmas2=numpy.ones(9)
671 676 else:
672 677 std2a=sigmas2[0]; std2b=sigmas2[4]
673 678
674 679
675 680
676 681 oneG=(chiSq1<5 and chiSq1/chiSq2<2.0) and (abs(lsq2[0][0]-lsq2[0][4])<(lsq2[0][1]+lsq2[0][5])/3. or abs(lsq2[0][0]-lsq2[0][4])<10)
677 682
678 683 if snrdB>-6: # when SNR is strong pick the peak with least shift (LOS velocity) error
679 684 if oneG:
680 685 choice=0
681 686 else:
682 687 w1=lsq2[0][1]; w2=lsq2[0][5]
683 688 a1=lsq2[0][2]; a2=lsq2[0][6]
684 689 p1=lsq2[0][3]; p2=lsq2[0][7]
685 690 s1=(2**(1+1./p1))*scipy.special.gamma(1./p1)/p1;
686 691 s2=(2**(1+1./p2))*scipy.special.gamma(1./p2)/p2;
687 692 gp1=a1*w1*s1; gp2=a2*w2*s2 # power content of each ggaussian with proper p scaling
688 693
689 694 if gp1>gp2:
690 695 if a1>0.7*a2:
691 696 choice=1
692 697 else:
693 698 choice=2
694 699 elif gp2>gp1:
695 700 if a2>0.7*a1:
696 701 choice=2
697 702 else:
698 703 choice=1
699 704 else:
700 705 choice=numpy.argmax([a1,a2])+1
701 706 #else:
702 707 #choice=argmin([std2a,std2b])+1
703 708
704 709 else: # with low SNR go to the most energetic peak
705 710 choice=numpy.argmax([lsq1[0][2]*lsq1[0][1],lsq2[0][2]*lsq2[0][1],lsq2[0][6]*lsq2[0][5]])
706 711
707 712
708 713 shift0=lsq2[0][0]; vel0=Vrange[0] + shift0*(Vrange[1]-Vrange[0])
709 714 shift1=lsq2[0][4]; vel1=Vrange[0] + shift1*(Vrange[1]-Vrange[0])
710 715
711 716 max_vel = 20
712 717
713 718 #first peak will be 0, second peak will be 1
714 719 if vel0 > 0 and vel0 < max_vel : #first peak is in the correct range
715 720 shift0=lsq2[0][0]
716 721 width0=lsq2[0][1]
717 722 Amplitude0=lsq2[0][2]
718 723 p0=lsq2[0][3]
719 724
720 725 shift1=lsq2[0][4]
721 726 width1=lsq2[0][5]
722 727 Amplitude1=lsq2[0][6]
723 728 p1=lsq2[0][7]
724 729 noise=lsq2[0][8]
725 730 else:
726 731 shift1=lsq2[0][0]
727 732 width1=lsq2[0][1]
728 733 Amplitude1=lsq2[0][2]
729 734 p1=lsq2[0][3]
730 735
731 736 shift0=lsq2[0][4]
732 737 width0=lsq2[0][5]
733 738 Amplitude0=lsq2[0][6]
734 739 p0=lsq2[0][7]
735 740 noise=lsq2[0][8]
736 741
737 742 if Amplitude0<0.1: # in case the peak is noise
738 743 shift0,width0,Amplitude0,p0 = 4*[numpy.NaN]
739 744 if Amplitude1<0.1:
740 745 shift1,width1,Amplitude1,p1 = 4*[numpy.NaN]
741 746
742 747
743 748 # if choice==0: # pick the single gaussian fit
744 749 # Amplitude0=lsq1[0][2]
745 750 # shift0=lsq1[0][0]
746 751 # width0=lsq1[0][1]
747 752 # p0=lsq1[0][3]
748 753 # Amplitude1=0.
749 754 # shift1=0.
750 755 # width1=0.
751 756 # p1=0.
752 757 # noise=lsq1[0][4]
753 758 # elif choice==1: # take the first one of the 2 gaussians fitted
754 759 # Amplitude0 = lsq2[0][2]
755 760 # shift0 = lsq2[0][0]
756 761 # width0 = lsq2[0][1]
757 762 # p0 = lsq2[0][3]
758 763 # Amplitude1 = lsq2[0][6] # This is 0 in gg1
759 764 # shift1 = lsq2[0][4] # This is 0 in gg1
760 765 # width1 = lsq2[0][5] # This is 0 in gg1
761 766 # p1 = lsq2[0][7] # This is 0 in gg1
762 767 # noise = lsq2[0][8]
763 768 # else: # the second one
764 769 # Amplitude0 = lsq2[0][6]
765 770 # shift0 = lsq2[0][4]
766 771 # width0 = lsq2[0][5]
767 772 # p0 = lsq2[0][7]
768 773 # Amplitude1 = lsq2[0][2] # This is 0 in gg1
769 774 # shift1 = lsq2[0][0] # This is 0 in gg1
770 775 # width1 = lsq2[0][1] # This is 0 in gg1
771 776 # p1 = lsq2[0][3] # This is 0 in gg1
772 777 # noise = lsq2[0][8]
773 778
774 779 #print len(noise + Amplitude0*numpy.exp(-0.5*(abs(x-shift0))/width0)**p0)
775 780 SPC_ch1[:,ht] = noise + Amplitude0*numpy.exp(-0.5*(abs(x-shift0))/width0)**p0
776 781 SPC_ch2[:,ht] = noise + Amplitude1*numpy.exp(-0.5*(abs(x-shift1))/width1)**p1
777 782 #print 'SPC_ch1.shape',SPC_ch1.shape
778 783 #print 'SPC_ch2.shape',SPC_ch2.shape
779 784 #dataOut.data_param = SPC_ch1
780 785 GauSPC = (SPC_ch1,SPC_ch2)
781 786 #GauSPC[1] = SPC_ch2
782 787
783 788 # print 'shift0', shift0
784 789 # print 'Amplitude0', Amplitude0
785 790 # print 'width0', width0
786 791 # print 'p0', p0
787 792 # print '========================'
788 793 # print 'shift1', shift1
789 794 # print 'Amplitude1', Amplitude1
790 795 # print 'width1', width1
791 796 # print 'p1', p1
792 797 # print 'noise', noise
793 798 # print 's_noise', wnoise
794 799
795 800 return GauSPC
796 801
797 802
798 803 def y_jacobian1(self,x,state): # This function is for further analysis of generalized Gaussians, it is not too importan for the signal discrimination.
799 804 y_model=self.y_model1(x,state)
800 805 s0,w0,a0,p0,n=state
801 806 e0=((x-s0)/w0)**2;
802 807
803 808 e0u=((x-s0-self.Num_Bin)/w0)**2;
804 809
805 810 e0d=((x-s0+self.Num_Bin)/w0)**2
806 811 m0=numpy.exp(-0.5*e0**(p0/2.));
807 812 m0u=numpy.exp(-0.5*e0u**(p0/2.));
808 813 m0d=numpy.exp(-0.5*e0d**(p0/2.))
809 814 JA=m0+m0u+m0d
810 815 JP=(-1/4.)*a0*m0*e0**(p0/2.)*numpy.log(e0)+(-1/4.)*a0*m0u*e0u**(p0/2.)*numpy.log(e0u)+(-1/4.)*a0*m0d*e0d**(p0/2.)*numpy.log(e0d)
811 816
812 817 JS=(p0/w0/2.)*a0*m0*e0**(p0/2.-1)*((x-s0)/w0)+(p0/w0/2.)*a0*m0u*e0u**(p0/2.-1)*((x-s0- self.Num_Bin )/w0)+(p0/w0/2.)*a0*m0d*e0d**(p0/2.-1)*((x-s0+ self.Num_Bin )/w0)
813 818
814 819 JW=(p0/w0/2.)*a0*m0*e0**(p0/2.-1)*((x-s0)/w0)**2+(p0/w0/2.)*a0*m0u*e0u**(p0/2.-1)*((x-s0- self.Num_Bin )/w0)**2+(p0/w0/2.)*a0*m0d*e0d**(p0/2.-1)*((x-s0+ self.Num_Bin )/w0)**2
815 820 jack1=numpy.sqrt(7)*numpy.array([JS/y_model,JW/y_model,JA/y_model,JP/y_model,1./y_model])
816 821 return jack1.T
817 822
818 823 def y_jacobian2(self,x,state):
819 824 y_model=self.y_model2(x,state)
820 825 s0,w0,a0,p0,s1,w1,a1,p1,n=state
821 826 e0=((x-s0)/w0)**2;
822 827
823 828 e0u=((x-s0- self.Num_Bin )/w0)**2;
824 829
825 830 e0d=((x-s0+ self.Num_Bin )/w0)**2
826 831 e1=((x-s1)/w1)**2;
827 832
828 833 e1u=((x-s1- self.Num_Bin )/w1)**2;
829 834
830 835 e1d=((x-s1+ self.Num_Bin )/w1)**2
831 836 m0=numpy.exp(-0.5*e0**(p0/2.));
832 837 m0u=numpy.exp(-0.5*e0u**(p0/2.));
833 838 m0d=numpy.exp(-0.5*e0d**(p0/2.))
834 839 m1=numpy.exp(-0.5*e1**(p1/2.));
835 840 m1u=numpy.exp(-0.5*e1u**(p1/2.));
836 841 m1d=numpy.exp(-0.5*e1d**(p1/2.))
837 842 JA=m0+m0u+m0d
838 843 JA1=m1+m1u+m1d
839 844 JP=(-1/4.)*a0*m0*e0**(p0/2.)*numpy.log(e0)+(-1/4.)*a0*m0u*e0u**(p0/2.)*numpy.log(e0u)+(-1/4.)*a0*m0d*e0d**(p0/2.)*numpy.log(e0d)
840 845 JP1=(-1/4.)*a1*m1*e1**(p1/2.)*numpy.log(e1)+(-1/4.)*a1*m1u*e1u**(p1/2.)*numpy.log(e1u)+(-1/4.)*a1*m1d*e1d**(p1/2.)*numpy.log(e1d)
841 846
842 847 JS=(p0/w0/2.)*a0*m0*e0**(p0/2.-1)*((x-s0)/w0)+(p0/w0/2.)*a0*m0u*e0u**(p0/2.-1)*((x-s0- self.Num_Bin )/w0)+(p0/w0/2.)*a0*m0d*e0d**(p0/2.-1)*((x-s0+ self.Num_Bin )/w0)
843 848
844 849 JS1=(p1/w1/2.)*a1*m1*e1**(p1/2.-1)*((x-s1)/w1)+(p1/w1/2.)*a1*m1u*e1u**(p1/2.-1)*((x-s1- self.Num_Bin )/w1)+(p1/w1/2.)*a1*m1d*e1d**(p1/2.-1)*((x-s1+ self.Num_Bin )/w1)
845 850
846 851 JW=(p0/w0/2.)*a0*m0*e0**(p0/2.-1)*((x-s0)/w0)**2+(p0/w0/2.)*a0*m0u*e0u**(p0/2.-1)*((x-s0- self.Num_Bin )/w0)**2+(p0/w0/2.)*a0*m0d*e0d**(p0/2.-1)*((x-s0+ self.Num_Bin )/w0)**2
847 852
848 853 JW1=(p1/w1/2.)*a1*m1*e1**(p1/2.-1)*((x-s1)/w1)**2+(p1/w1/2.)*a1*m1u*e1u**(p1/2.-1)*((x-s1- self.Num_Bin )/w1)**2+(p1/w1/2.)*a1*m1d*e1d**(p1/2.-1)*((x-s1+ self.Num_Bin )/w1)**2
849 854 jack2=numpy.sqrt(7)*numpy.array([JS/y_model,JW/y_model,JA/y_model,JP/y_model,JS1/y_model,JW1/y_model,JA1/y_model,JP1/y_model,1./y_model])
850 855 return jack2.T
851 856
852 857 def y_model1(self,x,state):
853 858 shift0,width0,amplitude0,power0,noise=state
854 859 model0=amplitude0*numpy.exp(-0.5*abs((x-shift0)/width0)**power0)
855 860
856 861 model0u=amplitude0*numpy.exp(-0.5*abs((x-shift0- self.Num_Bin )/width0)**power0)
857 862
858 863 model0d=amplitude0*numpy.exp(-0.5*abs((x-shift0+ self.Num_Bin )/width0)**power0)
859 864 return model0+model0u+model0d+noise
860 865
861 866 def y_model2(self,x,state): #Equation for two generalized Gaussians with Nyquist
862 867 shift0,width0,amplitude0,power0,shift1,width1,amplitude1,power1,noise=state
863 868 model0=amplitude0*numpy.exp(-0.5*abs((x-shift0)/width0)**power0)
864 869
865 870 model0u=amplitude0*numpy.exp(-0.5*abs((x-shift0- self.Num_Bin )/width0)**power0)
866 871
867 872 model0d=amplitude0*numpy.exp(-0.5*abs((x-shift0+ self.Num_Bin )/width0)**power0)
868 873 model1=amplitude1*numpy.exp(-0.5*abs((x-shift1)/width1)**power1)
869 874
870 875 model1u=amplitude1*numpy.exp(-0.5*abs((x-shift1- self.Num_Bin )/width1)**power1)
871 876
872 877 model1d=amplitude1*numpy.exp(-0.5*abs((x-shift1+ self.Num_Bin )/width1)**power1)
873 878 return model0+model0u+model0d+model1+model1u+model1d+noise
874 879
875 880 def misfit1(self,state,y_data,x,num_intg): # This function compares how close real data is with the model data, the close it is, the better it is.
876 881
877 882 return num_intg*sum((numpy.log(y_data)-numpy.log(self.y_model1(x,state)))**2)#/(64-5.) # /(64-5.) can be commented
878 883
879 884 def misfit2(self,state,y_data,x,num_intg):
880 885 return num_intg*sum((numpy.log(y_data)-numpy.log(self.y_model2(x,state)))**2)#/(64-9.)
881 886
882 887
883 888 class PrecipitationProc(Operation):
884 889
885 890 '''
886 891 Operator that estimates Reflectivity factor (Z), and estimates rainfall Rate (R)
887 892
888 893 Input:
889 894 self.dataOut.data_pre : SelfSpectra
890 895
891 896 Output:
892 897
893 898 self.dataOut.data_output : Reflectivity factor, rainfall Rate
894 899
895 900
896 901 Parameters affected:
897 902 '''
898 903
899 904
900 905 def run(self, dataOut, radar=None, Pt=None, Gt=None, Gr=None, Lambda=None, aL=None,
901 906 tauW=None, ThetaT=None, ThetaR=None, Km = 0.93, Altitude=None):
902 907
903 908 self.spc = dataOut.data_pre[0].copy()
904 909 self.Num_Hei = self.spc.shape[2]
905 910 self.Num_Bin = self.spc.shape[1]
906 911 self.Num_Chn = self.spc.shape[0]
907 912
908 913 Velrange = dataOut.abscissaList
909 914
910 915 if radar == "MIRA35C" :
911 916
912 917 Ze = self.dBZeMODE2(dataOut)
913 918
914 919 else:
915 920
916 921 self.Pt = Pt
917 922 self.Gt = Gt
918 923 self.Gr = Gr
919 924 self.Lambda = Lambda
920 925 self.aL = aL
921 926 self.tauW = tauW
922 927 self.ThetaT = ThetaT
923 928 self.ThetaR = ThetaR
924 929
925 930 RadarConstant = GetRadarConstant()
926 931 SPCmean = numpy.mean(self.spc,0)
927 932 ETA = numpy.zeros(self.Num_Hei)
928 933 Pr = numpy.sum(SPCmean,0)
929 934
930 935 #for R in range(self.Num_Hei):
931 936 # ETA[R] = RadarConstant * Pr[R] * R**2 #Reflectivity (ETA)
932 937
933 938 D_range = numpy.zeros(self.Num_Hei)
934 939 EqSec = numpy.zeros(self.Num_Hei)
935 940 del_V = numpy.zeros(self.Num_Hei)
936 941
937 942 for R in range(self.Num_Hei):
938 943 ETA[R] = RadarConstant * Pr[R] * R**2 #Reflectivity (ETA)
939 944
940 945 h = R + Altitude #Range from ground to radar pulse altitude
941 946 del_V[R] = 1 + 3.68 * 10**-5 * h + 1.71 * 10**-9 * h**2 #Density change correction for velocity
942 947
943 948 D_range[R] = numpy.log( (9.65 - (Velrange[R]/del_V[R])) / 10.3 ) / -0.6 #Range of Diameter of drops related to velocity
944 949 SIGMA[R] = numpy.pi**5 / Lambda**4 * Km * D_range[R]**6 #Equivalent Section of drops (sigma)
945 950
946 951 N_dist[R] = ETA[R] / SIGMA[R]
947 952
948 953 Ze = (ETA * Lambda**4) / (numpy.pi * Km)
949 954 Z = numpy.sum( N_dist * D_range**6 )
950 955 RR = 6*10**-4*numpy.pi * numpy.sum( D_range**3 * N_dist * Velrange ) #Rainfall rate
951 956
952 957
953 958 RR = (Ze/200)**(1/1.6)
954 959 dBRR = 10*numpy.log10(RR)
955 960
956 961 dBZe = 10*numpy.log10(Ze)
957 962 dataOut.data_output = Ze
958 963 dataOut.data_param = numpy.ones([2,self.Num_Hei])
959 964 dataOut.channelList = [0,1]
960 965 print('channelList', dataOut.channelList)
961 966 dataOut.data_param[0]=dBZe
962 967 dataOut.data_param[1]=dBRR
963 968 print('RR SHAPE', dBRR.shape)
964 969 print('Ze SHAPE', dBZe.shape)
965 970 print('dataOut.data_param SHAPE', dataOut.data_param.shape)
966 971
967 972
968 973 def dBZeMODE2(self, dataOut): # Processing for MIRA35C
969 974
970 975 NPW = dataOut.NPW
971 976 COFA = dataOut.COFA
972 977
973 978 SNR = numpy.array([self.spc[0,:,:] / NPW[0]]) #, self.spc[1,:,:] / NPW[1]])
974 979 RadarConst = dataOut.RadarConst
975 980 #frequency = 34.85*10**9
976 981
977 982 ETA = numpy.zeros(([self.Num_Chn ,self.Num_Hei]))
978 983 data_output = numpy.ones([self.Num_Chn , self.Num_Hei])*numpy.NaN
979 984
980 985 ETA = numpy.sum(SNR,1)
981 986 print('ETA' , ETA)
982 987 ETA = numpy.where(ETA is not 0. , ETA, numpy.NaN)
983 988
984 989 Ze = numpy.ones([self.Num_Chn, self.Num_Hei] )
985 990
986 991 for r in range(self.Num_Hei):
987 992
988 993 Ze[0,r] = ( ETA[0,r] ) * COFA[0,r][0] * RadarConst * ((r/5000.)**2)
989 994 #Ze[1,r] = ( ETA[1,r] ) * COFA[1,r][0] * RadarConst * ((r/5000.)**2)
990 995
991 996 return Ze
992 997
993 998 def GetRadarConstant(self):
994 999
995 1000 """
996 1001 Constants:
997 1002
998 1003 Pt: Transmission Power dB
999 1004 Gt: Transmission Gain dB
1000 1005 Gr: Reception Gain dB
1001 1006 Lambda: Wavelenght m
1002 1007 aL: Attenuation loses dB
1003 1008 tauW: Width of transmission pulse s
1004 1009 ThetaT: Transmission antenna bean angle rad
1005 1010 ThetaR: Reception antenna beam angle rad
1006 1011
1007 1012 """
1008 1013 Numerator = ( (4*numpy.pi)**3 * aL**2 * 16 * numpy.log(2) )
1009 1014 Denominator = ( Pt * Gt * Gr * Lambda**2 * SPEED_OF_LIGHT * TauW * numpy.pi * ThetaT * TheraR)
1010 1015 RadarConstant = Numerator / Denominator
1011 1016
1012 1017 return RadarConstant
1013 1018
1014 1019
1015 1020
1016 1021 class FullSpectralAnalysis(Operation):
1017 1022
1018 1023 """
1019 1024 Function that implements Full Spectral Analisys technique.
1020 1025
1021 1026 Input:
1022 1027 self.dataOut.data_pre : SelfSpectra and CrossSPectra data
1023 1028 self.dataOut.groupList : Pairlist of channels
1024 1029 self.dataOut.ChanDist : Physical distance between receivers
1025 1030
1026 1031
1027 1032 Output:
1028 1033
1029 1034 self.dataOut.data_output : Zonal wind, Meridional wind and Vertical wind
1030 1035
1031 1036
1032 1037 Parameters affected: Winds, height range, SNR
1033 1038
1034 1039 """
1035 1040 def run(self, dataOut, E01=None, E02=None, E12=None, N01=None, N02=None, N12=None, SNRlimit=7):
1036 1041
1037 1042 spc = dataOut.data_pre[0].copy()
1038 1043 cspc = dataOut.data_pre[1].copy()
1039 1044
1040 1045 nChannel = spc.shape[0]
1041 1046 nProfiles = spc.shape[1]
1042 1047 nHeights = spc.shape[2]
1043 1048
1044 1049 pairsList = dataOut.groupList
1045 1050 if dataOut.ChanDist is not None :
1046 1051 ChanDist = dataOut.ChanDist
1047 1052 else:
1048 1053 ChanDist = numpy.array([[E01, N01],[E02,N02],[E12,N12]])
1049 1054
1050 1055 #print 'ChanDist', ChanDist
1051 1056
1052 1057 if dataOut.VelRange is not None:
1053 1058 VelRange= dataOut.VelRange
1054 1059 else:
1055 1060 VelRange= dataOut.abscissaList
1056 1061
1057 1062 ySamples=numpy.ones([nChannel,nProfiles])
1058 1063 phase=numpy.ones([nChannel,nProfiles])
1059 1064 CSPCSamples=numpy.ones([nChannel,nProfiles],dtype=numpy.complex_)
1060 1065 coherence=numpy.ones([nChannel,nProfiles])
1061 1066 PhaseSlope=numpy.ones(nChannel)
1062 1067 PhaseInter=numpy.ones(nChannel)
1063 1068 dataSNR = dataOut.data_SNR
1064 1069
1065 1070
1066 1071
1067 1072 data = dataOut.data_pre
1068 1073 noise = dataOut.noise
1069 1074 print('noise',noise)
1070 1075 #SNRdB = 10*numpy.log10(dataOut.data_SNR)
1071 1076
1072 1077 FirstMoment = numpy.average(dataOut.data_param[:,1,:],0)
1073 1078 #SNRdBMean = []
1074 1079
1075 1080
1076 1081 #for j in range(nHeights):
1077 1082 # FirstMoment = numpy.append(FirstMoment,numpy.mean([dataOut.data_param[0,1,j],dataOut.data_param[1,1,j],dataOut.data_param[2,1,j]]))
1078 1083 # SNRdBMean = numpy.append(SNRdBMean,numpy.mean([SNRdB[0,j],SNRdB[1,j],SNRdB[2,j]]))
1079 1084
1080 1085 data_output=numpy.ones([3,spc.shape[2]])*numpy.NaN
1081 1086
1082 1087 velocityX=[]
1083 1088 velocityY=[]
1084 1089 velocityV=[]
1085 1090
1086 1091 dbSNR = 10*numpy.log10(dataSNR)
1087 1092 dbSNR = numpy.average(dbSNR,0)
1088 1093 for Height in range(nHeights):
1089 1094
1090 1095 [Vzon,Vmer,Vver, GaussCenter]= self.WindEstimation(spc, cspc, pairsList, ChanDist, Height, noise, VelRange, dbSNR[Height], SNRlimit)
1091 1096
1092 1097 if abs(Vzon)<100. and abs(Vzon)> 0.:
1093 1098 velocityX=numpy.append(velocityX, Vzon)#Vmag
1094 1099
1095 1100 else:
1096 1101 print('Vzon',Vzon)
1097 1102 velocityX=numpy.append(velocityX, numpy.NaN)
1098 1103
1099 1104 if abs(Vmer)<100. and abs(Vmer) > 0.:
1100 1105 velocityY=numpy.append(velocityY, Vmer)#Vang
1101 1106
1102 1107 else:
1103 1108 print('Vmer',Vmer)
1104 1109 velocityY=numpy.append(velocityY, numpy.NaN)
1105 1110
1106 1111 if dbSNR[Height] > SNRlimit:
1107 1112 velocityV=numpy.append(velocityV, FirstMoment[Height])
1108 1113 else:
1109 1114 velocityV=numpy.append(velocityV, numpy.NaN)
1110 1115 #FirstMoment[Height]= numpy.NaN
1111 1116 # if SNRdBMean[Height] <12:
1112 1117 # FirstMoment[Height] = numpy.NaN
1113 1118 # velocityX[Height] = numpy.NaN
1114 1119 # velocityY[Height] = numpy.NaN
1115 1120
1116 1121
1117 1122 data_output[0]=numpy.array(velocityX)
1118 1123 data_output[1]=numpy.array(velocityY)
1119 1124 data_output[2]=-velocityV#FirstMoment
1120 1125
1121 1126 print(' ')
1122 1127 #print 'FirstMoment'
1123 1128 #print FirstMoment
1124 1129 print('velocityX',data_output[0])
1125 1130 print(' ')
1126 1131 print('velocityY',data_output[1])
1127 1132 #print numpy.array(velocityY)
1128 1133 print(' ')
1129 1134 #print 'SNR'
1130 1135 #print 10*numpy.log10(dataOut.data_SNR)
1131 1136 #print numpy.shape(10*numpy.log10(dataOut.data_SNR))
1132 1137 print(' ')
1133 1138
1134 1139
1135 1140 dataOut.data_output=data_output
1136 1141 return
1137 1142
1138 1143
1139 1144 def moving_average(self,x, N=2):
1140 1145 return numpy.convolve(x, numpy.ones((N,))/N)[(N-1):]
1141 1146
1142 1147 def gaus(self,xSamples,a,x0,sigma):
1143 1148 return a*numpy.exp(-(xSamples-x0)**2/(2*sigma**2))
1144 1149
1145 1150 def Find(self,x,value):
1146 1151 for index in range(len(x)):
1147 1152 if x[index]==value:
1148 1153 return index
1149 1154
1150 1155 def WindEstimation(self, spc, cspc, pairsList, ChanDist, Height, noise, VelRange, dbSNR, SNRlimit):
1151 1156
1152 1157 ySamples=numpy.ones([spc.shape[0],spc.shape[1]])
1153 1158 phase=numpy.ones([spc.shape[0],spc.shape[1]])
1154 1159 CSPCSamples=numpy.ones([spc.shape[0],spc.shape[1]],dtype=numpy.complex_)
1155 1160 coherence=numpy.ones([spc.shape[0],spc.shape[1]])
1156 1161 PhaseSlope=numpy.ones(spc.shape[0])
1157 1162 PhaseInter=numpy.ones(spc.shape[0])
1158 1163 xFrec=VelRange
1159 1164
1160 1165 '''Getting Eij and Nij'''
1161 1166
1162 1167 E01=ChanDist[0][0]
1163 1168 N01=ChanDist[0][1]
1164 1169
1165 1170 E02=ChanDist[1][0]
1166 1171 N02=ChanDist[1][1]
1167 1172
1168 1173 E12=ChanDist[2][0]
1169 1174 N12=ChanDist[2][1]
1170 1175
1171 1176 z = spc.copy()
1172 1177 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1173 1178
1174 1179 for i in range(spc.shape[0]):
1175 1180
1176 1181 '''****** Line of Data SPC ******'''
1177 1182 zline=z[i,:,Height]
1178 1183
1179 1184 '''****** SPC is normalized ******'''
1180 1185 FactNorm= (zline.copy()-noise[i]) / numpy.sum(zline.copy())
1181 1186 FactNorm= FactNorm/numpy.sum(FactNorm)
1182 1187
1183 1188 SmoothSPC=self.moving_average(FactNorm,N=3)
1184 1189
1185 1190 xSamples = ar(list(range(len(SmoothSPC))))
1186 1191 ySamples[i] = SmoothSPC
1187 1192
1188 1193 #dbSNR=10*numpy.log10(dataSNR)
1189 1194 print(' ')
1190 1195 print(' ')
1191 1196 print(' ')
1192 1197
1193 1198 #print 'dataSNR', dbSNR.shape, dbSNR[0,40:120]
1194 1199 print('SmoothSPC', SmoothSPC.shape, SmoothSPC[0:20])
1195 1200 print('noise',noise)
1196 1201 print('zline',zline.shape, zline[0:20])
1197 1202 print('FactNorm',FactNorm.shape, FactNorm[0:20])
1198 1203 print('FactNorm suma', numpy.sum(FactNorm))
1199 1204
1200 1205 for i in range(spc.shape[0]):
1201 1206
1202 1207 '''****** Line of Data CSPC ******'''
1203 1208 cspcLine=cspc[i,:,Height].copy()
1204 1209
1205 1210 '''****** CSPC is normalized ******'''
1206 1211 chan_index0 = pairsList[i][0]
1207 1212 chan_index1 = pairsList[i][1]
1208 1213 CSPCFactor= abs(numpy.sum(ySamples[chan_index0]) * numpy.sum(ySamples[chan_index1])) #
1209 1214
1210 1215 CSPCNorm = (cspcLine.copy() -noise[i]) / numpy.sqrt(CSPCFactor)
1211 1216
1212 1217 CSPCSamples[i] = CSPCNorm
1213 1218 coherence[i] = numpy.abs(CSPCSamples[i]) / numpy.sqrt(CSPCFactor)
1214 1219
1215 1220 coherence[i]= self.moving_average(coherence[i],N=2)
1216 1221
1217 1222 phase[i] = self.moving_average( numpy.arctan2(CSPCSamples[i].imag, CSPCSamples[i].real),N=1)#*180/numpy.pi
1218 1223
1219 1224 print('cspcLine', cspcLine.shape, cspcLine[0:20])
1220 1225 print('CSPCFactor', CSPCFactor)#, CSPCFactor[0:20]
1221 1226 print(numpy.sum(ySamples[chan_index0]), numpy.sum(ySamples[chan_index1]), -noise[i])
1222 1227 print('CSPCNorm', CSPCNorm.shape, CSPCNorm[0:20])
1223 1228 print('CSPCNorm suma', numpy.sum(CSPCNorm))
1224 1229 print('CSPCSamples', CSPCSamples.shape, CSPCSamples[0,0:20])
1225 1230
1226 1231 '''****** Getting fij width ******'''
1227 1232
1228 1233 yMean=[]
1229 1234 yMean2=[]
1230 1235
1231 1236 for j in range(len(ySamples[1])):
1232 1237 yMean=numpy.append(yMean,numpy.mean([ySamples[0,j],ySamples[1,j],ySamples[2,j]]))
1233 1238
1234 1239 '''******* Getting fitting Gaussian ******'''
1235 1240 meanGauss=sum(xSamples*yMean) / len(xSamples)
1236 1241 sigma=sum(yMean*(xSamples-meanGauss)**2) / len(xSamples)
1237 1242
1238 1243 print('****************************')
1239 1244 print('len(xSamples): ',len(xSamples))
1240 1245 print('yMean: ', yMean.shape, yMean[0:20])
1241 1246 print('ySamples', ySamples.shape, ySamples[0,0:20])
1242 1247 print('xSamples: ',xSamples.shape, xSamples[0:20])
1243 1248
1244 1249 print('meanGauss',meanGauss)
1245 1250 print('sigma',sigma)
1246 1251
1247 1252 #if (abs(meanGauss/sigma**2) > 0.0001) : #0.000000001):
1248 1253 if dbSNR > SNRlimit :
1249 1254 try:
1250 1255 popt,pcov = curve_fit(self.gaus,xSamples,yMean,p0=[1,meanGauss,sigma])
1251 1256
1252 1257 if numpy.amax(popt)>numpy.amax(yMean)*0.3:
1253 1258 FitGauss=self.gaus(xSamples,*popt)
1254 1259
1255 1260 else:
1256 1261 FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean)
1257 1262 print('Verificador: Dentro', Height)
1258 1263 except :#RuntimeError:
1259 1264 FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean)
1260 1265
1261 1266
1262 1267 else:
1263 1268 FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean)
1264 1269
1265 1270 Maximun=numpy.amax(yMean)
1266 1271 eMinus1=Maximun*numpy.exp(-1)#*0.8
1267 1272
1268 1273 HWpos=self.Find(FitGauss,min(FitGauss, key=lambda value:abs(value-eMinus1)))
1269 1274 HalfWidth= xFrec[HWpos]
1270 1275 GCpos=self.Find(FitGauss, numpy.amax(FitGauss))
1271 1276 Vpos=self.Find(FactNorm, numpy.amax(FactNorm))
1272 1277
1273 1278 #Vpos=FirstMoment[]
1274 1279
1275 1280 '''****** Getting Fij ******'''
1276 1281
1277 1282 GaussCenter=xFrec[GCpos]
1278 1283 if (GaussCenter<0 and HalfWidth>0) or (GaussCenter>0 and HalfWidth<0):
1279 1284 Fij=abs(GaussCenter)+abs(HalfWidth)+0.0000001
1280 1285 else:
1281 1286 Fij=abs(GaussCenter-HalfWidth)+0.0000001
1282 1287
1283 1288 '''****** Getting Frecuency range of significant data ******'''
1284 1289
1285 1290 Rangpos=self.Find(FitGauss,min(FitGauss, key=lambda value:abs(value-Maximun*0.10)))
1286 1291
1287 1292 if Rangpos<GCpos:
1288 1293 Range=numpy.array([Rangpos,2*GCpos-Rangpos])
1289 1294 elif Rangpos< ( len(xFrec)- len(xFrec)*0.1):
1290 1295 Range=numpy.array([2*GCpos-Rangpos,Rangpos])
1291 1296 else:
1292 1297 Range = numpy.array([0,0])
1293 1298
1294 1299 print(' ')
1295 1300 print('GCpos',GCpos, ( len(xFrec)- len(xFrec)*0.1))
1296 1301 print('Rangpos',Rangpos)
1297 1302 print('RANGE: ', Range)
1298 1303 FrecRange=xFrec[Range[0]:Range[1]]
1299 1304
1300 1305 '''****** Getting SCPC Slope ******'''
1301 1306
1302 1307 for i in range(spc.shape[0]):
1303 1308
1304 1309 if len(FrecRange)>5 and len(FrecRange)<spc.shape[1]*0.5:
1305 1310 PhaseRange=self.moving_average(phase[i,Range[0]:Range[1]],N=3)
1306 1311
1307 1312 print('FrecRange', len(FrecRange) , FrecRange)
1308 1313 print('PhaseRange', len(PhaseRange), PhaseRange)
1309 1314 print(' ')
1310 1315 if len(FrecRange) == len(PhaseRange):
1311 1316 slope, intercept, r_value, p_value, std_err = stats.linregress(FrecRange,PhaseRange)
1312 1317 PhaseSlope[i]=slope
1313 1318 PhaseInter[i]=intercept
1314 1319 else:
1315 1320 PhaseSlope[i]=0
1316 1321 PhaseInter[i]=0
1317 1322 else:
1318 1323 PhaseSlope[i]=0
1319 1324 PhaseInter[i]=0
1320 1325
1321 1326 '''Getting constant C'''
1322 1327 cC=(Fij*numpy.pi)**2
1323 1328
1324 1329 '''****** Getting constants F and G ******'''
1325 1330 MijEijNij=numpy.array([[E02,N02], [E12,N12]])
1326 1331 MijResult0=(-PhaseSlope[1]*cC) / (2*numpy.pi)
1327 1332 MijResult1=(-PhaseSlope[2]*cC) / (2*numpy.pi)
1328 1333 MijResults=numpy.array([MijResult0,MijResult1])
1329 1334 (cF,cG) = numpy.linalg.solve(MijEijNij, MijResults)
1330 1335
1331 1336 '''****** Getting constants A, B and H ******'''
1332 1337 W01=numpy.amax(coherence[0])
1333 1338 W02=numpy.amax(coherence[1])
1334 1339 W12=numpy.amax(coherence[2])
1335 1340
1336 1341 WijResult0=((cF*E01+cG*N01)**2)/cC - numpy.log(W01 / numpy.sqrt(numpy.pi/cC))
1337 1342 WijResult1=((cF*E02+cG*N02)**2)/cC - numpy.log(W02 / numpy.sqrt(numpy.pi/cC))
1338 1343 WijResult2=((cF*E12+cG*N12)**2)/cC - numpy.log(W12 / numpy.sqrt(numpy.pi/cC))
1339 1344
1340 1345 WijResults=numpy.array([WijResult0, WijResult1, WijResult2])
1341 1346
1342 1347 WijEijNij=numpy.array([ [E01**2, N01**2, 2*E01*N01] , [E02**2, N02**2, 2*E02*N02] , [E12**2, N12**2, 2*E12*N12] ])
1343 1348 (cA,cB,cH) = numpy.linalg.solve(WijEijNij, WijResults)
1344 1349
1345 1350 VxVy=numpy.array([[cA,cH],[cH,cB]])
1346 1351
1347 1352 VxVyResults=numpy.array([-cF,-cG])
1348 1353 (Vx,Vy) = numpy.linalg.solve(VxVy, VxVyResults)
1349 1354
1350 1355 Vzon = Vy
1351 1356 Vmer = Vx
1352 1357 Vmag=numpy.sqrt(Vzon**2+Vmer**2)
1353 1358 Vang=numpy.arctan2(Vmer,Vzon)
1354 1359 Vver=xFrec[Vpos]
1355 1360 print('vzon y vmer', Vzon, Vmer)
1356 1361 return Vzon, Vmer, Vver, GaussCenter
1357
1362
1358 1363 class SpectralMoments(Operation):
1359 1364
1360 1365 '''
1361 1366 Function SpectralMoments()
1362 1367
1363 1368 Calculates moments (power, mean, standard deviation) and SNR of the signal
1364 1369
1365 1370 Type of dataIn: Spectra
1366 1371
1367 1372 Configuration Parameters:
1368 1373
1369 1374 dirCosx : Cosine director in X axis
1370 1375 dirCosy : Cosine director in Y axis
1371 1376
1372 1377 elevation :
1373 1378 azimuth :
1374 1379
1375 1380 Input:
1376 1381 channelList : simple channel list to select e.g. [2,3,7]
1377 1382 self.dataOut.data_pre : Spectral data
1378 1383 self.dataOut.abscissaList : List of frequencies
1379 1384 self.dataOut.noise : Noise level per channel
1380 1385
1381 1386 Affected:
1382 1387 self.dataOut.data_param : Parameters per channel
1383 1388 self.dataOut.data_SNR : SNR per channel
1384 1389
1385 1390 '''
1386 1391
1387 1392 def run(self, dataOut):
1388 1393
1389 1394 #dataOut.data_pre = dataOut.data_pre[0]
1390 1395 data = dataOut.data_pre[0]
1391 1396 absc = dataOut.abscissaList[:-1]
1392 1397 noise = dataOut.noise
1393 1398 nChannel = data.shape[0]
1394 1399 data_param = numpy.zeros((nChannel, 4, data.shape[2]))
1395 1400
1396 1401 for ind in range(nChannel):
1397 1402 data_param[ind,:,:] = self.__calculateMoments( data[ind,:,:] , absc , noise[ind] )
1398 1403
1399 1404 dataOut.data_param = data_param[:,1:,:]
1400 1405 dataOut.data_SNR = data_param[:,0]
1401 1406 dataOut.data_DOP = data_param[:,1]
1402 1407 dataOut.data_MEAN = data_param[:,2]
1403 1408 dataOut.data_STD = data_param[:,3]
1404 return
1409 return dataOut
1405 1410
1406 1411 def __calculateMoments(self, oldspec, oldfreq, n0,
1407 1412 nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None):
1408 1413
1409 1414 if (nicoh is None): nicoh = 1
1410 1415 if (graph is None): graph = 0
1411 1416 if (smooth is None): smooth = 0
1412 1417 elif (self.smooth < 3): smooth = 0
1413 1418
1414 1419 if (type1 is None): type1 = 0
1415 1420 if (fwindow is None): fwindow = numpy.zeros(oldfreq.size) + 1
1416 1421 if (snrth is None): snrth = -3
1417 1422 if (dc is None): dc = 0
1418 1423 if (aliasing is None): aliasing = 0
1419 1424 if (oldfd is None): oldfd = 0
1420 1425 if (wwauto is None): wwauto = 0
1421 1426
1422 1427 if (n0 < 1.e-20): n0 = 1.e-20
1423 1428
1424 1429 freq = oldfreq
1425 1430 vec_power = numpy.zeros(oldspec.shape[1])
1426 1431 vec_fd = numpy.zeros(oldspec.shape[1])
1427 1432 vec_w = numpy.zeros(oldspec.shape[1])
1428 1433 vec_snr = numpy.zeros(oldspec.shape[1])
1429 1434
1430 1435 for ind in range(oldspec.shape[1]):
1431 1436
1432 1437 spec = oldspec[:,ind]
1433 1438 aux = spec*fwindow
1434 1439 max_spec = aux.max()
1435 1440 m = list(aux).index(max_spec)
1436 1441
1437 1442 #Smooth
1438 1443 if (smooth == 0): spec2 = spec
1439 1444 else: spec2 = scipy.ndimage.filters.uniform_filter1d(spec,size=smooth)
1440 1445
1441 1446 # Calculo de Momentos
1442 1447 bb = spec2[list(range(m,spec2.size))]
1443 1448 bb = (bb<n0).nonzero()
1444 1449 bb = bb[0]
1445 1450
1446 1451 ss = spec2[list(range(0,m + 1))]
1447 1452 ss = (ss<n0).nonzero()
1448 1453 ss = ss[0]
1449 1454
1450 1455 if (bb.size == 0):
1451 1456 bb0 = spec.size - 1 - m
1452 1457 else:
1453 1458 bb0 = bb[0] - 1
1454 1459 if (bb0 < 0):
1455 1460 bb0 = 0
1456 1461
1457 1462 if (ss.size == 0): ss1 = 1
1458 1463 else: ss1 = max(ss) + 1
1459 1464
1460 1465 if (ss1 > m): ss1 = m
1461 1466
1462 1467 valid = numpy.asarray(list(range(int(m + bb0 - ss1 + 1)))) + ss1
1463 1468 power = ((spec2[valid] - n0)*fwindow[valid]).sum()
1464 1469 fd = ((spec2[valid]- n0)*freq[valid]*fwindow[valid]).sum()/power
1465 1470 w = math.sqrt(((spec2[valid] - n0)*fwindow[valid]*(freq[valid]- fd)**2).sum()/power)
1466 1471 snr = (spec2.mean()-n0)/n0
1467 1472
1468 1473 if (snr < 1.e-20) :
1469 1474 snr = 1.e-20
1470 1475
1471 1476 vec_power[ind] = power
1472 1477 vec_fd[ind] = fd
1473 1478 vec_w[ind] = w
1474 1479 vec_snr[ind] = snr
1475 1480
1476 1481 moments = numpy.vstack((vec_snr, vec_power, vec_fd, vec_w))
1477 1482 return moments
1478 1483
1479 1484 #------------------ Get SA Parameters --------------------------
1480 1485
1481 1486 def GetSAParameters(self):
1482 1487 #SA en frecuencia
1483 1488 pairslist = self.dataOut.groupList
1484 1489 num_pairs = len(pairslist)
1485 1490
1486 1491 vel = self.dataOut.abscissaList
1487 1492 spectra = self.dataOut.data_pre
1488 1493 cspectra = self.dataIn.data_cspc
1489 1494 delta_v = vel[1] - vel[0]
1490 1495
1491 1496 #Calculating the power spectrum
1492 1497 spc_pow = numpy.sum(spectra, 3)*delta_v
1493 1498 #Normalizing Spectra
1494 1499 norm_spectra = spectra/spc_pow
1495 1500 #Calculating the norm_spectra at peak
1496 1501 max_spectra = numpy.max(norm_spectra, 3)
1497 1502
1498 1503 #Normalizing Cross Spectra
1499 1504 norm_cspectra = numpy.zeros(cspectra.shape)
1500 1505
1501 1506 for i in range(num_chan):
1502 1507 norm_cspectra[i,:,:] = cspectra[i,:,:]/numpy.sqrt(spc_pow[pairslist[i][0],:]*spc_pow[pairslist[i][1],:])
1503 1508
1504 1509 max_cspectra = numpy.max(norm_cspectra,2)
1505 1510 max_cspectra_index = numpy.argmax(norm_cspectra, 2)
1506 1511
1507 1512 for i in range(num_pairs):
1508 1513 cspc_par[i,:,:] = __calculateMoments(norm_cspectra)
1509 1514 #------------------- Get Lags ----------------------------------
1510 1515
1511 1516 class SALags(Operation):
1512 1517 '''
1513 1518 Function GetMoments()
1514 1519
1515 1520 Input:
1516 1521 self.dataOut.data_pre
1517 1522 self.dataOut.abscissaList
1518 1523 self.dataOut.noise
1519 1524 self.dataOut.normFactor
1520 1525 self.dataOut.data_SNR
1521 1526 self.dataOut.groupList
1522 1527 self.dataOut.nChannels
1523 1528
1524 1529 Affected:
1525 1530 self.dataOut.data_param
1526 1531
1527 1532 '''
1528 1533 def run(self, dataOut):
1529 1534 data_acf = dataOut.data_pre[0]
1530 1535 data_ccf = dataOut.data_pre[1]
1531 1536 normFactor_acf = dataOut.normFactor[0]
1532 1537 normFactor_ccf = dataOut.normFactor[1]
1533 1538 pairs_acf = dataOut.groupList[0]
1534 1539 pairs_ccf = dataOut.groupList[1]
1535 1540
1536 1541 nHeights = dataOut.nHeights
1537 1542 absc = dataOut.abscissaList
1538 1543 noise = dataOut.noise
1539 1544 SNR = dataOut.data_SNR
1540 1545 nChannels = dataOut.nChannels
1541 1546 # pairsList = dataOut.groupList
1542 1547 # pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairsList, nChannels)
1543 1548
1544 1549 for l in range(len(pairs_acf)):
1545 1550 data_acf[l,:,:] = data_acf[l,:,:]/normFactor_acf[l,:]
1546 1551
1547 1552 for l in range(len(pairs_ccf)):
1548 1553 data_ccf[l,:,:] = data_ccf[l,:,:]/normFactor_ccf[l,:]
1549 1554
1550 1555 dataOut.data_param = numpy.zeros((len(pairs_ccf)*2 + 1, nHeights))
1551 1556 dataOut.data_param[:-1,:] = self.__calculateTaus(data_acf, data_ccf, absc)
1552 1557 dataOut.data_param[-1,:] = self.__calculateLag1Phase(data_acf, absc)
1553 1558 return
1554 1559
1555 1560 # def __getPairsAutoCorr(self, pairsList, nChannels):
1556 1561 #
1557 1562 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
1558 1563 #
1559 1564 # for l in range(len(pairsList)):
1560 1565 # firstChannel = pairsList[l][0]
1561 1566 # secondChannel = pairsList[l][1]
1562 1567 #
1563 1568 # #Obteniendo pares de Autocorrelacion
1564 1569 # if firstChannel == secondChannel:
1565 1570 # pairsAutoCorr[firstChannel] = int(l)
1566 1571 #
1567 1572 # pairsAutoCorr = pairsAutoCorr.astype(int)
1568 1573 #
1569 1574 # pairsCrossCorr = range(len(pairsList))
1570 1575 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
1571 1576 #
1572 1577 # return pairsAutoCorr, pairsCrossCorr
1573 1578
1574 1579 def __calculateTaus(self, data_acf, data_ccf, lagRange):
1575 1580
1576 1581 lag0 = data_acf.shape[1]/2
1577 1582 #Funcion de Autocorrelacion
1578 1583 mean_acf = stats.nanmean(data_acf, axis = 0)
1579 1584
1580 1585 #Obtencion Indice de TauCross
1581 1586 ind_ccf = data_ccf.argmax(axis = 1)
1582 1587 #Obtencion Indice de TauAuto
1583 1588 ind_acf = numpy.zeros(ind_ccf.shape,dtype = 'int')
1584 1589 ccf_lag0 = data_ccf[:,lag0,:]
1585 1590
1586 1591 for i in range(ccf_lag0.shape[0]):
1587 1592 ind_acf[i,:] = numpy.abs(mean_acf - ccf_lag0[i,:]).argmin(axis = 0)
1588 1593
1589 1594 #Obtencion de TauCross y TauAuto
1590 1595 tau_ccf = lagRange[ind_ccf]
1591 1596 tau_acf = lagRange[ind_acf]
1592 1597
1593 1598 Nan1, Nan2 = numpy.where(tau_ccf == lagRange[0])
1594 1599
1595 1600 tau_ccf[Nan1,Nan2] = numpy.nan
1596 1601 tau_acf[Nan1,Nan2] = numpy.nan
1597 1602 tau = numpy.vstack((tau_ccf,tau_acf))
1598 1603
1599 1604 return tau
1600 1605
1601 1606 def __calculateLag1Phase(self, data, lagTRange):
1602 1607 data1 = stats.nanmean(data, axis = 0)
1603 1608 lag1 = numpy.where(lagTRange == 0)[0][0] + 1
1604 1609
1605 1610 phase = numpy.angle(data1[lag1,:])
1606 1611
1607 1612 return phase
1608 1613
1609 1614 class SpectralFitting(Operation):
1610 1615 '''
1611 1616 Function GetMoments()
1612 1617
1613 1618 Input:
1614 1619 Output:
1615 1620 Variables modified:
1616 1621 '''
1617 1622
1618 1623 def run(self, dataOut, getSNR = True, path=None, file=None, groupList=None):
1619 1624
1620 1625
1621 1626 if path != None:
1622 1627 sys.path.append(path)
1623 1628 self.dataOut.library = importlib.import_module(file)
1624 1629
1625 1630 #To be inserted as a parameter
1626 1631 groupArray = numpy.array(groupList)
1627 1632 # groupArray = numpy.array([[0,1],[2,3]])
1628 1633 self.dataOut.groupList = groupArray
1629 1634
1630 1635 nGroups = groupArray.shape[0]
1631 1636 nChannels = self.dataIn.nChannels
1632 1637 nHeights=self.dataIn.heightList.size
1633 1638
1634 1639 #Parameters Array
1635 1640 self.dataOut.data_param = None
1636 1641
1637 1642 #Set constants
1638 1643 constants = self.dataOut.library.setConstants(self.dataIn)
1639 1644 self.dataOut.constants = constants
1640 1645 M = self.dataIn.normFactor
1641 1646 N = self.dataIn.nFFTPoints
1642 1647 ippSeconds = self.dataIn.ippSeconds
1643 1648 K = self.dataIn.nIncohInt
1644 1649 pairsArray = numpy.array(self.dataIn.pairsList)
1645 1650
1646 1651 #List of possible combinations
1647 1652 listComb = itertools.combinations(numpy.arange(groupArray.shape[1]),2)
1648 1653 indCross = numpy.zeros(len(list(listComb)), dtype = 'int')
1649 1654
1650 1655 if getSNR:
1651 1656 listChannels = groupArray.reshape((groupArray.size))
1652 1657 listChannels.sort()
1653 1658 noise = self.dataIn.getNoise()
1654 1659 self.dataOut.data_SNR = self.__getSNR(self.dataIn.data_spc[listChannels,:,:], noise[listChannels])
1655 1660
1656 1661 for i in range(nGroups):
1657 1662 coord = groupArray[i,:]
1658 1663
1659 1664 #Input data array
1660 1665 data = self.dataIn.data_spc[coord,:,:]/(M*N)
1661 1666 data = data.reshape((data.shape[0]*data.shape[1],data.shape[2]))
1662 1667
1663 1668 #Cross Spectra data array for Covariance Matrixes
1664 1669 ind = 0
1665 1670 for pairs in listComb:
1666 1671 pairsSel = numpy.array([coord[x],coord[y]])
1667 1672 indCross[ind] = int(numpy.where(numpy.all(pairsArray == pairsSel, axis = 1))[0][0])
1668 1673 ind += 1
1669 1674 dataCross = self.dataIn.data_cspc[indCross,:,:]/(M*N)
1670 1675 dataCross = dataCross**2/K
1671 1676
1672 1677 for h in range(nHeights):
1673 1678 # print self.dataOut.heightList[h]
1674 1679
1675 1680 #Input
1676 1681 d = data[:,h]
1677 1682
1678 1683 #Covariance Matrix
1679 1684 D = numpy.diag(d**2/K)
1680 1685 ind = 0
1681 1686 for pairs in listComb:
1682 1687 #Coordinates in Covariance Matrix
1683 1688 x = pairs[0]
1684 1689 y = pairs[1]
1685 1690 #Channel Index
1686 1691 S12 = dataCross[ind,:,h]
1687 1692 D12 = numpy.diag(S12)
1688 1693 #Completing Covariance Matrix with Cross Spectras
1689 1694 D[x*N:(x+1)*N,y*N:(y+1)*N] = D12
1690 1695 D[y*N:(y+1)*N,x*N:(x+1)*N] = D12
1691 1696 ind += 1
1692 1697 Dinv=numpy.linalg.inv(D)
1693 1698 L=numpy.linalg.cholesky(Dinv)
1694 1699 LT=L.T
1695 1700
1696 1701 dp = numpy.dot(LT,d)
1697 1702
1698 1703 #Initial values
1699 1704 data_spc = self.dataIn.data_spc[coord,:,h]
1700 1705
1701 1706 if (h>0)and(error1[3]<5):
1702 1707 p0 = self.dataOut.data_param[i,:,h-1]
1703 1708 else:
1704 1709 p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants, i))
1705 1710
1706 1711 try:
1707 1712 #Least Squares
1708 1713 minp,covp,infodict,mesg,ier = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants),full_output=True)
1709 1714 # minp,covp = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants))
1710 1715 #Chi square error
1711 1716 error0 = numpy.sum(infodict['fvec']**2)/(2*N)
1712 1717 #Error with Jacobian
1713 1718 error1 = self.dataOut.library.errorFunction(minp,constants,LT)
1714 1719 except:
1715 1720 minp = p0*numpy.nan
1716 1721 error0 = numpy.nan
1717 1722 error1 = p0*numpy.nan
1718 1723
1719 1724 #Save
1720 1725 if self.dataOut.data_param is None:
1721 1726 self.dataOut.data_param = numpy.zeros((nGroups, p0.size, nHeights))*numpy.nan
1722 1727 self.dataOut.data_error = numpy.zeros((nGroups, p0.size + 1, nHeights))*numpy.nan
1723 1728
1724 1729 self.dataOut.data_error[i,:,h] = numpy.hstack((error0,error1))
1725 1730 self.dataOut.data_param[i,:,h] = minp
1726 1731 return
1727 1732
1728 1733 def __residFunction(self, p, dp, LT, constants):
1729 1734
1730 1735 fm = self.dataOut.library.modelFunction(p, constants)
1731 1736 fmp=numpy.dot(LT,fm)
1732 1737
1733 1738 return dp-fmp
1734 1739
1735 1740 def __getSNR(self, z, noise):
1736 1741
1737 1742 avg = numpy.average(z, axis=1)
1738 1743 SNR = (avg.T-noise)/noise
1739 1744 SNR = SNR.T
1740 1745 return SNR
1741 1746
1742 1747 def __chisq(p,chindex,hindex):
1743 1748 #similar to Resid but calculates CHI**2
1744 1749 [LT,d,fm]=setupLTdfm(p,chindex,hindex)
1745 1750 dp=numpy.dot(LT,d)
1746 1751 fmp=numpy.dot(LT,fm)
1747 1752 chisq=numpy.dot((dp-fmp).T,(dp-fmp))
1748 1753 return chisq
1749 1754
1750 1755 class WindProfiler(Operation):
1751 1756
1752 1757 __isConfig = False
1753 1758
1754 1759 __initime = None
1755 1760 __lastdatatime = None
1756 1761 __integrationtime = None
1757 1762
1758 1763 __buffer = None
1759 1764
1760 1765 __dataReady = False
1761 1766
1762 1767 __firstdata = None
1763 1768
1764 1769 n = None
1765 1770
1766 1771 def __init__(self, **kwargs):
1767 1772 Operation.__init__(self, **kwargs)
1768 1773
1769 1774 def __calculateCosDir(self, elev, azim):
1770 1775 zen = (90 - elev)*numpy.pi/180
1771 1776 azim = azim*numpy.pi/180
1772 1777 cosDirX = numpy.sqrt((1-numpy.cos(zen)**2)/((1+numpy.tan(azim)**2)))
1773 1778 cosDirY = numpy.sqrt(1-numpy.cos(zen)**2-cosDirX**2)
1774 1779
1775 1780 signX = numpy.sign(numpy.cos(azim))
1776 1781 signY = numpy.sign(numpy.sin(azim))
1777 1782
1778 1783 cosDirX = numpy.copysign(cosDirX, signX)
1779 1784 cosDirY = numpy.copysign(cosDirY, signY)
1780 1785 return cosDirX, cosDirY
1781 1786
1782 1787 def __calculateAngles(self, theta_x, theta_y, azimuth):
1783 1788
1784 1789 dir_cosw = numpy.sqrt(1-theta_x**2-theta_y**2)
1785 1790 zenith_arr = numpy.arccos(dir_cosw)
1786 1791 azimuth_arr = numpy.arctan2(theta_x,theta_y) + azimuth*math.pi/180
1787 1792
1788 1793 dir_cosu = numpy.sin(azimuth_arr)*numpy.sin(zenith_arr)
1789 1794 dir_cosv = numpy.cos(azimuth_arr)*numpy.sin(zenith_arr)
1790 1795
1791 1796 return azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw
1792 1797
1793 1798 def __calculateMatA(self, dir_cosu, dir_cosv, dir_cosw, horOnly):
1794 1799
1795 1800 #
1796 1801 if horOnly:
1797 1802 A = numpy.c_[dir_cosu,dir_cosv]
1798 1803 else:
1799 1804 A = numpy.c_[dir_cosu,dir_cosv,dir_cosw]
1800 1805 A = numpy.asmatrix(A)
1801 1806 A1 = numpy.linalg.inv(A.transpose()*A)*A.transpose()
1802 1807
1803 1808 return A1
1804 1809
1805 1810 def __correctValues(self, heiRang, phi, velRadial, SNR):
1806 1811 listPhi = phi.tolist()
1807 1812 maxid = listPhi.index(max(listPhi))
1808 1813 minid = listPhi.index(min(listPhi))
1809 1814
1810 1815 rango = list(range(len(phi)))
1811 1816 # rango = numpy.delete(rango,maxid)
1812 1817
1813 1818 heiRang1 = heiRang*math.cos(phi[maxid])
1814 1819 heiRangAux = heiRang*math.cos(phi[minid])
1815 1820 indOut = (heiRang1 < heiRangAux[0]).nonzero()
1816 1821 heiRang1 = numpy.delete(heiRang1,indOut)
1817 1822
1818 1823 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
1819 1824 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
1820 1825
1821 1826 for i in rango:
1822 1827 x = heiRang*math.cos(phi[i])
1823 1828 y1 = velRadial[i,:]
1824 1829 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
1825 1830
1826 1831 x1 = heiRang1
1827 1832 y11 = f1(x1)
1828 1833
1829 1834 y2 = SNR[i,:]
1830 1835 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
1831 1836 y21 = f2(x1)
1832 1837
1833 1838 velRadial1[i,:] = y11
1834 1839 SNR1[i,:] = y21
1835 1840
1836 1841 return heiRang1, velRadial1, SNR1
1837 1842
1838 1843 def __calculateVelUVW(self, A, velRadial):
1839 1844
1840 1845 #Operacion Matricial
1841 1846 # velUVW = numpy.zeros((velRadial.shape[1],3))
1842 1847 # for ind in range(velRadial.shape[1]):
1843 1848 # velUVW[ind,:] = numpy.dot(A,velRadial[:,ind])
1844 1849 # velUVW = velUVW.transpose()
1845 1850 velUVW = numpy.zeros((A.shape[0],velRadial.shape[1]))
1846 1851 velUVW[:,:] = numpy.dot(A,velRadial)
1847 1852
1848 1853
1849 1854 return velUVW
1850 1855
1851 1856 # def techniqueDBS(self, velRadial0, dirCosx, disrCosy, azimuth, correct, horizontalOnly, heiRang, SNR0):
1852 1857
1853 1858 def techniqueDBS(self, kwargs):
1854 1859 """
1855 1860 Function that implements Doppler Beam Swinging (DBS) technique.
1856 1861
1857 1862 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
1858 1863 Direction correction (if necessary), Ranges and SNR
1859 1864
1860 1865 Output: Winds estimation (Zonal, Meridional and Vertical)
1861 1866
1862 1867 Parameters affected: Winds, height range, SNR
1863 1868 """
1864 1869 velRadial0 = kwargs['velRadial']
1865 1870 heiRang = kwargs['heightList']
1866 1871 SNR0 = kwargs['SNR']
1867 1872
1868 1873 if 'dirCosx' in kwargs and 'dirCosy' in kwargs:
1869 1874 theta_x = numpy.array(kwargs['dirCosx'])
1870 1875 theta_y = numpy.array(kwargs['dirCosy'])
1871 1876 else:
1872 1877 elev = numpy.array(kwargs['elevation'])
1873 1878 azim = numpy.array(kwargs['azimuth'])
1874 1879 theta_x, theta_y = self.__calculateCosDir(elev, azim)
1875 1880 azimuth = kwargs['correctAzimuth']
1876 1881 if 'horizontalOnly' in kwargs:
1877 1882 horizontalOnly = kwargs['horizontalOnly']
1878 1883 else: horizontalOnly = False
1879 1884 if 'correctFactor' in kwargs:
1880 1885 correctFactor = kwargs['correctFactor']
1881 1886 else: correctFactor = 1
1882 1887 if 'channelList' in kwargs:
1883 1888 channelList = kwargs['channelList']
1884 1889 if len(channelList) == 2:
1885 1890 horizontalOnly = True
1886 1891 arrayChannel = numpy.array(channelList)
1887 1892 param = param[arrayChannel,:,:]
1888 1893 theta_x = theta_x[arrayChannel]
1889 1894 theta_y = theta_y[arrayChannel]
1890 1895
1891 1896 azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw = self.__calculateAngles(theta_x, theta_y, azimuth)
1892 1897 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, zenith_arr, correctFactor*velRadial0, SNR0)
1893 1898 A = self.__calculateMatA(dir_cosu, dir_cosv, dir_cosw, horizontalOnly)
1894 1899
1895 1900 #Calculo de Componentes de la velocidad con DBS
1896 1901 winds = self.__calculateVelUVW(A,velRadial1)
1897 1902
1898 1903 return winds, heiRang1, SNR1
1899 1904
1900 1905 def __calculateDistance(self, posx, posy, pairs_ccf, azimuth = None):
1901 1906
1902 1907 nPairs = len(pairs_ccf)
1903 1908 posx = numpy.asarray(posx)
1904 1909 posy = numpy.asarray(posy)
1905 1910
1906 1911 #Rotacion Inversa para alinear con el azimuth
1907 1912 if azimuth!= None:
1908 1913 azimuth = azimuth*math.pi/180
1909 1914 posx1 = posx*math.cos(azimuth) + posy*math.sin(azimuth)
1910 1915 posy1 = -posx*math.sin(azimuth) + posy*math.cos(azimuth)
1911 1916 else:
1912 1917 posx1 = posx
1913 1918 posy1 = posy
1914 1919
1915 1920 #Calculo de Distancias
1916 1921 distx = numpy.zeros(nPairs)
1917 1922 disty = numpy.zeros(nPairs)
1918 1923 dist = numpy.zeros(nPairs)
1919 1924 ang = numpy.zeros(nPairs)
1920 1925
1921 1926 for i in range(nPairs):
1922 1927 distx[i] = posx1[pairs_ccf[i][1]] - posx1[pairs_ccf[i][0]]
1923 1928 disty[i] = posy1[pairs_ccf[i][1]] - posy1[pairs_ccf[i][0]]
1924 1929 dist[i] = numpy.sqrt(distx[i]**2 + disty[i]**2)
1925 1930 ang[i] = numpy.arctan2(disty[i],distx[i])
1926 1931
1927 1932 return distx, disty, dist, ang
1928 1933 #Calculo de Matrices
1929 1934 # nPairs = len(pairs)
1930 1935 # ang1 = numpy.zeros((nPairs, 2, 1))
1931 1936 # dist1 = numpy.zeros((nPairs, 2, 1))
1932 1937 #
1933 1938 # for j in range(nPairs):
1934 1939 # dist1[j,0,0] = dist[pairs[j][0]]
1935 1940 # dist1[j,1,0] = dist[pairs[j][1]]
1936 1941 # ang1[j,0,0] = ang[pairs[j][0]]
1937 1942 # ang1[j,1,0] = ang[pairs[j][1]]
1938 1943 #
1939 1944 # return distx,disty, dist1,ang1
1940 1945
1941 1946
1942 1947 def __calculateVelVer(self, phase, lagTRange, _lambda):
1943 1948
1944 1949 Ts = lagTRange[1] - lagTRange[0]
1945 1950 velW = -_lambda*phase/(4*math.pi*Ts)
1946 1951
1947 1952 return velW
1948 1953
1949 1954 def __calculateVelHorDir(self, dist, tau1, tau2, ang):
1950 1955 nPairs = tau1.shape[0]
1951 1956 nHeights = tau1.shape[1]
1952 1957 vel = numpy.zeros((nPairs,3,nHeights))
1953 1958 dist1 = numpy.reshape(dist, (dist.size,1))
1954 1959
1955 1960 angCos = numpy.cos(ang)
1956 1961 angSin = numpy.sin(ang)
1957 1962
1958 1963 vel0 = dist1*tau1/(2*tau2**2)
1959 1964 vel[:,0,:] = (vel0*angCos).sum(axis = 1)
1960 1965 vel[:,1,:] = (vel0*angSin).sum(axis = 1)
1961 1966
1962 1967 ind = numpy.where(numpy.isinf(vel))
1963 1968 vel[ind] = numpy.nan
1964 1969
1965 1970 return vel
1966 1971
1967 1972 # def __getPairsAutoCorr(self, pairsList, nChannels):
1968 1973 #
1969 1974 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
1970 1975 #
1971 1976 # for l in range(len(pairsList)):
1972 1977 # firstChannel = pairsList[l][0]
1973 1978 # secondChannel = pairsList[l][1]
1974 1979 #
1975 1980 # #Obteniendo pares de Autocorrelacion
1976 1981 # if firstChannel == secondChannel:
1977 1982 # pairsAutoCorr[firstChannel] = int(l)
1978 1983 #
1979 1984 # pairsAutoCorr = pairsAutoCorr.astype(int)
1980 1985 #
1981 1986 # pairsCrossCorr = range(len(pairsList))
1982 1987 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
1983 1988 #
1984 1989 # return pairsAutoCorr, pairsCrossCorr
1985 1990
1986 1991 # def techniqueSA(self, pairsSelected, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, lagTRange, correctFactor):
1987 1992 def techniqueSA(self, kwargs):
1988 1993
1989 1994 """
1990 1995 Function that implements Spaced Antenna (SA) technique.
1991 1996
1992 1997 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
1993 1998 Direction correction (if necessary), Ranges and SNR
1994 1999
1995 2000 Output: Winds estimation (Zonal, Meridional and Vertical)
1996 2001
1997 2002 Parameters affected: Winds
1998 2003 """
1999 2004 position_x = kwargs['positionX']
2000 2005 position_y = kwargs['positionY']
2001 2006 azimuth = kwargs['azimuth']
2002 2007
2003 2008 if 'correctFactor' in kwargs:
2004 2009 correctFactor = kwargs['correctFactor']
2005 2010 else:
2006 2011 correctFactor = 1
2007 2012
2008 2013 groupList = kwargs['groupList']
2009 2014 pairs_ccf = groupList[1]
2010 2015 tau = kwargs['tau']
2011 2016 _lambda = kwargs['_lambda']
2012 2017
2013 2018 #Cross Correlation pairs obtained
2014 2019 # pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairssList, nChannels)
2015 2020 # pairsArray = numpy.array(pairsList)[pairsCrossCorr]
2016 2021 # pairsSelArray = numpy.array(pairsSelected)
2017 2022 # pairs = []
2018 2023 #
2019 2024 # #Wind estimation pairs obtained
2020 2025 # for i in range(pairsSelArray.shape[0]/2):
2021 2026 # ind1 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i], axis = 1))[0][0]
2022 2027 # ind2 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i + 1], axis = 1))[0][0]
2023 2028 # pairs.append((ind1,ind2))
2024 2029
2025 2030 indtau = tau.shape[0]/2
2026 2031 tau1 = tau[:indtau,:]
2027 2032 tau2 = tau[indtau:-1,:]
2028 2033 # tau1 = tau1[pairs,:]
2029 2034 # tau2 = tau2[pairs,:]
2030 2035 phase1 = tau[-1,:]
2031 2036
2032 2037 #---------------------------------------------------------------------
2033 2038 #Metodo Directo
2034 2039 distx, disty, dist, ang = self.__calculateDistance(position_x, position_y, pairs_ccf,azimuth)
2035 2040 winds = self.__calculateVelHorDir(dist, tau1, tau2, ang)
2036 2041 winds = stats.nanmean(winds, axis=0)
2037 2042 #---------------------------------------------------------------------
2038 2043 #Metodo General
2039 2044 # distx, disty, dist = self.calculateDistance(position_x,position_y,pairsCrossCorr, pairsList, azimuth)
2040 2045 # #Calculo Coeficientes de Funcion de Correlacion
2041 2046 # F,G,A,B,H = self.calculateCoef(tau1,tau2,distx,disty,n)
2042 2047 # #Calculo de Velocidades
2043 2048 # winds = self.calculateVelUV(F,G,A,B,H)
2044 2049
2045 2050 #---------------------------------------------------------------------
2046 2051 winds[2,:] = self.__calculateVelVer(phase1, lagTRange, _lambda)
2047 2052 winds = correctFactor*winds
2048 2053 return winds
2049 2054
2050 2055 def __checkTime(self, currentTime, paramInterval, outputInterval):
2051 2056
2052 2057 dataTime = currentTime + paramInterval
2053 2058 deltaTime = dataTime - self.__initime
2054 2059
2055 2060 if deltaTime >= outputInterval or deltaTime < 0:
2056 2061 self.__dataReady = True
2057 2062 return
2058 2063
2059 2064 def techniqueMeteors(self, arrayMeteor, meteorThresh, heightMin, heightMax):
2060 2065 '''
2061 2066 Function that implements winds estimation technique with detected meteors.
2062 2067
2063 2068 Input: Detected meteors, Minimum meteor quantity to wind estimation
2064 2069
2065 2070 Output: Winds estimation (Zonal and Meridional)
2066 2071
2067 2072 Parameters affected: Winds
2068 2073 '''
2069 2074 # print arrayMeteor.shape
2070 2075 #Settings
2071 2076 nInt = (heightMax - heightMin)/2
2072 2077 # print nInt
2073 2078 nInt = int(nInt)
2074 2079 # print nInt
2075 2080 winds = numpy.zeros((2,nInt))*numpy.nan
2076 2081
2077 2082 #Filter errors
2078 2083 error = numpy.where(arrayMeteor[:,-1] == 0)[0]
2079 2084 finalMeteor = arrayMeteor[error,:]
2080 2085
2081 2086 #Meteor Histogram
2082 2087 finalHeights = finalMeteor[:,2]
2083 2088 hist = numpy.histogram(finalHeights, bins = nInt, range = (heightMin,heightMax))
2084 2089 nMeteorsPerI = hist[0]
2085 2090 heightPerI = hist[1]
2086 2091
2087 2092 #Sort of meteors
2088 2093 indSort = finalHeights.argsort()
2089 2094 finalMeteor2 = finalMeteor[indSort,:]
2090 2095
2091 2096 # Calculating winds
2092 2097 ind1 = 0
2093 2098 ind2 = 0
2094 2099
2095 2100 for i in range(nInt):
2096 2101 nMet = nMeteorsPerI[i]
2097 2102 ind1 = ind2
2098 2103 ind2 = ind1 + nMet
2099 2104
2100 2105 meteorAux = finalMeteor2[ind1:ind2,:]
2101 2106
2102 2107 if meteorAux.shape[0] >= meteorThresh:
2103 2108 vel = meteorAux[:, 6]
2104 2109 zen = meteorAux[:, 4]*numpy.pi/180
2105 2110 azim = meteorAux[:, 3]*numpy.pi/180
2106 2111
2107 2112 n = numpy.cos(zen)
2108 2113 # m = (1 - n**2)/(1 - numpy.tan(azim)**2)
2109 2114 # l = m*numpy.tan(azim)
2110 2115 l = numpy.sin(zen)*numpy.sin(azim)
2111 2116 m = numpy.sin(zen)*numpy.cos(azim)
2112 2117
2113 2118 A = numpy.vstack((l, m)).transpose()
2114 2119 A1 = numpy.dot(numpy.linalg.inv( numpy.dot(A.transpose(),A) ),A.transpose())
2115 2120 windsAux = numpy.dot(A1, vel)
2116 2121
2117 2122 winds[0,i] = windsAux[0]
2118 2123 winds[1,i] = windsAux[1]
2119 2124
2120 2125 return winds, heightPerI[:-1]
2121 2126
2122 2127 def techniqueNSM_SA(self, **kwargs):
2123 2128 metArray = kwargs['metArray']
2124 2129 heightList = kwargs['heightList']
2125 2130 timeList = kwargs['timeList']
2126 2131
2127 2132 rx_location = kwargs['rx_location']
2128 2133 groupList = kwargs['groupList']
2129 2134 azimuth = kwargs['azimuth']
2130 2135 dfactor = kwargs['dfactor']
2131 2136 k = kwargs['k']
2132 2137
2133 2138 azimuth1, dist = self.__calculateAzimuth1(rx_location, groupList, azimuth)
2134 2139 d = dist*dfactor
2135 2140 #Phase calculation
2136 2141 metArray1 = self.__getPhaseSlope(metArray, heightList, timeList)
2137 2142
2138 2143 metArray1[:,-2] = metArray1[:,-2]*metArray1[:,2]*1000/(k*d[metArray1[:,1].astype(int)]) #angles into velocities
2139 2144
2140 2145 velEst = numpy.zeros((heightList.size,2))*numpy.nan
2141 2146 azimuth1 = azimuth1*numpy.pi/180
2142 2147
2143 2148 for i in range(heightList.size):
2144 2149 h = heightList[i]
2145 2150 indH = numpy.where((metArray1[:,2] == h)&(numpy.abs(metArray1[:,-2]) < 100))[0]
2146 2151 metHeight = metArray1[indH,:]
2147 2152 if metHeight.shape[0] >= 2:
2148 2153 velAux = numpy.asmatrix(metHeight[:,-2]).T #Radial Velocities
2149 2154 iazim = metHeight[:,1].astype(int)
2150 2155 azimAux = numpy.asmatrix(azimuth1[iazim]).T #Azimuths
2151 2156 A = numpy.hstack((numpy.cos(azimAux),numpy.sin(azimAux)))
2152 2157 A = numpy.asmatrix(A)
2153 2158 A1 = numpy.linalg.pinv(A.transpose()*A)*A.transpose()
2154 2159 velHor = numpy.dot(A1,velAux)
2155 2160
2156 2161 velEst[i,:] = numpy.squeeze(velHor)
2157 2162 return velEst
2158 2163
2159 2164 def __getPhaseSlope(self, metArray, heightList, timeList):
2160 2165 meteorList = []
2161 2166 #utctime sec1 height SNR velRad ph0 ph1 ph2 coh0 coh1 coh2
2162 2167 #Putting back together the meteor matrix
2163 2168 utctime = metArray[:,0]
2164 2169 uniqueTime = numpy.unique(utctime)
2165 2170
2166 2171 phaseDerThresh = 0.5
2167 2172 ippSeconds = timeList[1] - timeList[0]
2168 2173 sec = numpy.where(timeList>1)[0][0]
2169 2174 nPairs = metArray.shape[1] - 6
2170 2175 nHeights = len(heightList)
2171 2176
2172 2177 for t in uniqueTime:
2173 2178 metArray1 = metArray[utctime==t,:]
2174 2179 # phaseDerThresh = numpy.pi/4 #reducir Phase thresh
2175 2180 tmet = metArray1[:,1].astype(int)
2176 2181 hmet = metArray1[:,2].astype(int)
2177 2182
2178 2183 metPhase = numpy.zeros((nPairs, heightList.size, timeList.size - 1))
2179 2184 metPhase[:,:] = numpy.nan
2180 2185 metPhase[:,hmet,tmet] = metArray1[:,6:].T
2181 2186
2182 2187 #Delete short trails
2183 2188 metBool = ~numpy.isnan(metPhase[0,:,:])
2184 2189 heightVect = numpy.sum(metBool, axis = 1)
2185 2190 metBool[heightVect<sec,:] = False
2186 2191 metPhase[:,heightVect<sec,:] = numpy.nan
2187 2192
2188 2193 #Derivative
2189 2194 metDer = numpy.abs(metPhase[:,:,1:] - metPhase[:,:,:-1])
2190 2195 phDerAux = numpy.dstack((numpy.full((nPairs,nHeights,1), False, dtype=bool),metDer > phaseDerThresh))
2191 2196 metPhase[phDerAux] = numpy.nan
2192 2197
2193 2198 #--------------------------METEOR DETECTION -----------------------------------------
2194 2199 indMet = numpy.where(numpy.any(metBool,axis=1))[0]
2195 2200
2196 2201 for p in numpy.arange(nPairs):
2197 2202 phase = metPhase[p,:,:]
2198 2203 phDer = metDer[p,:,:]
2199 2204
2200 2205 for h in indMet:
2201 2206 height = heightList[h]
2202 2207 phase1 = phase[h,:] #82
2203 2208 phDer1 = phDer[h,:]
2204 2209
2205 2210 phase1[~numpy.isnan(phase1)] = numpy.unwrap(phase1[~numpy.isnan(phase1)]) #Unwrap
2206 2211
2207 2212 indValid = numpy.where(~numpy.isnan(phase1))[0]
2208 2213 initMet = indValid[0]
2209 2214 endMet = 0
2210 2215
2211 2216 for i in range(len(indValid)-1):
2212 2217
2213 2218 #Time difference
2214 2219 inow = indValid[i]
2215 2220 inext = indValid[i+1]
2216 2221 idiff = inext - inow
2217 2222 #Phase difference
2218 2223 phDiff = numpy.abs(phase1[inext] - phase1[inow])
2219 2224
2220 2225 if idiff>sec or phDiff>numpy.pi/4 or inext==indValid[-1]: #End of Meteor
2221 2226 sizeTrail = inow - initMet + 1
2222 2227 if sizeTrail>3*sec: #Too short meteors
2223 2228 x = numpy.arange(initMet,inow+1)*ippSeconds
2224 2229 y = phase1[initMet:inow+1]
2225 2230 ynnan = ~numpy.isnan(y)
2226 2231 x = x[ynnan]
2227 2232 y = y[ynnan]
2228 2233 slope, intercept, r_value, p_value, std_err = stats.linregress(x,y)
2229 2234 ylin = x*slope + intercept
2230 2235 rsq = r_value**2
2231 2236 if rsq > 0.5:
2232 2237 vel = slope#*height*1000/(k*d)
2233 2238 estAux = numpy.array([utctime,p,height, vel, rsq])
2234 2239 meteorList.append(estAux)
2235 2240 initMet = inext
2236 2241 metArray2 = numpy.array(meteorList)
2237 2242
2238 2243 return metArray2
2239 2244
2240 2245 def __calculateAzimuth1(self, rx_location, pairslist, azimuth0):
2241 2246
2242 2247 azimuth1 = numpy.zeros(len(pairslist))
2243 2248 dist = numpy.zeros(len(pairslist))
2244 2249
2245 2250 for i in range(len(rx_location)):
2246 2251 ch0 = pairslist[i][0]
2247 2252 ch1 = pairslist[i][1]
2248 2253
2249 2254 diffX = rx_location[ch0][0] - rx_location[ch1][0]
2250 2255 diffY = rx_location[ch0][1] - rx_location[ch1][1]
2251 2256 azimuth1[i] = numpy.arctan2(diffY,diffX)*180/numpy.pi
2252 2257 dist[i] = numpy.sqrt(diffX**2 + diffY**2)
2253 2258
2254 2259 azimuth1 -= azimuth0
2255 2260 return azimuth1, dist
2256 2261
2257 2262 def techniqueNSM_DBS(self, **kwargs):
2258 2263 metArray = kwargs['metArray']
2259 2264 heightList = kwargs['heightList']
2260 2265 timeList = kwargs['timeList']
2261 2266 azimuth = kwargs['azimuth']
2262 2267 theta_x = numpy.array(kwargs['theta_x'])
2263 2268 theta_y = numpy.array(kwargs['theta_y'])
2264 2269
2265 2270 utctime = metArray[:,0]
2266 2271 cmet = metArray[:,1].astype(int)
2267 2272 hmet = metArray[:,3].astype(int)
2268 2273 SNRmet = metArray[:,4]
2269 2274 vmet = metArray[:,5]
2270 2275 spcmet = metArray[:,6]
2271 2276
2272 2277 nChan = numpy.max(cmet) + 1
2273 2278 nHeights = len(heightList)
2274 2279
2275 2280 azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw = self.__calculateAngles(theta_x, theta_y, azimuth)
2276 2281 hmet = heightList[hmet]
2277 2282 h1met = hmet*numpy.cos(zenith_arr[cmet]) #Corrected heights
2278 2283
2279 2284 velEst = numpy.zeros((heightList.size,2))*numpy.nan
2280 2285
2281 2286 for i in range(nHeights - 1):
2282 2287 hmin = heightList[i]
2283 2288 hmax = heightList[i + 1]
2284 2289
2285 2290 thisH = (h1met>=hmin) & (h1met<hmax) & (cmet!=2) & (SNRmet>8) & (vmet<50) & (spcmet<10)
2286 2291 indthisH = numpy.where(thisH)
2287 2292
2288 2293 if numpy.size(indthisH) > 3:
2289 2294
2290 2295 vel_aux = vmet[thisH]
2291 2296 chan_aux = cmet[thisH]
2292 2297 cosu_aux = dir_cosu[chan_aux]
2293 2298 cosv_aux = dir_cosv[chan_aux]
2294 2299 cosw_aux = dir_cosw[chan_aux]
2295 2300
2296 2301 nch = numpy.size(numpy.unique(chan_aux))
2297 2302 if nch > 1:
2298 2303 A = self.__calculateMatA(cosu_aux, cosv_aux, cosw_aux, True)
2299 2304 velEst[i,:] = numpy.dot(A,vel_aux)
2300 2305
2301 2306 return velEst
2302 2307
2303 2308 def run(self, dataOut, technique, nHours=1, hmin=70, hmax=110, **kwargs):
2304 2309
2305 2310 param = dataOut.data_param
2306 2311 if dataOut.abscissaList != None:
2307 2312 absc = dataOut.abscissaList[:-1]
2308 2313 # noise = dataOut.noise
2309 2314 heightList = dataOut.heightList
2310 2315 SNR = dataOut.data_SNR
2311 2316
2312 2317 if technique == 'DBS':
2313 2318
2314 2319 kwargs['velRadial'] = param[:,1,:] #Radial velocity
2315 2320 kwargs['heightList'] = heightList
2316 2321 kwargs['SNR'] = SNR
2317 2322
2318 2323 dataOut.data_output, dataOut.heightList, dataOut.data_SNR = self.techniqueDBS(kwargs) #DBS Function
2319 2324 dataOut.utctimeInit = dataOut.utctime
2320 2325 dataOut.outputInterval = dataOut.paramInterval
2321 2326
2322 2327 elif technique == 'SA':
2323 2328
2324 2329 #Parameters
2325 2330 # position_x = kwargs['positionX']
2326 2331 # position_y = kwargs['positionY']
2327 2332 # azimuth = kwargs['azimuth']
2328 2333 #
2329 2334 # if kwargs.has_key('crosspairsList'):
2330 2335 # pairs = kwargs['crosspairsList']
2331 2336 # else:
2332 2337 # pairs = None
2333 2338 #
2334 2339 # if kwargs.has_key('correctFactor'):
2335 2340 # correctFactor = kwargs['correctFactor']
2336 2341 # else:
2337 2342 # correctFactor = 1
2338 2343
2339 2344 # tau = dataOut.data_param
2340 2345 # _lambda = dataOut.C/dataOut.frequency
2341 2346 # pairsList = dataOut.groupList
2342 2347 # nChannels = dataOut.nChannels
2343 2348
2344 2349 kwargs['groupList'] = dataOut.groupList
2345 2350 kwargs['tau'] = dataOut.data_param
2346 2351 kwargs['_lambda'] = dataOut.C/dataOut.frequency
2347 2352 # dataOut.data_output = self.techniqueSA(pairs, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, absc, correctFactor)
2348 2353 dataOut.data_output = self.techniqueSA(kwargs)
2349 2354 dataOut.utctimeInit = dataOut.utctime
2350 2355 dataOut.outputInterval = dataOut.timeInterval
2351 2356
2352 2357 elif technique == 'Meteors':
2353 2358 dataOut.flagNoData = True
2354 2359 self.__dataReady = False
2355 2360
2356 2361 if 'nHours' in kwargs:
2357 2362 nHours = kwargs['nHours']
2358 2363 else:
2359 2364 nHours = 1
2360 2365
2361 2366 if 'meteorsPerBin' in kwargs:
2362 2367 meteorThresh = kwargs['meteorsPerBin']
2363 2368 else:
2364 2369 meteorThresh = 6
2365 2370
2366 2371 if 'hmin' in kwargs:
2367 2372 hmin = kwargs['hmin']
2368 2373 else: hmin = 70
2369 2374 if 'hmax' in kwargs:
2370 2375 hmax = kwargs['hmax']
2371 2376 else: hmax = 110
2372 2377
2373 2378 dataOut.outputInterval = nHours*3600
2374 2379
2375 2380 if self.__isConfig == False:
2376 2381 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
2377 2382 #Get Initial LTC time
2378 2383 self.__initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
2379 2384 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
2380 2385
2381 2386 self.__isConfig = True
2382 2387
2383 2388 if self.__buffer is None:
2384 2389 self.__buffer = dataOut.data_param
2385 2390 self.__firstdata = copy.copy(dataOut)
2386 2391
2387 2392 else:
2388 2393 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
2389 2394
2390 2395 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
2391 2396
2392 2397 if self.__dataReady:
2393 2398 dataOut.utctimeInit = self.__initime
2394 2399
2395 2400 self.__initime += dataOut.outputInterval #to erase time offset
2396 2401
2397 2402 dataOut.data_output, dataOut.heightList = self.techniqueMeteors(self.__buffer, meteorThresh, hmin, hmax)
2398 2403 dataOut.flagNoData = False
2399 2404 self.__buffer = None
2400 2405
2401 2406 elif technique == 'Meteors1':
2402 2407 dataOut.flagNoData = True
2403 2408 self.__dataReady = False
2404 2409
2405 2410 if 'nMins' in kwargs:
2406 2411 nMins = kwargs['nMins']
2407 2412 else: nMins = 20
2408 2413 if 'rx_location' in kwargs:
2409 2414 rx_location = kwargs['rx_location']
2410 2415 else: rx_location = [(0,1),(1,1),(1,0)]
2411 2416 if 'azimuth' in kwargs:
2412 2417 azimuth = kwargs['azimuth']
2413 2418 else: azimuth = 51.06
2414 2419 if 'dfactor' in kwargs:
2415 2420 dfactor = kwargs['dfactor']
2416 2421 if 'mode' in kwargs:
2417 2422 mode = kwargs['mode']
2418 2423 if 'theta_x' in kwargs:
2419 2424 theta_x = kwargs['theta_x']
2420 2425 if 'theta_y' in kwargs:
2421 2426 theta_y = kwargs['theta_y']
2422 2427 else: mode = 'SA'
2423 2428
2424 2429 #Borrar luego esto
2425 2430 if dataOut.groupList is None:
2426 2431 dataOut.groupList = [(0,1),(0,2),(1,2)]
2427 2432 groupList = dataOut.groupList
2428 2433 C = 3e8
2429 2434 freq = 50e6
2430 2435 lamb = C/freq
2431 2436 k = 2*numpy.pi/lamb
2432 2437
2433 2438 timeList = dataOut.abscissaList
2434 2439 heightList = dataOut.heightList
2435 2440
2436 2441 if self.__isConfig == False:
2437 2442 dataOut.outputInterval = nMins*60
2438 2443 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
2439 2444 #Get Initial LTC time
2440 2445 initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
2441 2446 minuteAux = initime.minute
2442 2447 minuteNew = int(numpy.floor(minuteAux/nMins)*nMins)
2443 2448 self.__initime = (initime.replace(minute = minuteNew, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
2444 2449
2445 2450 self.__isConfig = True
2446 2451
2447 2452 if self.__buffer is None:
2448 2453 self.__buffer = dataOut.data_param
2449 2454 self.__firstdata = copy.copy(dataOut)
2450 2455
2451 2456 else:
2452 2457 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
2453 2458
2454 2459 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
2455 2460
2456 2461 if self.__dataReady:
2457 2462 dataOut.utctimeInit = self.__initime
2458 2463 self.__initime += dataOut.outputInterval #to erase time offset
2459 2464
2460 2465 metArray = self.__buffer
2461 2466 if mode == 'SA':
2462 2467 dataOut.data_output = self.techniqueNSM_SA(rx_location=rx_location, groupList=groupList, azimuth=azimuth, dfactor=dfactor, k=k,metArray=metArray, heightList=heightList,timeList=timeList)
2463 2468 elif mode == 'DBS':
2464 2469 dataOut.data_output = self.techniqueNSM_DBS(metArray=metArray,heightList=heightList,timeList=timeList, azimuth=azimuth, theta_x=theta_x, theta_y=theta_y)
2465 2470 dataOut.data_output = dataOut.data_output.T
2466 2471 dataOut.flagNoData = False
2467 2472 self.__buffer = None
2468 2473
2469 2474 return
2470 2475
2471 2476 class EWDriftsEstimation(Operation):
2472 2477
2473 2478 def __init__(self, **kwargs):
2474 2479 Operation.__init__(self, **kwargs)
2475 2480
2476 2481 def __correctValues(self, heiRang, phi, velRadial, SNR):
2477 2482 listPhi = phi.tolist()
2478 2483 maxid = listPhi.index(max(listPhi))
2479 2484 minid = listPhi.index(min(listPhi))
2480 2485
2481 2486 rango = list(range(len(phi)))
2482 2487 # rango = numpy.delete(rango,maxid)
2483 2488
2484 2489 heiRang1 = heiRang*math.cos(phi[maxid])
2485 2490 heiRangAux = heiRang*math.cos(phi[minid])
2486 2491 indOut = (heiRang1 < heiRangAux[0]).nonzero()
2487 2492 heiRang1 = numpy.delete(heiRang1,indOut)
2488 2493
2489 2494 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
2490 2495 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
2491 2496
2492 2497 for i in rango:
2493 2498 x = heiRang*math.cos(phi[i])
2494 2499 y1 = velRadial[i,:]
2495 2500 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
2496 2501
2497 2502 x1 = heiRang1
2498 2503 y11 = f1(x1)
2499 2504
2500 2505 y2 = SNR[i,:]
2501 2506 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
2502 2507 y21 = f2(x1)
2503 2508
2504 2509 velRadial1[i,:] = y11
2505 2510 SNR1[i,:] = y21
2506 2511
2507 2512 return heiRang1, velRadial1, SNR1
2508 2513
2509 2514 def run(self, dataOut, zenith, zenithCorrection):
2510 2515 heiRang = dataOut.heightList
2511 2516 velRadial = dataOut.data_param[:,3,:]
2512 2517 SNR = dataOut.data_SNR
2513 2518
2514 2519 zenith = numpy.array(zenith)
2515 2520 zenith -= zenithCorrection
2516 2521 zenith *= numpy.pi/180
2517 2522
2518 2523 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, numpy.abs(zenith), velRadial, SNR)
2519 2524
2520 2525 alp = zenith[0]
2521 2526 bet = zenith[1]
2522 2527
2523 2528 w_w = velRadial1[0,:]
2524 2529 w_e = velRadial1[1,:]
2525 2530
2526 2531 w = (w_w*numpy.sin(bet) - w_e*numpy.sin(alp))/(numpy.cos(alp)*numpy.sin(bet) - numpy.cos(bet)*numpy.sin(alp))
2527 2532 u = (w_w*numpy.cos(bet) - w_e*numpy.cos(alp))/(numpy.sin(alp)*numpy.cos(bet) - numpy.sin(bet)*numpy.cos(alp))
2528 2533
2529 2534 winds = numpy.vstack((u,w))
2530 2535
2531 2536 dataOut.heightList = heiRang1
2532 2537 dataOut.data_output = winds
2533 2538 dataOut.data_SNR = SNR1
2534 2539
2535 2540 dataOut.utctimeInit = dataOut.utctime
2536 2541 dataOut.outputInterval = dataOut.timeInterval
2537 2542 return
2538 2543
2539 2544 #--------------- Non Specular Meteor ----------------
2540 2545
2541 2546 class NonSpecularMeteorDetection(Operation):
2542 2547
2543 2548 def run(self, dataOut, mode, SNRthresh=8, phaseDerThresh=0.5, cohThresh=0.8, allData = False):
2544 2549 data_acf = dataOut.data_pre[0]
2545 2550 data_ccf = dataOut.data_pre[1]
2546 2551 pairsList = dataOut.groupList[1]
2547 2552
2548 2553 lamb = dataOut.C/dataOut.frequency
2549 2554 tSamp = dataOut.ippSeconds*dataOut.nCohInt
2550 2555 paramInterval = dataOut.paramInterval
2551 2556
2552 2557 nChannels = data_acf.shape[0]
2553 2558 nLags = data_acf.shape[1]
2554 2559 nProfiles = data_acf.shape[2]
2555 2560 nHeights = dataOut.nHeights
2556 2561 nCohInt = dataOut.nCohInt
2557 2562 sec = numpy.round(nProfiles/dataOut.paramInterval)
2558 2563 heightList = dataOut.heightList
2559 2564 ippSeconds = dataOut.ippSeconds*dataOut.nCohInt*dataOut.nAvg
2560 2565 utctime = dataOut.utctime
2561 2566
2562 2567 dataOut.abscissaList = numpy.arange(0,paramInterval+ippSeconds,ippSeconds)
2563 2568
2564 2569 #------------------------ SNR --------------------------------------
2565 2570 power = data_acf[:,0,:,:].real
2566 2571 noise = numpy.zeros(nChannels)
2567 2572 SNR = numpy.zeros(power.shape)
2568 2573 for i in range(nChannels):
2569 2574 noise[i] = hildebrand_sekhon(power[i,:], nCohInt)
2570 2575 SNR[i] = (power[i]-noise[i])/noise[i]
2571 2576 SNRm = numpy.nanmean(SNR, axis = 0)
2572 2577 SNRdB = 10*numpy.log10(SNR)
2573 2578
2574 2579 if mode == 'SA':
2575 2580 dataOut.groupList = dataOut.groupList[1]
2576 2581 nPairs = data_ccf.shape[0]
2577 2582 #---------------------- Coherence and Phase --------------------------
2578 2583 phase = numpy.zeros(data_ccf[:,0,:,:].shape)
2579 2584 # phase1 = numpy.copy(phase)
2580 2585 coh1 = numpy.zeros(data_ccf[:,0,:,:].shape)
2581 2586
2582 2587 for p in range(nPairs):
2583 2588 ch0 = pairsList[p][0]
2584 2589 ch1 = pairsList[p][1]
2585 2590 ccf = data_ccf[p,0,:,:]/numpy.sqrt(data_acf[ch0,0,:,:]*data_acf[ch1,0,:,:])
2586 2591 phase[p,:,:] = ndimage.median_filter(numpy.angle(ccf), size = (5,1)) #median filter
2587 2592 # phase1[p,:,:] = numpy.angle(ccf) #median filter
2588 2593 coh1[p,:,:] = ndimage.median_filter(numpy.abs(ccf), 5) #median filter
2589 2594 # coh1[p,:,:] = numpy.abs(ccf) #median filter
2590 2595 coh = numpy.nanmax(coh1, axis = 0)
2591 2596 # struc = numpy.ones((5,1))
2592 2597 # coh = ndimage.morphology.grey_dilation(coh, size=(10,1))
2593 2598 #---------------------- Radial Velocity ----------------------------
2594 2599 phaseAux = numpy.mean(numpy.angle(data_acf[:,1,:,:]), axis = 0)
2595 2600 velRad = phaseAux*lamb/(4*numpy.pi*tSamp)
2596 2601
2597 2602 if allData:
2598 2603 boolMetFin = ~numpy.isnan(SNRm)
2599 2604 # coh[:-1,:] = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
2600 2605 else:
2601 2606 #------------------------ Meteor mask ---------------------------------
2602 2607 # #SNR mask
2603 2608 # boolMet = (SNRdB>SNRthresh)#|(~numpy.isnan(SNRdB))
2604 2609 #
2605 2610 # #Erase small objects
2606 2611 # boolMet1 = self.__erase_small(boolMet, 2*sec, 5)
2607 2612 #
2608 2613 # auxEEJ = numpy.sum(boolMet1,axis=0)
2609 2614 # indOver = auxEEJ>nProfiles*0.8 #Use this later
2610 2615 # indEEJ = numpy.where(indOver)[0]
2611 2616 # indNEEJ = numpy.where(~indOver)[0]
2612 2617 #
2613 2618 # boolMetFin = boolMet1
2614 2619 #
2615 2620 # if indEEJ.size > 0:
2616 2621 # boolMet1[:,indEEJ] = False #Erase heights with EEJ
2617 2622 #
2618 2623 # boolMet2 = coh > cohThresh
2619 2624 # boolMet2 = self.__erase_small(boolMet2, 2*sec,5)
2620 2625 #
2621 2626 # #Final Meteor mask
2622 2627 # boolMetFin = boolMet1|boolMet2
2623 2628
2624 2629 #Coherence mask
2625 2630 boolMet1 = coh > 0.75
2626 2631 struc = numpy.ones((30,1))
2627 2632 boolMet1 = ndimage.morphology.binary_dilation(boolMet1, structure=struc)
2628 2633
2629 2634 #Derivative mask
2630 2635 derPhase = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
2631 2636 boolMet2 = derPhase < 0.2
2632 2637 # boolMet2 = ndimage.morphology.binary_opening(boolMet2)
2633 2638 # boolMet2 = ndimage.morphology.binary_closing(boolMet2, structure = numpy.ones((10,1)))
2634 2639 boolMet2 = ndimage.median_filter(boolMet2,size=5)
2635 2640 boolMet2 = numpy.vstack((boolMet2,numpy.full((1,nHeights), True, dtype=bool)))
2636 2641 # #Final mask
2637 2642 # boolMetFin = boolMet2
2638 2643 boolMetFin = boolMet1&boolMet2
2639 2644 # boolMetFin = ndimage.morphology.binary_dilation(boolMetFin)
2640 2645 #Creating data_param
2641 2646 coordMet = numpy.where(boolMetFin)
2642 2647
2643 2648 tmet = coordMet[0]
2644 2649 hmet = coordMet[1]
2645 2650
2646 2651 data_param = numpy.zeros((tmet.size, 6 + nPairs))
2647 2652 data_param[:,0] = utctime
2648 2653 data_param[:,1] = tmet
2649 2654 data_param[:,2] = hmet
2650 2655 data_param[:,3] = SNRm[tmet,hmet]
2651 2656 data_param[:,4] = velRad[tmet,hmet]
2652 2657 data_param[:,5] = coh[tmet,hmet]
2653 2658 data_param[:,6:] = phase[:,tmet,hmet].T
2654 2659
2655 2660 elif mode == 'DBS':
2656 2661 dataOut.groupList = numpy.arange(nChannels)
2657 2662
2658 2663 #Radial Velocities
2659 2664 phase = numpy.angle(data_acf[:,1,:,:])
2660 2665 # phase = ndimage.median_filter(numpy.angle(data_acf[:,1,:,:]), size = (1,5,1))
2661 2666 velRad = phase*lamb/(4*numpy.pi*tSamp)
2662 2667
2663 2668 #Spectral width
2664 2669 # acf1 = ndimage.median_filter(numpy.abs(data_acf[:,1,:,:]), size = (1,5,1))
2665 2670 # acf2 = ndimage.median_filter(numpy.abs(data_acf[:,2,:,:]), size = (1,5,1))
2666 2671 acf1 = data_acf[:,1,:,:]
2667 2672 acf2 = data_acf[:,2,:,:]
2668 2673
2669 2674 spcWidth = (lamb/(2*numpy.sqrt(6)*numpy.pi*tSamp))*numpy.sqrt(numpy.log(acf1/acf2))
2670 2675 # velRad = ndimage.median_filter(velRad, size = (1,5,1))
2671 2676 if allData:
2672 2677 boolMetFin = ~numpy.isnan(SNRdB)
2673 2678 else:
2674 2679 #SNR
2675 2680 boolMet1 = (SNRdB>SNRthresh) #SNR mask
2676 2681 boolMet1 = ndimage.median_filter(boolMet1, size=(1,5,5))
2677 2682
2678 2683 #Radial velocity
2679 2684 boolMet2 = numpy.abs(velRad) < 20
2680 2685 boolMet2 = ndimage.median_filter(boolMet2, (1,5,5))
2681 2686
2682 2687 #Spectral Width
2683 2688 boolMet3 = spcWidth < 30
2684 2689 boolMet3 = ndimage.median_filter(boolMet3, (1,5,5))
2685 2690 # boolMetFin = self.__erase_small(boolMet1, 10,5)
2686 2691 boolMetFin = boolMet1&boolMet2&boolMet3
2687 2692
2688 2693 #Creating data_param
2689 2694 coordMet = numpy.where(boolMetFin)
2690 2695
2691 2696 cmet = coordMet[0]
2692 2697 tmet = coordMet[1]
2693 2698 hmet = coordMet[2]
2694 2699
2695 2700 data_param = numpy.zeros((tmet.size, 7))
2696 2701 data_param[:,0] = utctime
2697 2702 data_param[:,1] = cmet
2698 2703 data_param[:,2] = tmet
2699 2704 data_param[:,3] = hmet
2700 2705 data_param[:,4] = SNR[cmet,tmet,hmet].T
2701 2706 data_param[:,5] = velRad[cmet,tmet,hmet].T
2702 2707 data_param[:,6] = spcWidth[cmet,tmet,hmet].T
2703 2708
2704 2709 # self.dataOut.data_param = data_int
2705 2710 if len(data_param) == 0:
2706 2711 dataOut.flagNoData = True
2707 2712 else:
2708 2713 dataOut.data_param = data_param
2709 2714
2710 2715 def __erase_small(self, binArray, threshX, threshY):
2711 2716 labarray, numfeat = ndimage.measurements.label(binArray)
2712 2717 binArray1 = numpy.copy(binArray)
2713 2718
2714 2719 for i in range(1,numfeat + 1):
2715 2720 auxBin = (labarray==i)
2716 2721 auxSize = auxBin.sum()
2717 2722
2718 2723 x,y = numpy.where(auxBin)
2719 2724 widthX = x.max() - x.min()
2720 2725 widthY = y.max() - y.min()
2721 2726
2722 2727 #width X: 3 seg -> 12.5*3
2723 2728 #width Y:
2724 2729
2725 2730 if (auxSize < 50) or (widthX < threshX) or (widthY < threshY):
2726 2731 binArray1[auxBin] = False
2727 2732
2728 2733 return binArray1
2729 2734
2730 2735 #--------------- Specular Meteor ----------------
2731 2736
2732 2737 class SMDetection(Operation):
2733 2738 '''
2734 2739 Function DetectMeteors()
2735 2740 Project developed with paper:
2736 2741 HOLDSWORTH ET AL. 2004
2737 2742
2738 2743 Input:
2739 2744 self.dataOut.data_pre
2740 2745
2741 2746 centerReceiverIndex: From the channels, which is the center receiver
2742 2747
2743 2748 hei_ref: Height reference for the Beacon signal extraction
2744 2749 tauindex:
2745 2750 predefinedPhaseShifts: Predefined phase offset for the voltge signals
2746 2751
2747 2752 cohDetection: Whether to user Coherent detection or not
2748 2753 cohDet_timeStep: Coherent Detection calculation time step
2749 2754 cohDet_thresh: Coherent Detection phase threshold to correct phases
2750 2755
2751 2756 noise_timeStep: Noise calculation time step
2752 2757 noise_multiple: Noise multiple to define signal threshold
2753 2758
2754 2759 multDet_timeLimit: Multiple Detection Removal time limit in seconds
2755 2760 multDet_rangeLimit: Multiple Detection Removal range limit in km
2756 2761
2757 2762 phaseThresh: Maximum phase difference between receiver to be consider a meteor
2758 2763 SNRThresh: Minimum SNR threshold of the meteor signal to be consider a meteor
2759 2764
2760 2765 hmin: Minimum Height of the meteor to use it in the further wind estimations
2761 2766 hmax: Maximum Height of the meteor to use it in the further wind estimations
2762 2767 azimuth: Azimuth angle correction
2763 2768
2764 2769 Affected:
2765 2770 self.dataOut.data_param
2766 2771
2767 2772 Rejection Criteria (Errors):
2768 2773 0: No error; analysis OK
2769 2774 1: SNR < SNR threshold
2770 2775 2: angle of arrival (AOA) ambiguously determined
2771 2776 3: AOA estimate not feasible
2772 2777 4: Large difference in AOAs obtained from different antenna baselines
2773 2778 5: echo at start or end of time series
2774 2779 6: echo less than 5 examples long; too short for analysis
2775 2780 7: echo rise exceeds 0.3s
2776 2781 8: echo decay time less than twice rise time
2777 2782 9: large power level before echo
2778 2783 10: large power level after echo
2779 2784 11: poor fit to amplitude for estimation of decay time
2780 2785 12: poor fit to CCF phase variation for estimation of radial drift velocity
2781 2786 13: height unresolvable echo: not valid height within 70 to 110 km
2782 2787 14: height ambiguous echo: more then one possible height within 70 to 110 km
2783 2788 15: radial drift velocity or projected horizontal velocity exceeds 200 m/s
2784 2789 16: oscilatory echo, indicating event most likely not an underdense echo
2785 2790
2786 2791 17: phase difference in meteor Reestimation
2787 2792
2788 2793 Data Storage:
2789 2794 Meteors for Wind Estimation (8):
2790 2795 Utc Time | Range Height
2791 2796 Azimuth Zenith errorCosDir
2792 2797 VelRad errorVelRad
2793 2798 Phase0 Phase1 Phase2 Phase3
2794 2799 TypeError
2795 2800
2796 2801 '''
2797 2802
2798 2803 def run(self, dataOut, hei_ref = None, tauindex = 0,
2799 2804 phaseOffsets = None,
2800 2805 cohDetection = False, cohDet_timeStep = 1, cohDet_thresh = 25,
2801 2806 noise_timeStep = 4, noise_multiple = 4,
2802 2807 multDet_timeLimit = 1, multDet_rangeLimit = 3,
2803 2808 phaseThresh = 20, SNRThresh = 5,
2804 2809 hmin = 50, hmax=150, azimuth = 0,
2805 2810 channelPositions = None) :
2806 2811
2807 2812
2808 2813 #Getting Pairslist
2809 2814 if channelPositions is None:
2810 2815 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
2811 2816 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
2812 2817 meteorOps = SMOperations()
2813 2818 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
2814 2819 heiRang = dataOut.getHeiRange()
2815 2820 #Get Beacon signal - No Beacon signal anymore
2816 2821 # newheis = numpy.where(self.dataOut.heightList>self.dataOut.radarControllerHeaderObj.Taus[tauindex])
2817 2822 #
2818 2823 # if hei_ref != None:
2819 2824 # newheis = numpy.where(self.dataOut.heightList>hei_ref)
2820 2825 #
2821 2826
2822 2827
2823 2828 #****************REMOVING HARDWARE PHASE DIFFERENCES***************
2824 2829 # see if the user put in pre defined phase shifts
2825 2830 voltsPShift = dataOut.data_pre.copy()
2826 2831
2827 2832 # if predefinedPhaseShifts != None:
2828 2833 # hardwarePhaseShifts = numpy.array(predefinedPhaseShifts)*numpy.pi/180
2829 2834 #
2830 2835 # # elif beaconPhaseShifts:
2831 2836 # # #get hardware phase shifts using beacon signal
2832 2837 # # hardwarePhaseShifts = self.__getHardwarePhaseDiff(self.dataOut.data_pre, pairslist, newheis, 10)
2833 2838 # # hardwarePhaseShifts = numpy.insert(hardwarePhaseShifts,centerReceiverIndex,0)
2834 2839 #
2835 2840 # else:
2836 2841 # hardwarePhaseShifts = numpy.zeros(5)
2837 2842 #
2838 2843 # voltsPShift = numpy.zeros((self.dataOut.data_pre.shape[0],self.dataOut.data_pre.shape[1],self.dataOut.data_pre.shape[2]), dtype = 'complex')
2839 2844 # for i in range(self.dataOut.data_pre.shape[0]):
2840 2845 # voltsPShift[i,:,:] = self.__shiftPhase(self.dataOut.data_pre[i,:,:], hardwarePhaseShifts[i])
2841 2846
2842 2847 #******************END OF REMOVING HARDWARE PHASE DIFFERENCES*********
2843 2848
2844 2849 #Remove DC
2845 2850 voltsDC = numpy.mean(voltsPShift,1)
2846 2851 voltsDC = numpy.mean(voltsDC,1)
2847 2852 for i in range(voltsDC.shape[0]):
2848 2853 voltsPShift[i] = voltsPShift[i] - voltsDC[i]
2849 2854
2850 2855 #Don't considerate last heights, theyre used to calculate Hardware Phase Shift
2851 2856 # voltsPShift = voltsPShift[:,:,:newheis[0][0]]
2852 2857
2853 2858 #************ FIND POWER OF DATA W/COH OR NON COH DETECTION (3.4) **********
2854 2859 #Coherent Detection
2855 2860 if cohDetection:
2856 2861 #use coherent detection to get the net power
2857 2862 cohDet_thresh = cohDet_thresh*numpy.pi/180
2858 2863 voltsPShift = self.__coherentDetection(voltsPShift, cohDet_timeStep, dataOut.timeInterval, pairslist0, cohDet_thresh)
2859 2864
2860 2865 #Non-coherent detection!
2861 2866 powerNet = numpy.nansum(numpy.abs(voltsPShift[:,:,:])**2,0)
2862 2867 #********** END OF COH/NON-COH POWER CALCULATION**********************
2863 2868
2864 2869 #********** FIND THE NOISE LEVEL AND POSSIBLE METEORS ****************
2865 2870 #Get noise
2866 2871 noise, noise1 = self.__getNoise(powerNet, noise_timeStep, dataOut.timeInterval)
2867 2872 # noise = self.getNoise1(powerNet, noise_timeStep, self.dataOut.timeInterval)
2868 2873 #Get signal threshold
2869 2874 signalThresh = noise_multiple*noise
2870 2875 #Meteor echoes detection
2871 2876 listMeteors = self.__findMeteors(powerNet, signalThresh)
2872 2877 #******* END OF NOISE LEVEL AND POSSIBLE METEORS CACULATION **********
2873 2878
2874 2879 #************** REMOVE MULTIPLE DETECTIONS (3.5) ***************************
2875 2880 #Parameters
2876 2881 heiRange = dataOut.getHeiRange()
2877 2882 rangeInterval = heiRange[1] - heiRange[0]
2878 2883 rangeLimit = multDet_rangeLimit/rangeInterval
2879 2884 timeLimit = multDet_timeLimit/dataOut.timeInterval
2880 2885 #Multiple detection removals
2881 2886 listMeteors1 = self.__removeMultipleDetections(listMeteors, rangeLimit, timeLimit)
2882 2887 #************ END OF REMOVE MULTIPLE DETECTIONS **********************
2883 2888
2884 2889 #********************* METEOR REESTIMATION (3.7, 3.8, 3.9, 3.10) ********************
2885 2890 #Parameters
2886 2891 phaseThresh = phaseThresh*numpy.pi/180
2887 2892 thresh = [phaseThresh, noise_multiple, SNRThresh]
2888 2893 #Meteor reestimation (Errors N 1, 6, 12, 17)
2889 2894 listMeteors2, listMeteorsPower, listMeteorsVolts = self.__meteorReestimation(listMeteors1, voltsPShift, pairslist0, thresh, noise, dataOut.timeInterval, dataOut.frequency)
2890 2895 # listMeteors2, listMeteorsPower, listMeteorsVolts = self.meteorReestimation3(listMeteors2, listMeteorsPower, listMeteorsVolts, voltsPShift, pairslist, thresh, noise)
2891 2896 #Estimation of decay times (Errors N 7, 8, 11)
2892 2897 listMeteors3 = self.__estimateDecayTime(listMeteors2, listMeteorsPower, dataOut.timeInterval, dataOut.frequency)
2893 2898 #******************* END OF METEOR REESTIMATION *******************
2894 2899
2895 2900 #********************* METEOR PARAMETERS CALCULATION (3.11, 3.12, 3.13) **************************
2896 2901 #Calculating Radial Velocity (Error N 15)
2897 2902 radialStdThresh = 10
2898 2903 listMeteors4 = self.__getRadialVelocity(listMeteors3, listMeteorsVolts, radialStdThresh, pairslist0, dataOut.timeInterval)
2899 2904
2900 2905 if len(listMeteors4) > 0:
2901 2906 #Setting New Array
2902 2907 date = dataOut.utctime
2903 2908 arrayParameters = self.__setNewArrays(listMeteors4, date, heiRang)
2904 2909
2905 2910 #Correcting phase offset
2906 2911 if phaseOffsets != None:
2907 2912 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
2908 2913 arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
2909 2914
2910 2915 #Second Pairslist
2911 2916 pairsList = []
2912 2917 pairx = (0,1)
2913 2918 pairy = (2,3)
2914 2919 pairsList.append(pairx)
2915 2920 pairsList.append(pairy)
2916 2921
2917 2922 jph = numpy.array([0,0,0,0])
2918 2923 h = (hmin,hmax)
2919 2924 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
2920 2925
2921 2926 # #Calculate AOA (Error N 3, 4)
2922 2927 # #JONES ET AL. 1998
2923 2928 # error = arrayParameters[:,-1]
2924 2929 # AOAthresh = numpy.pi/8
2925 2930 # phases = -arrayParameters[:,9:13]
2926 2931 # arrayParameters[:,4:7], arrayParameters[:,-1] = meteorOps.getAOA(phases, pairsList, error, AOAthresh, azimuth)
2927 2932 #
2928 2933 # #Calculate Heights (Error N 13 and 14)
2929 2934 # error = arrayParameters[:,-1]
2930 2935 # Ranges = arrayParameters[:,2]
2931 2936 # zenith = arrayParameters[:,5]
2932 2937 # arrayParameters[:,3], arrayParameters[:,-1] = meteorOps.getHeights(Ranges, zenith, error, hmin, hmax)
2933 2938 # error = arrayParameters[:,-1]
2934 2939 #********************* END OF PARAMETERS CALCULATION **************************
2935 2940
2936 2941 #***************************+ PASS DATA TO NEXT STEP **********************
2937 2942 # arrayFinal = arrayParameters.reshape((1,arrayParameters.shape[0],arrayParameters.shape[1]))
2938 2943 dataOut.data_param = arrayParameters
2939 2944
2940 2945 if arrayParameters is None:
2941 2946 dataOut.flagNoData = True
2942 2947 else:
2943 2948 dataOut.flagNoData = True
2944 2949
2945 2950 return
2946 2951
2947 2952 def __getHardwarePhaseDiff(self, voltage0, pairslist, newheis, n):
2948 2953
2949 2954 minIndex = min(newheis[0])
2950 2955 maxIndex = max(newheis[0])
2951 2956
2952 2957 voltage = voltage0[:,:,minIndex:maxIndex+1]
2953 2958 nLength = voltage.shape[1]/n
2954 2959 nMin = 0
2955 2960 nMax = 0
2956 2961 phaseOffset = numpy.zeros((len(pairslist),n))
2957 2962
2958 2963 for i in range(n):
2959 2964 nMax += nLength
2960 2965 phaseCCF = -numpy.angle(self.__calculateCCF(voltage[:,nMin:nMax,:], pairslist, [0]))
2961 2966 phaseCCF = numpy.mean(phaseCCF, axis = 2)
2962 2967 phaseOffset[:,i] = phaseCCF.transpose()
2963 2968 nMin = nMax
2964 2969 # phaseDiff, phaseArrival = self.estimatePhaseDifference(voltage, pairslist)
2965 2970
2966 2971 #Remove Outliers
2967 2972 factor = 2
2968 2973 wt = phaseOffset - signal.medfilt(phaseOffset,(1,5))
2969 2974 dw = numpy.std(wt,axis = 1)
2970 2975 dw = dw.reshape((dw.size,1))
2971 2976 ind = numpy.where(numpy.logical_or(wt>dw*factor,wt<-dw*factor))
2972 2977 phaseOffset[ind] = numpy.nan
2973 2978 phaseOffset = stats.nanmean(phaseOffset, axis=1)
2974 2979
2975 2980 return phaseOffset
2976 2981
2977 2982 def __shiftPhase(self, data, phaseShift):
2978 2983 #this will shift the phase of a complex number
2979 2984 dataShifted = numpy.abs(data) * numpy.exp((numpy.angle(data)+phaseShift)*1j)
2980 2985 return dataShifted
2981 2986
2982 2987 def __estimatePhaseDifference(self, array, pairslist):
2983 2988 nChannel = array.shape[0]
2984 2989 nHeights = array.shape[2]
2985 2990 numPairs = len(pairslist)
2986 2991 # phaseCCF = numpy.zeros((nChannel, 5, nHeights))
2987 2992 phaseCCF = numpy.angle(self.__calculateCCF(array, pairslist, [-2,-1,0,1,2]))
2988 2993
2989 2994 #Correct phases
2990 2995 derPhaseCCF = phaseCCF[:,1:,:] - phaseCCF[:,0:-1,:]
2991 2996 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
2992 2997
2993 2998 if indDer[0].shape[0] > 0:
2994 2999 for i in range(indDer[0].shape[0]):
2995 3000 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i],indDer[2][i]])
2996 3001 phaseCCF[indDer[0][i],indDer[1][i]+1:,:] += signo*2*numpy.pi
2997 3002
2998 3003 # for j in range(numSides):
2999 3004 # phaseCCFAux = self.calculateCCF(arrayCenter, arraySides[j,:,:], [-2,1,0,1,2])
3000 3005 # phaseCCF[j,:,:] = numpy.angle(phaseCCFAux)
3001 3006 #
3002 3007 #Linear
3003 3008 phaseInt = numpy.zeros((numPairs,1))
3004 3009 angAllCCF = phaseCCF[:,[0,1,3,4],0]
3005 3010 for j in range(numPairs):
3006 3011 fit = stats.linregress([-2,-1,1,2],angAllCCF[j,:])
3007 3012 phaseInt[j] = fit[1]
3008 3013 #Phase Differences
3009 3014 phaseDiff = phaseInt - phaseCCF[:,2,:]
3010 3015 phaseArrival = phaseInt.reshape(phaseInt.size)
3011 3016
3012 3017 #Dealias
3013 3018 phaseArrival = numpy.angle(numpy.exp(1j*phaseArrival))
3014 3019 # indAlias = numpy.where(phaseArrival > numpy.pi)
3015 3020 # phaseArrival[indAlias] -= 2*numpy.pi
3016 3021 # indAlias = numpy.where(phaseArrival < -numpy.pi)
3017 3022 # phaseArrival[indAlias] += 2*numpy.pi
3018 3023
3019 3024 return phaseDiff, phaseArrival
3020 3025
3021 3026 def __coherentDetection(self, volts, timeSegment, timeInterval, pairslist, thresh):
3022 3027 #this function will run the coherent detection used in Holdworth et al. 2004 and return the net power
3023 3028 #find the phase shifts of each channel over 1 second intervals
3024 3029 #only look at ranges below the beacon signal
3025 3030 numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
3026 3031 numBlocks = int(volts.shape[1]/numProfPerBlock)
3027 3032 numHeights = volts.shape[2]
3028 3033 nChannel = volts.shape[0]
3029 3034 voltsCohDet = volts.copy()
3030 3035
3031 3036 pairsarray = numpy.array(pairslist)
3032 3037 indSides = pairsarray[:,1]
3033 3038 # indSides = numpy.array(range(nChannel))
3034 3039 # indSides = numpy.delete(indSides, indCenter)
3035 3040 #
3036 3041 # listCenter = numpy.array_split(volts[indCenter,:,:], numBlocks, 0)
3037 3042 listBlocks = numpy.array_split(volts, numBlocks, 1)
3038 3043
3039 3044 startInd = 0
3040 3045 endInd = 0
3041 3046
3042 3047 for i in range(numBlocks):
3043 3048 startInd = endInd
3044 3049 endInd = endInd + listBlocks[i].shape[1]
3045 3050
3046 3051 arrayBlock = listBlocks[i]
3047 3052 # arrayBlockCenter = listCenter[i]
3048 3053
3049 3054 #Estimate the Phase Difference
3050 3055 phaseDiff, aux = self.__estimatePhaseDifference(arrayBlock, pairslist)
3051 3056 #Phase Difference RMS
3052 3057 arrayPhaseRMS = numpy.abs(phaseDiff)
3053 3058 phaseRMSaux = numpy.sum(arrayPhaseRMS < thresh,0)
3054 3059 indPhase = numpy.where(phaseRMSaux==4)
3055 3060 #Shifting
3056 3061 if indPhase[0].shape[0] > 0:
3057 3062 for j in range(indSides.size):
3058 3063 arrayBlock[indSides[j],:,indPhase] = self.__shiftPhase(arrayBlock[indSides[j],:,indPhase], phaseDiff[j,indPhase].transpose())
3059 3064 voltsCohDet[:,startInd:endInd,:] = arrayBlock
3060 3065
3061 3066 return voltsCohDet
3062 3067
3063 3068 def __calculateCCF(self, volts, pairslist ,laglist):
3064 3069
3065 3070 nHeights = volts.shape[2]
3066 3071 nPoints = volts.shape[1]
3067 3072 voltsCCF = numpy.zeros((len(pairslist), len(laglist), nHeights),dtype = 'complex')
3068 3073
3069 3074 for i in range(len(pairslist)):
3070 3075 volts1 = volts[pairslist[i][0]]
3071 3076 volts2 = volts[pairslist[i][1]]
3072 3077
3073 3078 for t in range(len(laglist)):
3074 3079 idxT = laglist[t]
3075 3080 if idxT >= 0:
3076 3081 vStacked = numpy.vstack((volts2[idxT:,:],
3077 3082 numpy.zeros((idxT, nHeights),dtype='complex')))
3078 3083 else:
3079 3084 vStacked = numpy.vstack((numpy.zeros((-idxT, nHeights),dtype='complex'),
3080 3085 volts2[:(nPoints + idxT),:]))
3081 3086 voltsCCF[i,t,:] = numpy.sum((numpy.conjugate(volts1)*vStacked),axis=0)
3082 3087
3083 3088 vStacked = None
3084 3089 return voltsCCF
3085 3090
3086 3091 def __getNoise(self, power, timeSegment, timeInterval):
3087 3092 numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
3088 3093 numBlocks = int(power.shape[0]/numProfPerBlock)
3089 3094 numHeights = power.shape[1]
3090 3095
3091 3096 listPower = numpy.array_split(power, numBlocks, 0)
3092 3097 noise = numpy.zeros((power.shape[0], power.shape[1]))
3093 3098 noise1 = numpy.zeros((power.shape[0], power.shape[1]))
3094 3099
3095 3100 startInd = 0
3096 3101 endInd = 0
3097 3102
3098 3103 for i in range(numBlocks): #split por canal
3099 3104 startInd = endInd
3100 3105 endInd = endInd + listPower[i].shape[0]
3101 3106
3102 3107 arrayBlock = listPower[i]
3103 3108 noiseAux = numpy.mean(arrayBlock, 0)
3104 3109 # noiseAux = numpy.median(noiseAux)
3105 3110 # noiseAux = numpy.mean(arrayBlock)
3106 3111 noise[startInd:endInd,:] = noise[startInd:endInd,:] + noiseAux
3107 3112
3108 3113 noiseAux1 = numpy.mean(arrayBlock)
3109 3114 noise1[startInd:endInd,:] = noise1[startInd:endInd,:] + noiseAux1
3110 3115
3111 3116 return noise, noise1
3112 3117
3113 3118 def __findMeteors(self, power, thresh):
3114 3119 nProf = power.shape[0]
3115 3120 nHeights = power.shape[1]
3116 3121 listMeteors = []
3117 3122
3118 3123 for i in range(nHeights):
3119 3124 powerAux = power[:,i]
3120 3125 threshAux = thresh[:,i]
3121 3126
3122 3127 indUPthresh = numpy.where(powerAux > threshAux)[0]
3123 3128 indDNthresh = numpy.where(powerAux <= threshAux)[0]
3124 3129
3125 3130 j = 0
3126 3131
3127 3132 while (j < indUPthresh.size - 2):
3128 3133 if (indUPthresh[j + 2] == indUPthresh[j] + 2):
3129 3134 indDNAux = numpy.where(indDNthresh > indUPthresh[j])
3130 3135 indDNthresh = indDNthresh[indDNAux]
3131 3136
3132 3137 if (indDNthresh.size > 0):
3133 3138 indEnd = indDNthresh[0] - 1
3134 3139 indInit = indUPthresh[j]
3135 3140
3136 3141 meteor = powerAux[indInit:indEnd + 1]
3137 3142 indPeak = meteor.argmax() + indInit
3138 3143 FLA = sum(numpy.conj(meteor)*numpy.hstack((meteor[1:],0)))
3139 3144
3140 3145 listMeteors.append(numpy.array([i,indInit,indPeak,indEnd,FLA])) #CHEQUEAR!!!!!
3141 3146 j = numpy.where(indUPthresh == indEnd)[0] + 1
3142 3147 else: j+=1
3143 3148 else: j+=1
3144 3149
3145 3150 return listMeteors
3146 3151
3147 3152 def __removeMultipleDetections(self,listMeteors, rangeLimit, timeLimit):
3148 3153
3149 3154 arrayMeteors = numpy.asarray(listMeteors)
3150 3155 listMeteors1 = []
3151 3156
3152 3157 while arrayMeteors.shape[0] > 0:
3153 3158 FLAs = arrayMeteors[:,4]
3154 3159 maxFLA = FLAs.argmax()
3155 3160 listMeteors1.append(arrayMeteors[maxFLA,:])
3156 3161
3157 3162 MeteorInitTime = arrayMeteors[maxFLA,1]
3158 3163 MeteorEndTime = arrayMeteors[maxFLA,3]
3159 3164 MeteorHeight = arrayMeteors[maxFLA,0]
3160 3165
3161 3166 #Check neighborhood
3162 3167 maxHeightIndex = MeteorHeight + rangeLimit
3163 3168 minHeightIndex = MeteorHeight - rangeLimit
3164 3169 minTimeIndex = MeteorInitTime - timeLimit
3165 3170 maxTimeIndex = MeteorEndTime + timeLimit
3166 3171
3167 3172 #Check Heights
3168 3173 indHeight = numpy.logical_and(arrayMeteors[:,0] >= minHeightIndex, arrayMeteors[:,0] <= maxHeightIndex)
3169 3174 indTime = numpy.logical_and(arrayMeteors[:,3] >= minTimeIndex, arrayMeteors[:,1] <= maxTimeIndex)
3170 3175 indBoth = numpy.where(numpy.logical_and(indTime,indHeight))
3171 3176
3172 3177 arrayMeteors = numpy.delete(arrayMeteors, indBoth, axis = 0)
3173 3178
3174 3179 return listMeteors1
3175 3180
3176 3181 def __meteorReestimation(self, listMeteors, volts, pairslist, thresh, noise, timeInterval,frequency):
3177 3182 numHeights = volts.shape[2]
3178 3183 nChannel = volts.shape[0]
3179 3184
3180 3185 thresholdPhase = thresh[0]
3181 3186 thresholdNoise = thresh[1]
3182 3187 thresholdDB = float(thresh[2])
3183 3188
3184 3189 thresholdDB1 = 10**(thresholdDB/10)
3185 3190 pairsarray = numpy.array(pairslist)
3186 3191 indSides = pairsarray[:,1]
3187 3192
3188 3193 pairslist1 = list(pairslist)
3189 3194 pairslist1.append((0,1))
3190 3195 pairslist1.append((3,4))
3191 3196
3192 3197 listMeteors1 = []
3193 3198 listPowerSeries = []
3194 3199 listVoltageSeries = []
3195 3200 #volts has the war data
3196 3201
3197 3202 if frequency == 30e6:
3198 3203 timeLag = 45*10**-3
3199 3204 else:
3200 3205 timeLag = 15*10**-3
3201 3206 lag = numpy.ceil(timeLag/timeInterval)
3202 3207
3203 3208 for i in range(len(listMeteors)):
3204 3209
3205 3210 ###################### 3.6 - 3.7 PARAMETERS REESTIMATION #########################
3206 3211 meteorAux = numpy.zeros(16)
3207 3212
3208 3213 #Loading meteor Data (mHeight, mStart, mPeak, mEnd)
3209 3214 mHeight = listMeteors[i][0]
3210 3215 mStart = listMeteors[i][1]
3211 3216 mPeak = listMeteors[i][2]
3212 3217 mEnd = listMeteors[i][3]
3213 3218
3214 3219 #get the volt data between the start and end times of the meteor
3215 3220 meteorVolts = volts[:,mStart:mEnd+1,mHeight]
3216 3221 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
3217 3222
3218 3223 #3.6. Phase Difference estimation
3219 3224 phaseDiff, aux = self.__estimatePhaseDifference(meteorVolts, pairslist)
3220 3225
3221 3226 #3.7. Phase difference removal & meteor start, peak and end times reestimated
3222 3227 #meteorVolts0.- all Channels, all Profiles
3223 3228 meteorVolts0 = volts[:,:,mHeight]
3224 3229 meteorThresh = noise[:,mHeight]*thresholdNoise
3225 3230 meteorNoise = noise[:,mHeight]
3226 3231 meteorVolts0[indSides,:] = self.__shiftPhase(meteorVolts0[indSides,:], phaseDiff) #Phase Shifting
3227 3232 powerNet0 = numpy.nansum(numpy.abs(meteorVolts0)**2, axis = 0) #Power
3228 3233
3229 3234 #Times reestimation
3230 3235 mStart1 = numpy.where(powerNet0[:mPeak] < meteorThresh[:mPeak])[0]
3231 3236 if mStart1.size > 0:
3232 3237 mStart1 = mStart1[-1] + 1
3233 3238
3234 3239 else:
3235 3240 mStart1 = mPeak
3236 3241
3237 3242 mEnd1 = numpy.where(powerNet0[mPeak:] < meteorThresh[mPeak:])[0][0] + mPeak - 1
3238 3243 mEndDecayTime1 = numpy.where(powerNet0[mPeak:] < meteorNoise[mPeak:])[0]
3239 3244 if mEndDecayTime1.size == 0:
3240 3245 mEndDecayTime1 = powerNet0.size
3241 3246 else:
3242 3247 mEndDecayTime1 = mEndDecayTime1[0] + mPeak - 1
3243 3248 # mPeak1 = meteorVolts0[mStart1:mEnd1 + 1].argmax()
3244 3249
3245 3250 #meteorVolts1.- all Channels, from start to end
3246 3251 meteorVolts1 = meteorVolts0[:,mStart1:mEnd1 + 1]
3247 3252 meteorVolts2 = meteorVolts0[:,mPeak + lag:mEnd1 + 1]
3248 3253 if meteorVolts2.shape[1] == 0:
3249 3254 meteorVolts2 = meteorVolts0[:,mPeak:mEnd1 + 1]
3250 3255 meteorVolts1 = meteorVolts1.reshape(meteorVolts1.shape[0], meteorVolts1.shape[1], 1)
3251 3256 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1], 1)
3252 3257 ##################### END PARAMETERS REESTIMATION #########################
3253 3258
3254 3259 ##################### 3.8 PHASE DIFFERENCE REESTIMATION ########################
3255 3260 # if mEnd1 - mStart1 > 4: #Error Number 6: echo less than 5 samples long; too short for analysis
3256 3261 if meteorVolts2.shape[1] > 0:
3257 3262 #Phase Difference re-estimation
3258 3263 phaseDiff1, phaseDiffint = self.__estimatePhaseDifference(meteorVolts2, pairslist1) #Phase Difference Estimation
3259 3264 # phaseDiff1, phaseDiffint = self.estimatePhaseDifference(meteorVolts2, pairslist)
3260 3265 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1])
3261 3266 phaseDiff11 = numpy.reshape(phaseDiff1, (phaseDiff1.shape[0],1))
3262 3267 meteorVolts2[indSides,:] = self.__shiftPhase(meteorVolts2[indSides,:], phaseDiff11[0:4]) #Phase Shifting
3263 3268
3264 3269 #Phase Difference RMS
3265 3270 phaseRMS1 = numpy.sqrt(numpy.mean(numpy.square(phaseDiff1)))
3266 3271 powerNet1 = numpy.nansum(numpy.abs(meteorVolts1[:,:])**2,0)
3267 3272 #Data from Meteor
3268 3273 mPeak1 = powerNet1.argmax() + mStart1
3269 3274 mPeakPower1 = powerNet1.max()
3270 3275 noiseAux = sum(noise[mStart1:mEnd1 + 1,mHeight])
3271 3276 mSNR1 = (sum(powerNet1)-noiseAux)/noiseAux
3272 3277 Meteor1 = numpy.array([mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1])
3273 3278 Meteor1 = numpy.hstack((Meteor1,phaseDiffint))
3274 3279 PowerSeries = powerNet0[mStart1:mEndDecayTime1 + 1]
3275 3280 #Vectorize
3276 3281 meteorAux[0:7] = [mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1]
3277 3282 meteorAux[7:11] = phaseDiffint[0:4]
3278 3283
3279 3284 #Rejection Criterions
3280 3285 if phaseRMS1 > thresholdPhase: #Error Number 17: Phase variation
3281 3286 meteorAux[-1] = 17
3282 3287 elif mSNR1 < thresholdDB1: #Error Number 1: SNR < threshold dB
3283 3288 meteorAux[-1] = 1
3284 3289
3285 3290
3286 3291 else:
3287 3292 meteorAux[0:4] = [mHeight, mStart, mPeak, mEnd]
3288 3293 meteorAux[-1] = 6 #Error Number 6: echo less than 5 samples long; too short for analysis
3289 3294 PowerSeries = 0
3290 3295
3291 3296 listMeteors1.append(meteorAux)
3292 3297 listPowerSeries.append(PowerSeries)
3293 3298 listVoltageSeries.append(meteorVolts1)
3294 3299
3295 3300 return listMeteors1, listPowerSeries, listVoltageSeries
3296 3301
3297 3302 def __estimateDecayTime(self, listMeteors, listPower, timeInterval, frequency):
3298 3303
3299 3304 threshError = 10
3300 3305 #Depending if it is 30 or 50 MHz
3301 3306 if frequency == 30e6:
3302 3307 timeLag = 45*10**-3
3303 3308 else:
3304 3309 timeLag = 15*10**-3
3305 3310 lag = numpy.ceil(timeLag/timeInterval)
3306 3311
3307 3312 listMeteors1 = []
3308 3313
3309 3314 for i in range(len(listMeteors)):
3310 3315 meteorPower = listPower[i]
3311 3316 meteorAux = listMeteors[i]
3312 3317
3313 3318 if meteorAux[-1] == 0:
3314 3319
3315 3320 try:
3316 3321 indmax = meteorPower.argmax()
3317 3322 indlag = indmax + lag
3318 3323
3319 3324 y = meteorPower[indlag:]
3320 3325 x = numpy.arange(0, y.size)*timeLag
3321 3326
3322 3327 #first guess
3323 3328 a = y[0]
3324 3329 tau = timeLag
3325 3330 #exponential fit
3326 3331 popt, pcov = optimize.curve_fit(self.__exponential_function, x, y, p0 = [a, tau])
3327 3332 y1 = self.__exponential_function(x, *popt)
3328 3333 #error estimation
3329 3334 error = sum((y - y1)**2)/(numpy.var(y)*(y.size - popt.size))
3330 3335
3331 3336 decayTime = popt[1]
3332 3337 riseTime = indmax*timeInterval
3333 3338 meteorAux[11:13] = [decayTime, error]
3334 3339
3335 3340 #Table items 7, 8 and 11
3336 3341 if (riseTime > 0.3): #Number 7: Echo rise exceeds 0.3s
3337 3342 meteorAux[-1] = 7
3338 3343 elif (decayTime < 2*riseTime) : #Number 8: Echo decay time less than than twice rise time
3339 3344 meteorAux[-1] = 8
3340 3345 if (error > threshError): #Number 11: Poor fit to amplitude for estimation of decay time
3341 3346 meteorAux[-1] = 11
3342 3347
3343 3348
3344 3349 except:
3345 3350 meteorAux[-1] = 11
3346 3351
3347 3352
3348 3353 listMeteors1.append(meteorAux)
3349 3354
3350 3355 return listMeteors1
3351 3356
3352 3357 #Exponential Function
3353 3358
3354 3359 def __exponential_function(self, x, a, tau):
3355 3360 y = a*numpy.exp(-x/tau)
3356 3361 return y
3357 3362
3358 3363 def __getRadialVelocity(self, listMeteors, listVolts, radialStdThresh, pairslist, timeInterval):
3359 3364
3360 3365 pairslist1 = list(pairslist)
3361 3366 pairslist1.append((0,1))
3362 3367 pairslist1.append((3,4))
3363 3368 numPairs = len(pairslist1)
3364 3369 #Time Lag
3365 3370 timeLag = 45*10**-3
3366 3371 c = 3e8
3367 3372 lag = numpy.ceil(timeLag/timeInterval)
3368 3373 freq = 30e6
3369 3374
3370 3375 listMeteors1 = []
3371 3376
3372 3377 for i in range(len(listMeteors)):
3373 3378 meteorAux = listMeteors[i]
3374 3379 if meteorAux[-1] == 0:
3375 3380 mStart = listMeteors[i][1]
3376 3381 mPeak = listMeteors[i][2]
3377 3382 mLag = mPeak - mStart + lag
3378 3383
3379 3384 #get the volt data between the start and end times of the meteor
3380 3385 meteorVolts = listVolts[i]
3381 3386 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
3382 3387
3383 3388 #Get CCF
3384 3389 allCCFs = self.__calculateCCF(meteorVolts, pairslist1, [-2,-1,0,1,2])
3385 3390
3386 3391 #Method 2
3387 3392 slopes = numpy.zeros(numPairs)
3388 3393 time = numpy.array([-2,-1,1,2])*timeInterval
3389 3394 angAllCCF = numpy.angle(allCCFs[:,[0,1,3,4],0])
3390 3395
3391 3396 #Correct phases
3392 3397 derPhaseCCF = angAllCCF[:,1:] - angAllCCF[:,0:-1]
3393 3398 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
3394 3399
3395 3400 if indDer[0].shape[0] > 0:
3396 3401 for i in range(indDer[0].shape[0]):
3397 3402 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i]])
3398 3403 angAllCCF[indDer[0][i],indDer[1][i]+1:] += signo*2*numpy.pi
3399 3404
3400 3405 # fit = scipy.stats.linregress(numpy.array([-2,-1,1,2])*timeInterval, numpy.array([phaseLagN2s[i],phaseLagN1s[i],phaseLag1s[i],phaseLag2s[i]]))
3401 3406 for j in range(numPairs):
3402 3407 fit = stats.linregress(time, angAllCCF[j,:])
3403 3408 slopes[j] = fit[0]
3404 3409
3405 3410 #Remove Outlier
3406 3411 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
3407 3412 # slopes = numpy.delete(slopes,indOut)
3408 3413 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
3409 3414 # slopes = numpy.delete(slopes,indOut)
3410 3415
3411 3416 radialVelocity = -numpy.mean(slopes)*(0.25/numpy.pi)*(c/freq)
3412 3417 radialError = numpy.std(slopes)*(0.25/numpy.pi)*(c/freq)
3413 3418 meteorAux[-2] = radialError
3414 3419 meteorAux[-3] = radialVelocity
3415 3420
3416 3421 #Setting Error
3417 3422 #Number 15: Radial Drift velocity or projected horizontal velocity exceeds 200 m/s
3418 3423 if numpy.abs(radialVelocity) > 200:
3419 3424 meteorAux[-1] = 15
3420 3425 #Number 12: Poor fit to CCF variation for estimation of radial drift velocity
3421 3426 elif radialError > radialStdThresh:
3422 3427 meteorAux[-1] = 12
3423 3428
3424 3429 listMeteors1.append(meteorAux)
3425 3430 return listMeteors1
3426 3431
3427 3432 def __setNewArrays(self, listMeteors, date, heiRang):
3428 3433
3429 3434 #New arrays
3430 3435 arrayMeteors = numpy.array(listMeteors)
3431 3436 arrayParameters = numpy.zeros((len(listMeteors), 13))
3432 3437
3433 3438 #Date inclusion
3434 3439 # date = re.findall(r'\((.*?)\)', date)
3435 3440 # date = date[0].split(',')
3436 3441 # date = map(int, date)
3437 3442 #
3438 3443 # if len(date)<6:
3439 3444 # date.append(0)
3440 3445 #
3441 3446 # date = [date[0]*10000 + date[1]*100 + date[2], date[3]*10000 + date[4]*100 + date[5]]
3442 3447 # arrayDate = numpy.tile(date, (len(listMeteors), 1))
3443 3448 arrayDate = numpy.tile(date, (len(listMeteors)))
3444 3449
3445 3450 #Meteor array
3446 3451 # arrayMeteors[:,0] = heiRang[arrayMeteors[:,0].astype(int)]
3447 3452 # arrayMeteors = numpy.hstack((arrayDate, arrayMeteors))
3448 3453
3449 3454 #Parameters Array
3450 3455 arrayParameters[:,0] = arrayDate #Date
3451 3456 arrayParameters[:,1] = heiRang[arrayMeteors[:,0].astype(int)] #Range
3452 3457 arrayParameters[:,6:8] = arrayMeteors[:,-3:-1] #Radial velocity and its error
3453 3458 arrayParameters[:,8:12] = arrayMeteors[:,7:11] #Phases
3454 3459 arrayParameters[:,-1] = arrayMeteors[:,-1] #Error
3455 3460
3456 3461
3457 3462 return arrayParameters
3458 3463
3459 3464 class CorrectSMPhases(Operation):
3460 3465
3461 3466 def run(self, dataOut, phaseOffsets, hmin = 50, hmax = 150, azimuth = 45, channelPositions = None):
3462 3467
3463 3468 arrayParameters = dataOut.data_param
3464 3469 pairsList = []
3465 3470 pairx = (0,1)
3466 3471 pairy = (2,3)
3467 3472 pairsList.append(pairx)
3468 3473 pairsList.append(pairy)
3469 3474 jph = numpy.zeros(4)
3470 3475
3471 3476 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
3472 3477 # arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
3473 3478 arrayParameters[:,8:12] = numpy.angle(numpy.exp(1j*(arrayParameters[:,8:12] + phaseOffsets)))
3474 3479
3475 3480 meteorOps = SMOperations()
3476 3481 if channelPositions is None:
3477 3482 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
3478 3483 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
3479 3484
3480 3485 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
3481 3486 h = (hmin,hmax)
3482 3487
3483 3488 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
3484 3489
3485 3490 dataOut.data_param = arrayParameters
3486 3491 return
3487 3492
3488 3493 class SMPhaseCalibration(Operation):
3489 3494
3490 3495 __buffer = None
3491 3496
3492 3497 __initime = None
3493 3498
3494 3499 __dataReady = False
3495 3500
3496 3501 __isConfig = False
3497 3502
3498 3503 def __checkTime(self, currentTime, initTime, paramInterval, outputInterval):
3499 3504
3500 3505 dataTime = currentTime + paramInterval
3501 3506 deltaTime = dataTime - initTime
3502 3507
3503 3508 if deltaTime >= outputInterval or deltaTime < 0:
3504 3509 return True
3505 3510
3506 3511 return False
3507 3512
3508 3513 def __getGammas(self, pairs, d, phases):
3509 3514 gammas = numpy.zeros(2)
3510 3515
3511 3516 for i in range(len(pairs)):
3512 3517
3513 3518 pairi = pairs[i]
3514 3519
3515 3520 phip3 = phases[:,pairi[0]]
3516 3521 d3 = d[pairi[0]]
3517 3522 phip2 = phases[:,pairi[1]]
3518 3523 d2 = d[pairi[1]]
3519 3524 #Calculating gamma
3520 3525 # jdcos = alp1/(k*d1)
3521 3526 # jgamma = numpy.angle(numpy.exp(1j*(d0*alp1/d1 - alp0)))
3522 3527 jgamma = -phip2*d3/d2 - phip3
3523 3528 jgamma = numpy.angle(numpy.exp(1j*jgamma))
3524 3529 # jgamma[jgamma>numpy.pi] -= 2*numpy.pi
3525 3530 # jgamma[jgamma<-numpy.pi] += 2*numpy.pi
3526 3531
3527 3532 #Revised distribution
3528 3533 jgammaArray = numpy.hstack((jgamma,jgamma+0.5*numpy.pi,jgamma-0.5*numpy.pi))
3529 3534
3530 3535 #Histogram
3531 3536 nBins = 64
3532 3537 rmin = -0.5*numpy.pi
3533 3538 rmax = 0.5*numpy.pi
3534 3539 phaseHisto = numpy.histogram(jgammaArray, bins=nBins, range=(rmin,rmax))
3535 3540
3536 3541 meteorsY = phaseHisto[0]
3537 3542 phasesX = phaseHisto[1][:-1]
3538 3543 width = phasesX[1] - phasesX[0]
3539 3544 phasesX += width/2
3540 3545
3541 3546 #Gaussian aproximation
3542 3547 bpeak = meteorsY.argmax()
3543 3548 peak = meteorsY.max()
3544 3549 jmin = bpeak - 5
3545 3550 jmax = bpeak + 5 + 1
3546 3551
3547 3552 if jmin<0:
3548 3553 jmin = 0
3549 3554 jmax = 6
3550 3555 elif jmax > meteorsY.size:
3551 3556 jmin = meteorsY.size - 6
3552 3557 jmax = meteorsY.size
3553 3558
3554 3559 x0 = numpy.array([peak,bpeak,50])
3555 3560 coeff = optimize.leastsq(self.__residualFunction, x0, args=(meteorsY[jmin:jmax], phasesX[jmin:jmax]))
3556 3561
3557 3562 #Gammas
3558 3563 gammas[i] = coeff[0][1]
3559 3564
3560 3565 return gammas
3561 3566
3562 3567 def __residualFunction(self, coeffs, y, t):
3563 3568
3564 3569 return y - self.__gauss_function(t, coeffs)
3565 3570
3566 3571 def __gauss_function(self, t, coeffs):
3567 3572
3568 3573 return coeffs[0]*numpy.exp(-0.5*((t - coeffs[1]) / coeffs[2])**2)
3569 3574
3570 3575 def __getPhases(self, azimuth, h, pairsList, d, gammas, meteorsArray):
3571 3576 meteorOps = SMOperations()
3572 3577 nchan = 4
3573 3578 pairx = pairsList[0] #x es 0
3574 3579 pairy = pairsList[1] #y es 1
3575 3580 center_xangle = 0
3576 3581 center_yangle = 0
3577 3582 range_angle = numpy.array([10*numpy.pi,numpy.pi,numpy.pi/2,numpy.pi/4])
3578 3583 ntimes = len(range_angle)
3579 3584
3580 3585 nstepsx = 20
3581 3586 nstepsy = 20
3582 3587
3583 3588 for iz in range(ntimes):
3584 3589 min_xangle = -range_angle[iz]/2 + center_xangle
3585 3590 max_xangle = range_angle[iz]/2 + center_xangle
3586 3591 min_yangle = -range_angle[iz]/2 + center_yangle
3587 3592 max_yangle = range_angle[iz]/2 + center_yangle
3588 3593
3589 3594 inc_x = (max_xangle-min_xangle)/nstepsx
3590 3595 inc_y = (max_yangle-min_yangle)/nstepsy
3591 3596
3592 3597 alpha_y = numpy.arange(nstepsy)*inc_y + min_yangle
3593 3598 alpha_x = numpy.arange(nstepsx)*inc_x + min_xangle
3594 3599 penalty = numpy.zeros((nstepsx,nstepsy))
3595 3600 jph_array = numpy.zeros((nchan,nstepsx,nstepsy))
3596 3601 jph = numpy.zeros(nchan)
3597 3602
3598 3603 # Iterations looking for the offset
3599 3604 for iy in range(int(nstepsy)):
3600 3605 for ix in range(int(nstepsx)):
3601 3606 d3 = d[pairsList[1][0]]
3602 3607 d2 = d[pairsList[1][1]]
3603 3608 d5 = d[pairsList[0][0]]
3604 3609 d4 = d[pairsList[0][1]]
3605 3610
3606 3611 alp2 = alpha_y[iy] #gamma 1
3607 3612 alp4 = alpha_x[ix] #gamma 0
3608 3613
3609 3614 alp3 = -alp2*d3/d2 - gammas[1]
3610 3615 alp5 = -alp4*d5/d4 - gammas[0]
3611 3616 # jph[pairy[1]] = alpha_y[iy]
3612 3617 # jph[pairy[0]] = -gammas[1] - alpha_y[iy]*d[pairy[1]]/d[pairy[0]]
3613 3618
3614 3619 # jph[pairx[1]] = alpha_x[ix]
3615 3620 # jph[pairx[0]] = -gammas[0] - alpha_x[ix]*d[pairx[1]]/d[pairx[0]]
3616 3621 jph[pairsList[0][1]] = alp4
3617 3622 jph[pairsList[0][0]] = alp5
3618 3623 jph[pairsList[1][0]] = alp3
3619 3624 jph[pairsList[1][1]] = alp2
3620 3625 jph_array[:,ix,iy] = jph
3621 3626 # d = [2.0,2.5,2.5,2.0]
3622 3627 #falta chequear si va a leer bien los meteoros
3623 3628 meteorsArray1 = meteorOps.getMeteorParams(meteorsArray, azimuth, h, pairsList, d, jph)
3624 3629 error = meteorsArray1[:,-1]
3625 3630 ind1 = numpy.where(error==0)[0]
3626 3631 penalty[ix,iy] = ind1.size
3627 3632
3628 3633 i,j = numpy.unravel_index(penalty.argmax(), penalty.shape)
3629 3634 phOffset = jph_array[:,i,j]
3630 3635
3631 3636 center_xangle = phOffset[pairx[1]]
3632 3637 center_yangle = phOffset[pairy[1]]
3633 3638
3634 3639 phOffset = numpy.angle(numpy.exp(1j*jph_array[:,i,j]))
3635 3640 phOffset = phOffset*180/numpy.pi
3636 3641 return phOffset
3637 3642
3638 3643
3639 3644 def run(self, dataOut, hmin, hmax, channelPositions=None, nHours = 1):
3640 3645
3641 3646 dataOut.flagNoData = True
3642 3647 self.__dataReady = False
3643 3648 dataOut.outputInterval = nHours*3600
3644 3649
3645 3650 if self.__isConfig == False:
3646 3651 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
3647 3652 #Get Initial LTC time
3648 3653 self.__initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
3649 3654 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
3650 3655
3651 3656 self.__isConfig = True
3652 3657
3653 3658 if self.__buffer is None:
3654 3659 self.__buffer = dataOut.data_param.copy()
3655 3660
3656 3661 else:
3657 3662 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
3658 3663
3659 3664 self.__dataReady = self.__checkTime(dataOut.utctime, self.__initime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
3660 3665
3661 3666 if self.__dataReady:
3662 3667 dataOut.utctimeInit = self.__initime
3663 3668 self.__initime += dataOut.outputInterval #to erase time offset
3664 3669
3665 3670 freq = dataOut.frequency
3666 3671 c = dataOut.C #m/s
3667 3672 lamb = c/freq
3668 3673 k = 2*numpy.pi/lamb
3669 3674 azimuth = 0
3670 3675 h = (hmin, hmax)
3671 3676 # pairs = ((0,1),(2,3)) #Estrella
3672 3677 # pairs = ((1,0),(2,3)) #T
3673 3678
3674 3679 if channelPositions is None:
3675 3680 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
3676 3681 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
3677 3682 meteorOps = SMOperations()
3678 3683 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
3679 3684
3680 3685 #Checking correct order of pairs
3681 3686 pairs = []
3682 3687 if distances[1] > distances[0]:
3683 3688 pairs.append((1,0))
3684 3689 else:
3685 3690 pairs.append((0,1))
3686 3691
3687 3692 if distances[3] > distances[2]:
3688 3693 pairs.append((3,2))
3689 3694 else:
3690 3695 pairs.append((2,3))
3691 3696 # distances1 = [-distances[0]*lamb, distances[1]*lamb, -distances[2]*lamb, distances[3]*lamb]
3692 3697
3693 3698 meteorsArray = self.__buffer
3694 3699 error = meteorsArray[:,-1]
3695 3700 boolError = (error==0)|(error==3)|(error==4)|(error==13)|(error==14)
3696 3701 ind1 = numpy.where(boolError)[0]
3697 3702 meteorsArray = meteorsArray[ind1,:]
3698 3703 meteorsArray[:,-1] = 0
3699 3704 phases = meteorsArray[:,8:12]
3700 3705
3701 3706 #Calculate Gammas
3702 3707 gammas = self.__getGammas(pairs, distances, phases)
3703 3708 # gammas = numpy.array([-21.70409463,45.76935864])*numpy.pi/180
3704 3709 #Calculate Phases
3705 3710 phasesOff = self.__getPhases(azimuth, h, pairs, distances, gammas, meteorsArray)
3706 3711 phasesOff = phasesOff.reshape((1,phasesOff.size))
3707 3712 dataOut.data_output = -phasesOff
3708 3713 dataOut.flagNoData = False
3709 3714 self.__buffer = None
3710 3715
3711 3716
3712 3717 return
3713 3718
3714 3719 class SMOperations():
3715 3720
3716 3721 def __init__(self):
3717 3722
3718 3723 return
3719 3724
3720 3725 def getMeteorParams(self, arrayParameters0, azimuth, h, pairsList, distances, jph):
3721 3726
3722 3727 arrayParameters = arrayParameters0.copy()
3723 3728 hmin = h[0]
3724 3729 hmax = h[1]
3725 3730
3726 3731 #Calculate AOA (Error N 3, 4)
3727 3732 #JONES ET AL. 1998
3728 3733 AOAthresh = numpy.pi/8
3729 3734 error = arrayParameters[:,-1]
3730 3735 phases = -arrayParameters[:,8:12] + jph
3731 3736 # phases = numpy.unwrap(phases)
3732 3737 arrayParameters[:,3:6], arrayParameters[:,-1] = self.__getAOA(phases, pairsList, distances, error, AOAthresh, azimuth)
3733 3738
3734 3739 #Calculate Heights (Error N 13 and 14)
3735 3740 error = arrayParameters[:,-1]
3736 3741 Ranges = arrayParameters[:,1]
3737 3742 zenith = arrayParameters[:,4]
3738 3743 arrayParameters[:,2], arrayParameters[:,-1] = self.__getHeights(Ranges, zenith, error, hmin, hmax)
3739 3744
3740 3745 #----------------------- Get Final data ------------------------------------
3741 3746 # error = arrayParameters[:,-1]
3742 3747 # ind1 = numpy.where(error==0)[0]
3743 3748 # arrayParameters = arrayParameters[ind1,:]
3744 3749
3745 3750 return arrayParameters
3746 3751
3747 3752 def __getAOA(self, phases, pairsList, directions, error, AOAthresh, azimuth):
3748 3753
3749 3754 arrayAOA = numpy.zeros((phases.shape[0],3))
3750 3755 cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList,directions)
3751 3756
3752 3757 arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
3753 3758 cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
3754 3759 arrayAOA[:,2] = cosDirError
3755 3760
3756 3761 azimuthAngle = arrayAOA[:,0]
3757 3762 zenithAngle = arrayAOA[:,1]
3758 3763
3759 3764 #Setting Error
3760 3765 indError = numpy.where(numpy.logical_or(error == 3, error == 4))[0]
3761 3766 error[indError] = 0
3762 3767 #Number 3: AOA not fesible
3763 3768 indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
3764 3769 error[indInvalid] = 3
3765 3770 #Number 4: Large difference in AOAs obtained from different antenna baselines
3766 3771 indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
3767 3772 error[indInvalid] = 4
3768 3773 return arrayAOA, error
3769 3774
3770 3775 def __getDirectionCosines(self, arrayPhase, pairsList, distances):
3771 3776
3772 3777 #Initializing some variables
3773 3778 ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
3774 3779 ang_aux = ang_aux.reshape(1,ang_aux.size)
3775 3780
3776 3781 cosdir = numpy.zeros((arrayPhase.shape[0],2))
3777 3782 cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
3778 3783
3779 3784
3780 3785 for i in range(2):
3781 3786 ph0 = arrayPhase[:,pairsList[i][0]]
3782 3787 ph1 = arrayPhase[:,pairsList[i][1]]
3783 3788 d0 = distances[pairsList[i][0]]
3784 3789 d1 = distances[pairsList[i][1]]
3785 3790
3786 3791 ph0_aux = ph0 + ph1
3787 3792 ph0_aux = numpy.angle(numpy.exp(1j*ph0_aux))
3788 3793 # ph0_aux[ph0_aux > numpy.pi] -= 2*numpy.pi
3789 3794 # ph0_aux[ph0_aux < -numpy.pi] += 2*numpy.pi
3790 3795 #First Estimation
3791 3796 cosdir0[:,i] = (ph0_aux)/(2*numpy.pi*(d0 - d1))
3792 3797
3793 3798 #Most-Accurate Second Estimation
3794 3799 phi1_aux = ph0 - ph1
3795 3800 phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
3796 3801 #Direction Cosine 1
3797 3802 cosdir1 = (phi1_aux + ang_aux)/(2*numpy.pi*(d0 + d1))
3798 3803
3799 3804 #Searching the correct Direction Cosine
3800 3805 cosdir0_aux = cosdir0[:,i]
3801 3806 cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
3802 3807 #Minimum Distance
3803 3808 cosDiff = (cosdir1 - cosdir0_aux)**2
3804 3809 indcos = cosDiff.argmin(axis = 1)
3805 3810 #Saving Value obtained
3806 3811 cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
3807 3812
3808 3813 return cosdir0, cosdir
3809 3814
3810 3815 def __calculateAOA(self, cosdir, azimuth):
3811 3816 cosdirX = cosdir[:,0]
3812 3817 cosdirY = cosdir[:,1]
3813 3818
3814 3819 zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
3815 3820 azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth#0 deg north, 90 deg east
3816 3821 angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
3817 3822
3818 3823 return angles
3819 3824
3820 3825 def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
3821 3826
3822 3827 Ramb = 375 #Ramb = c/(2*PRF)
3823 3828 Re = 6371 #Earth Radius
3824 3829 heights = numpy.zeros(Ranges.shape)
3825 3830
3826 3831 R_aux = numpy.array([0,1,2])*Ramb
3827 3832 R_aux = R_aux.reshape(1,R_aux.size)
3828 3833
3829 3834 Ranges = Ranges.reshape(Ranges.size,1)
3830 3835
3831 3836 Ri = Ranges + R_aux
3832 3837 hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
3833 3838
3834 3839 #Check if there is a height between 70 and 110 km
3835 3840 h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
3836 3841 ind_h = numpy.where(h_bool == 1)[0]
3837 3842
3838 3843 hCorr = hi[ind_h, :]
3839 3844 ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
3840 3845
3841 3846 hCorr = hi[ind_hCorr][:len(ind_h)]
3842 3847 heights[ind_h] = hCorr
3843 3848
3844 3849 #Setting Error
3845 3850 #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
3846 3851 #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
3847 3852 indError = numpy.where(numpy.logical_or(error == 13, error == 14))[0]
3848 3853 error[indError] = 0
3849 3854 indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
3850 3855 error[indInvalid2] = 14
3851 3856 indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
3852 3857 error[indInvalid1] = 13
3853 3858
3854 3859 return heights, error
3855 3860
3856 3861 def getPhasePairs(self, channelPositions):
3857 3862 chanPos = numpy.array(channelPositions)
3858 3863 listOper = list(itertools.combinations(list(range(5)),2))
3859 3864
3860 3865 distances = numpy.zeros(4)
3861 3866 axisX = []
3862 3867 axisY = []
3863 3868 distX = numpy.zeros(3)
3864 3869 distY = numpy.zeros(3)
3865 3870 ix = 0
3866 3871 iy = 0
3867 3872
3868 3873 pairX = numpy.zeros((2,2))
3869 3874 pairY = numpy.zeros((2,2))
3870 3875
3871 3876 for i in range(len(listOper)):
3872 3877 pairi = listOper[i]
3873 3878
3874 3879 posDif = numpy.abs(chanPos[pairi[0],:] - chanPos[pairi[1],:])
3875 3880
3876 3881 if posDif[0] == 0:
3877 3882 axisY.append(pairi)
3878 3883 distY[iy] = posDif[1]
3879 3884 iy += 1
3880 3885 elif posDif[1] == 0:
3881 3886 axisX.append(pairi)
3882 3887 distX[ix] = posDif[0]
3883 3888 ix += 1
3884 3889
3885 3890 for i in range(2):
3886 3891 if i==0:
3887 3892 dist0 = distX
3888 3893 axis0 = axisX
3889 3894 else:
3890 3895 dist0 = distY
3891 3896 axis0 = axisY
3892 3897
3893 3898 side = numpy.argsort(dist0)[:-1]
3894 3899 axis0 = numpy.array(axis0)[side,:]
3895 3900 chanC = int(numpy.intersect1d(axis0[0,:], axis0[1,:])[0])
3896 3901 axis1 = numpy.unique(numpy.reshape(axis0,4))
3897 3902 side = axis1[axis1 != chanC]
3898 3903 diff1 = chanPos[chanC,i] - chanPos[side[0],i]
3899 3904 diff2 = chanPos[chanC,i] - chanPos[side[1],i]
3900 3905 if diff1<0:
3901 3906 chan2 = side[0]
3902 3907 d2 = numpy.abs(diff1)
3903 3908 chan1 = side[1]
3904 3909 d1 = numpy.abs(diff2)
3905 3910 else:
3906 3911 chan2 = side[1]
3907 3912 d2 = numpy.abs(diff2)
3908 3913 chan1 = side[0]
3909 3914 d1 = numpy.abs(diff1)
3910 3915
3911 3916 if i==0:
3912 3917 chanCX = chanC
3913 3918 chan1X = chan1
3914 3919 chan2X = chan2
3915 3920 distances[0:2] = numpy.array([d1,d2])
3916 3921 else:
3917 3922 chanCY = chanC
3918 3923 chan1Y = chan1
3919 3924 chan2Y = chan2
3920 3925 distances[2:4] = numpy.array([d1,d2])
3921 3926 # axisXsides = numpy.reshape(axisX[ix,:],4)
3922 3927 #
3923 3928 # channelCentX = int(numpy.intersect1d(pairX[0,:], pairX[1,:])[0])
3924 3929 # channelCentY = int(numpy.intersect1d(pairY[0,:], pairY[1,:])[0])
3925 3930 #
3926 3931 # ind25X = numpy.where(pairX[0,:] != channelCentX)[0][0]
3927 3932 # ind20X = numpy.where(pairX[1,:] != channelCentX)[0][0]
3928 3933 # channel25X = int(pairX[0,ind25X])
3929 3934 # channel20X = int(pairX[1,ind20X])
3930 3935 # ind25Y = numpy.where(pairY[0,:] != channelCentY)[0][0]
3931 3936 # ind20Y = numpy.where(pairY[1,:] != channelCentY)[0][0]
3932 3937 # channel25Y = int(pairY[0,ind25Y])
3933 3938 # channel20Y = int(pairY[1,ind20Y])
3934 3939
3935 3940 # pairslist = [(channelCentX, channel25X),(channelCentX, channel20X),(channelCentY,channel25Y),(channelCentY, channel20Y)]
3936 3941 pairslist = [(chanCX, chan1X),(chanCX, chan2X),(chanCY,chan1Y),(chanCY, chan2Y)]
3937 3942
3938 3943 return pairslist, distances
3939 3944 # def __getAOA(self, phases, pairsList, error, AOAthresh, azimuth):
3940 3945 #
3941 3946 # arrayAOA = numpy.zeros((phases.shape[0],3))
3942 3947 # cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList)
3943 3948 #
3944 3949 # arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
3945 3950 # cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
3946 3951 # arrayAOA[:,2] = cosDirError
3947 3952 #
3948 3953 # azimuthAngle = arrayAOA[:,0]
3949 3954 # zenithAngle = arrayAOA[:,1]
3950 3955 #
3951 3956 # #Setting Error
3952 3957 # #Number 3: AOA not fesible
3953 3958 # indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
3954 3959 # error[indInvalid] = 3
3955 3960 # #Number 4: Large difference in AOAs obtained from different antenna baselines
3956 3961 # indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
3957 3962 # error[indInvalid] = 4
3958 3963 # return arrayAOA, error
3959 3964 #
3960 3965 # def __getDirectionCosines(self, arrayPhase, pairsList):
3961 3966 #
3962 3967 # #Initializing some variables
3963 3968 # ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
3964 3969 # ang_aux = ang_aux.reshape(1,ang_aux.size)
3965 3970 #
3966 3971 # cosdir = numpy.zeros((arrayPhase.shape[0],2))
3967 3972 # cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
3968 3973 #
3969 3974 #
3970 3975 # for i in range(2):
3971 3976 # #First Estimation
3972 3977 # phi0_aux = arrayPhase[:,pairsList[i][0]] + arrayPhase[:,pairsList[i][1]]
3973 3978 # #Dealias
3974 3979 # indcsi = numpy.where(phi0_aux > numpy.pi)
3975 3980 # phi0_aux[indcsi] -= 2*numpy.pi
3976 3981 # indcsi = numpy.where(phi0_aux < -numpy.pi)
3977 3982 # phi0_aux[indcsi] += 2*numpy.pi
3978 3983 # #Direction Cosine 0
3979 3984 # cosdir0[:,i] = -(phi0_aux)/(2*numpy.pi*0.5)
3980 3985 #
3981 3986 # #Most-Accurate Second Estimation
3982 3987 # phi1_aux = arrayPhase[:,pairsList[i][0]] - arrayPhase[:,pairsList[i][1]]
3983 3988 # phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
3984 3989 # #Direction Cosine 1
3985 3990 # cosdir1 = -(phi1_aux + ang_aux)/(2*numpy.pi*4.5)
3986 3991 #
3987 3992 # #Searching the correct Direction Cosine
3988 3993 # cosdir0_aux = cosdir0[:,i]
3989 3994 # cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
3990 3995 # #Minimum Distance
3991 3996 # cosDiff = (cosdir1 - cosdir0_aux)**2
3992 3997 # indcos = cosDiff.argmin(axis = 1)
3993 3998 # #Saving Value obtained
3994 3999 # cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
3995 4000 #
3996 4001 # return cosdir0, cosdir
3997 4002 #
3998 4003 # def __calculateAOA(self, cosdir, azimuth):
3999 4004 # cosdirX = cosdir[:,0]
4000 4005 # cosdirY = cosdir[:,1]
4001 4006 #
4002 4007 # zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
4003 4008 # azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth #0 deg north, 90 deg east
4004 4009 # angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
4005 4010 #
4006 4011 # return angles
4007 4012 #
4008 4013 # def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
4009 4014 #
4010 4015 # Ramb = 375 #Ramb = c/(2*PRF)
4011 4016 # Re = 6371 #Earth Radius
4012 4017 # heights = numpy.zeros(Ranges.shape)
4013 4018 #
4014 4019 # R_aux = numpy.array([0,1,2])*Ramb
4015 4020 # R_aux = R_aux.reshape(1,R_aux.size)
4016 4021 #
4017 4022 # Ranges = Ranges.reshape(Ranges.size,1)
4018 4023 #
4019 4024 # Ri = Ranges + R_aux
4020 4025 # hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
4021 4026 #
4022 4027 # #Check if there is a height between 70 and 110 km
4023 4028 # h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
4024 4029 # ind_h = numpy.where(h_bool == 1)[0]
4025 4030 #
4026 4031 # hCorr = hi[ind_h, :]
4027 4032 # ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
4028 4033 #
4029 4034 # hCorr = hi[ind_hCorr]
4030 4035 # heights[ind_h] = hCorr
4031 4036 #
4032 4037 # #Setting Error
4033 4038 # #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
4034 4039 # #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
4035 4040 #
4036 4041 # indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
4037 4042 # error[indInvalid2] = 14
4038 4043 # indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
4039 4044 # error[indInvalid1] = 13
4040 4045 #
4041 4046 # return heights, error
4042 4047 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,964 +1,951
1 1 import itertools
2 2
3 3 import numpy
4 4
5 5 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
6 6 from schainpy.model.data.jrodata import Spectra
7 7 from schainpy.model.data.jrodata import hildebrand_sekhon
8 8 from schainpy.utils import log
9 9
10 10 @MPDecorator
11 11 class SpectraProc(ProcessingUnit):
12 12
13 METHODS = {'selectHeights' : ['minHei', 'maxHei'],
14 'selectChannels' : 'channelList',
15 'selectChannelsByIndex': 'channelIndexList',
16 'getBeaconSignal' : ['tauindex', 'channelindex', 'hei_ref'],
17 'selectHeightsByIndex' : ['minIndex', 'maxIndex']
18 }
19 13
20 def __init__(self):#, **kwargs):
14 def __init__(self):
21 15
22 ProcessingUnit.__init__(self)#, **kwargs)
16 ProcessingUnit.__init__(self)
23 17
24 18 self.buffer = None
25 19 self.firstdatatime = None
26 20 self.profIndex = 0
27 21 self.dataOut = Spectra()
28 22 self.id_min = None
29 23 self.id_max = None
30 24 self.setupReq = False #Agregar a todas las unidades de proc
31 25
32 26 def __updateSpecFromVoltage(self):
33 27
34 28 self.dataOut.timeZone = self.dataIn.timeZone
35 29 self.dataOut.dstFlag = self.dataIn.dstFlag
36 30 self.dataOut.errorCount = self.dataIn.errorCount
37 31 self.dataOut.useLocalTime = self.dataIn.useLocalTime
38 32 try:
39 33 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
40 34 except:
41 35 pass
42 36 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
43 37 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
44 38 self.dataOut.channelList = self.dataIn.channelList
45 39 self.dataOut.heightList = self.dataIn.heightList
46 40 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
47 41
48 42 self.dataOut.nBaud = self.dataIn.nBaud
49 43 self.dataOut.nCode = self.dataIn.nCode
50 44 self.dataOut.code = self.dataIn.code
51 45 self.dataOut.nProfiles = self.dataOut.nFFTPoints
52 46
53 47 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
54 48 self.dataOut.utctime = self.firstdatatime
55 49 # asumo q la data esta decodificada
56 50 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
57 51 # asumo q la data esta sin flip
58 52 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
59 53 self.dataOut.flagShiftFFT = False
60 54
61 55 self.dataOut.nCohInt = self.dataIn.nCohInt
62 56 self.dataOut.nIncohInt = 1
63 57
64 58 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
65 59
66 60 self.dataOut.frequency = self.dataIn.frequency
67 61 self.dataOut.realtime = self.dataIn.realtime
68 62
69 63 self.dataOut.azimuth = self.dataIn.azimuth
70 64 self.dataOut.zenith = self.dataIn.zenith
71 65
72 66 self.dataOut.beam.codeList = self.dataIn.beam.codeList
73 67 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
74 68 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
75 69
76 70 def __getFft(self):
77 71 """
78 72 Convierte valores de Voltaje a Spectra
79 73
80 74 Affected:
81 75 self.dataOut.data_spc
82 76 self.dataOut.data_cspc
83 77 self.dataOut.data_dc
84 78 self.dataOut.heightList
85 79 self.profIndex
86 80 self.buffer
87 81 self.dataOut.flagNoData
88 82 """
89 83 fft_volt = numpy.fft.fft(
90 84 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
91 85 fft_volt = fft_volt.astype(numpy.dtype('complex'))
92 86 dc = fft_volt[:, 0, :]
93 87
94 88 # calculo de self-spectra
95 89 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
96 90 spc = fft_volt * numpy.conjugate(fft_volt)
97 91 spc = spc.real
98 92
99 93 blocksize = 0
100 94 blocksize += dc.size
101 95 blocksize += spc.size
102 96
103 97 cspc = None
104 98 pairIndex = 0
105 99 if self.dataOut.pairsList != None:
106 100 # calculo de cross-spectra
107 101 cspc = numpy.zeros(
108 102 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
109 103 for pair in self.dataOut.pairsList:
110 104 if pair[0] not in self.dataOut.channelList:
111 105 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
112 106 str(pair), str(self.dataOut.channelList)))
113 107 if pair[1] not in self.dataOut.channelList:
114 108 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
115 109 str(pair), str(self.dataOut.channelList)))
116 110
117 111 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
118 112 numpy.conjugate(fft_volt[pair[1], :, :])
119 113 pairIndex += 1
120 114 blocksize += cspc.size
121 115
122 116 self.dataOut.data_spc = spc
123 117 self.dataOut.data_cspc = cspc
124 118 self.dataOut.data_dc = dc
125 119 self.dataOut.blockSize = blocksize
126 120 self.dataOut.flagShiftFFT = True
127 121
128 122 def run(self, nProfiles=None, nFFTPoints=None, pairsList=[], ippFactor=None, shift_fft=False):
129 123
130 124 if self.dataIn.type == "Spectra":
131 125 self.dataOut.copy(self.dataIn)
132 # if not pairsList:
133 # pairsList = itertools.combinations(self.dataOut.channelList, 2)
134 # if self.dataOut.data_cspc is not None:
135 # self.__selectPairs(pairsList)
136 126 if shift_fft:
137 127 #desplaza a la derecha en el eje 2 determinadas posiciones
138 128 shift = int(self.dataOut.nFFTPoints/2)
139 129 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
140 130
141 131 if self.dataOut.data_cspc is not None:
142 132 #desplaza a la derecha en el eje 2 determinadas posiciones
143 133 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
144 134
145 135 return True
146 136
147 137 if self.dataIn.type == "Voltage":
148 138
149 139 if nFFTPoints == None:
150 140 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
151 141
152 142 if nProfiles == None:
153 143 nProfiles = nFFTPoints
154 144
155 145 if ippFactor == None:
156 146 ippFactor = 1
157 147
158 148 self.dataOut.ippFactor = ippFactor
159 149
160 150 self.dataOut.nFFTPoints = nFFTPoints
161 151 self.dataOut.pairsList = pairsList
162 152
163 153 if self.buffer is None:
164 154 self.buffer = numpy.zeros((self.dataIn.nChannels,
165 155 nProfiles,
166 156 self.dataIn.nHeights),
167 157 dtype='complex')
168 158
169 159 if self.dataIn.flagDataAsBlock:
170 160 # data dimension: [nChannels, nProfiles, nSamples]
171 161 nVoltProfiles = self.dataIn.data.shape[1]
172 162 # nVoltProfiles = self.dataIn.nProfiles
173 163
174 164 if nVoltProfiles == nProfiles:
175 165 self.buffer = self.dataIn.data.copy()
176 166 self.profIndex = nVoltProfiles
177 167
178 168 elif nVoltProfiles < nProfiles:
179 169
180 170 if self.profIndex == 0:
181 171 self.id_min = 0
182 172 self.id_max = nVoltProfiles
183 173
184 174 self.buffer[:, self.id_min:self.id_max,
185 175 :] = self.dataIn.data
186 176 self.profIndex += nVoltProfiles
187 177 self.id_min += nVoltProfiles
188 178 self.id_max += nVoltProfiles
189 179 else:
190 180 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
191 181 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
192 182 self.dataOut.flagNoData = True
193 183 return 0
194 184 else:
195 185 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
196 186 self.profIndex += 1
197 187
198 188 if self.firstdatatime == None:
199 189 self.firstdatatime = self.dataIn.utctime
200 190
201 191 if self.profIndex == nProfiles:
202 192 self.__updateSpecFromVoltage()
203 193 self.__getFft()
204 194
205 195 self.dataOut.flagNoData = False
206 196 self.firstdatatime = None
207 197 self.profIndex = 0
208 198
209 199 return True
210 200
211 201 raise ValueError("The type of input object '%s' is not valid" % (
212 202 self.dataIn.type))
213 203
214 204 def __selectPairs(self, pairsList):
215 205
216 206 if not pairsList:
217 207 return
218 208
219 209 pairs = []
220 210 pairsIndex = []
221 211
222 212 for pair in pairsList:
223 213 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
224 214 continue
225 215 pairs.append(pair)
226 216 pairsIndex.append(pairs.index(pair))
227 217
228 218 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
229 219 self.dataOut.pairsList = pairs
230 220
231 221 return
232 222
233 223 def __selectPairsByChannel(self, channelList=None):
234 224
235 225 if channelList == None:
236 226 return
237 227
238 228 pairsIndexListSelected = []
239 229 for pairIndex in self.dataOut.pairsIndexList:
240 230 # First pair
241 231 if self.dataOut.pairsList[pairIndex][0] not in channelList:
242 232 continue
243 233 # Second pair
244 234 if self.dataOut.pairsList[pairIndex][1] not in channelList:
245 235 continue
246 236
247 237 pairsIndexListSelected.append(pairIndex)
248 238
249 239 if not pairsIndexListSelected:
250 240 self.dataOut.data_cspc = None
251 241 self.dataOut.pairsList = []
252 242 return
253 243
254 244 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndexListSelected]
255 245 self.dataOut.pairsList = [self.dataOut.pairsList[i]
256 246 for i in pairsIndexListSelected]
257 247
258 248 return
259 249
260 250 def selectChannels(self, channelList):
261 251
262 252 channelIndexList = []
263 253
264 254 for channel in channelList:
265 255 if channel not in self.dataOut.channelList:
266 256 raise ValueError("Error selecting channels, Channel %d is not valid.\nAvailable channels = %s" % (
267 257 channel, str(self.dataOut.channelList)))
268 258
269 259 index = self.dataOut.channelList.index(channel)
270 260 channelIndexList.append(index)
271 261
272 262 self.selectChannelsByIndex(channelIndexList)
273 263
274 264 def selectChannelsByIndex(self, channelIndexList):
275 265 """
276 266 Selecciona un bloque de datos en base a canales segun el channelIndexList
277 267
278 268 Input:
279 269 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
280 270
281 271 Affected:
282 272 self.dataOut.data_spc
283 273 self.dataOut.channelIndexList
284 274 self.dataOut.nChannels
285 275
286 276 Return:
287 277 None
288 278 """
289 279
290 280 for channelIndex in channelIndexList:
291 281 if channelIndex not in self.dataOut.channelIndexList:
292 282 raise ValueError("Error selecting channels: The value %d in channelIndexList is not valid.\nAvailable channel indexes = " % (
293 283 channelIndex, self.dataOut.channelIndexList))
294 284
295 285 # nChannels = len(channelIndexList)
296 286
297 287 data_spc = self.dataOut.data_spc[channelIndexList, :]
298 288 data_dc = self.dataOut.data_dc[channelIndexList, :]
299 289
300 290 self.dataOut.data_spc = data_spc
301 291 self.dataOut.data_dc = data_dc
302 292
303 293 self.dataOut.channelList = [
304 294 self.dataOut.channelList[i] for i in channelIndexList]
305 295 # self.dataOut.nChannels = nChannels
306 296
307 297 self.__selectPairsByChannel(self.dataOut.channelList)
308 298
309 299 return 1
310 300
311 301 def selectHeights(self, minHei, maxHei):
312 302 """
313 303 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
314 304 minHei <= height <= maxHei
315 305
316 306 Input:
317 307 minHei : valor minimo de altura a considerar
318 308 maxHei : valor maximo de altura a considerar
319 309
320 310 Affected:
321 311 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
322 312
323 313 Return:
324 314 1 si el metodo se ejecuto con exito caso contrario devuelve 0
325 315 """
326 316
327 317 if (minHei > maxHei):
328 318 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (
329 319 minHei, maxHei))
330 320
331 321 if (minHei < self.dataOut.heightList[0]):
332 322 minHei = self.dataOut.heightList[0]
333 323
334 324 if (maxHei > self.dataOut.heightList[-1]):
335 325 maxHei = self.dataOut.heightList[-1]
336 326
337 327 minIndex = 0
338 328 maxIndex = 0
339 329 heights = self.dataOut.heightList
340 330
341 331 inda = numpy.where(heights >= minHei)
342 332 indb = numpy.where(heights <= maxHei)
343 333
344 334 try:
345 335 minIndex = inda[0][0]
346 336 except:
347 337 minIndex = 0
348 338
349 339 try:
350 340 maxIndex = indb[0][-1]
351 341 except:
352 342 maxIndex = len(heights)
353 343
354 344 self.selectHeightsByIndex(minIndex, maxIndex)
355 345
356 346 return 1
357 347
358 348 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
359 349 newheis = numpy.where(
360 350 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
361 351
362 352 if hei_ref != None:
363 353 newheis = numpy.where(self.dataOut.heightList > hei_ref)
364 354
365 355 minIndex = min(newheis[0])
366 356 maxIndex = max(newheis[0])
367 357 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
368 358 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
369 359
370 360 # determina indices
371 361 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
372 362 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
373 363 avg_dB = 10 * \
374 364 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
375 365 beacon_dB = numpy.sort(avg_dB)[-nheis:]
376 366 beacon_heiIndexList = []
377 367 for val in avg_dB.tolist():
378 368 if val >= beacon_dB[0]:
379 369 beacon_heiIndexList.append(avg_dB.tolist().index(val))
380 370
381 371 #data_spc = data_spc[:,:,beacon_heiIndexList]
382 372 data_cspc = None
383 373 if self.dataOut.data_cspc is not None:
384 374 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
385 375 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
386 376
387 377 data_dc = None
388 378 if self.dataOut.data_dc is not None:
389 379 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
390 380 #data_dc = data_dc[:,beacon_heiIndexList]
391 381
392 382 self.dataOut.data_spc = data_spc
393 383 self.dataOut.data_cspc = data_cspc
394 384 self.dataOut.data_dc = data_dc
395 385 self.dataOut.heightList = heightList
396 386 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
397 387
398 388 return 1
399 389
400 390 def selectHeightsByIndex(self, minIndex, maxIndex):
401 391 """
402 392 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
403 393 minIndex <= index <= maxIndex
404 394
405 395 Input:
406 396 minIndex : valor de indice minimo de altura a considerar
407 397 maxIndex : valor de indice maximo de altura a considerar
408 398
409 399 Affected:
410 400 self.dataOut.data_spc
411 401 self.dataOut.data_cspc
412 402 self.dataOut.data_dc
413 403 self.dataOut.heightList
414 404
415 405 Return:
416 406 1 si el metodo se ejecuto con exito caso contrario devuelve 0
417 407 """
418 408
419 409 if (minIndex < 0) or (minIndex > maxIndex):
420 410 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (
421 411 minIndex, maxIndex))
422 412
423 413 if (maxIndex >= self.dataOut.nHeights):
424 414 maxIndex = self.dataOut.nHeights - 1
425 415
426 416 # Spectra
427 417 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
428 418
429 419 data_cspc = None
430 420 if self.dataOut.data_cspc is not None:
431 421 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
432 422
433 423 data_dc = None
434 424 if self.dataOut.data_dc is not None:
435 425 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
436 426
437 427 self.dataOut.data_spc = data_spc
438 428 self.dataOut.data_cspc = data_cspc
439 429 self.dataOut.data_dc = data_dc
440 430
441 431 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
442 432
443 433 return 1
444 434
445 435 def removeDC(self, mode=2):
446 436 jspectra = self.dataOut.data_spc
447 437 jcspectra = self.dataOut.data_cspc
448 438
449 439 num_chan = jspectra.shape[0]
450 440 num_hei = jspectra.shape[2]
451 441
452 442 if jcspectra is not None:
453 443 jcspectraExist = True
454 444 num_pairs = jcspectra.shape[0]
455 445 else:
456 446 jcspectraExist = False
457 447
458 448 freq_dc = int(jspectra.shape[1] / 2)
459 449 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
460 450 ind_vel = ind_vel.astype(int)
461 451
462 452 if ind_vel[0] < 0:
463 453 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
464 454
465 455 if mode == 1:
466 456 jspectra[:, freq_dc, :] = (
467 457 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
468 458
469 459 if jcspectraExist:
470 460 jcspectra[:, freq_dc, :] = (
471 461 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
472 462
473 463 if mode == 2:
474 464
475 465 vel = numpy.array([-2, -1, 1, 2])
476 466 xx = numpy.zeros([4, 4])
477 467
478 468 for fil in range(4):
479 469 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
480 470
481 471 xx_inv = numpy.linalg.inv(xx)
482 472 xx_aux = xx_inv[0, :]
483 473
484 474 for ich in range(num_chan):
485 475 yy = jspectra[ich, ind_vel, :]
486 476 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
487 477
488 478 junkid = jspectra[ich, freq_dc, :] <= 0
489 479 cjunkid = sum(junkid)
490 480
491 481 if cjunkid.any():
492 482 jspectra[ich, freq_dc, junkid.nonzero()] = (
493 483 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
494 484
495 485 if jcspectraExist:
496 486 for ip in range(num_pairs):
497 487 yy = jcspectra[ip, ind_vel, :]
498 488 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
499 489
500 490 self.dataOut.data_spc = jspectra
501 491 self.dataOut.data_cspc = jcspectra
502 492
503 493 return 1
504 494
505 495 def removeInterference(self, interf=2, hei_interf=None, nhei_interf=None, offhei_interf=None):
506 496
507 497 jspectra = self.dataOut.data_spc
508 498 jcspectra = self.dataOut.data_cspc
509 499 jnoise = self.dataOut.getNoise()
510 500 num_incoh = self.dataOut.nIncohInt
511 501
512 502 num_channel = jspectra.shape[0]
513 503 num_prof = jspectra.shape[1]
514 504 num_hei = jspectra.shape[2]
515 505
516 506 # hei_interf
517 507 if hei_interf is None:
518 508 count_hei = num_hei / 2 # Como es entero no importa
519 509 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
520 510 hei_interf = numpy.asarray(hei_interf)[0]
521 511 # nhei_interf
522 512 if (nhei_interf == None):
523 513 nhei_interf = 5
524 514 if (nhei_interf < 1):
525 515 nhei_interf = 1
526 516 if (nhei_interf > count_hei):
527 517 nhei_interf = count_hei
528 518 if (offhei_interf == None):
529 519 offhei_interf = 0
530 520
531 521 ind_hei = list(range(num_hei))
532 522 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
533 523 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
534 524 mask_prof = numpy.asarray(list(range(num_prof)))
535 525 num_mask_prof = mask_prof.size
536 526 comp_mask_prof = [0, num_prof / 2]
537 527
538 528 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
539 529 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
540 530 jnoise = numpy.nan
541 531 noise_exist = jnoise[0] < numpy.Inf
542 532
543 533 # Subrutina de Remocion de la Interferencia
544 534 for ich in range(num_channel):
545 535 # Se ordena los espectros segun su potencia (menor a mayor)
546 536 power = jspectra[ich, mask_prof, :]
547 537 power = power[:, hei_interf]
548 538 power = power.sum(axis=0)
549 539 psort = power.ravel().argsort()
550 540
551 541 # Se estima la interferencia promedio en los Espectros de Potencia empleando
552 542 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
553 543 offhei_interf, nhei_interf + offhei_interf))]]]
554 544
555 545 if noise_exist:
556 546 # tmp_noise = jnoise[ich] / num_prof
557 547 tmp_noise = jnoise[ich]
558 548 junkspc_interf = junkspc_interf - tmp_noise
559 549 #junkspc_interf[:,comp_mask_prof] = 0
560 550
561 551 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
562 552 jspc_interf = jspc_interf.transpose()
563 553 # Calculando el espectro de interferencia promedio
564 554 noiseid = numpy.where(
565 555 jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
566 556 noiseid = noiseid[0]
567 557 cnoiseid = noiseid.size
568 558 interfid = numpy.where(
569 559 jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
570 560 interfid = interfid[0]
571 561 cinterfid = interfid.size
572 562
573 563 if (cnoiseid > 0):
574 564 jspc_interf[noiseid] = 0
575 565
576 566 # Expandiendo los perfiles a limpiar
577 567 if (cinterfid > 0):
578 568 new_interfid = (
579 569 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
580 570 new_interfid = numpy.asarray(new_interfid)
581 571 new_interfid = {x for x in new_interfid}
582 572 new_interfid = numpy.array(list(new_interfid))
583 573 new_cinterfid = new_interfid.size
584 574 else:
585 575 new_cinterfid = 0
586 576
587 577 for ip in range(new_cinterfid):
588 578 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
589 579 jspc_interf[new_interfid[ip]
590 580 ] = junkspc_interf[ind[nhei_interf / 2], new_interfid[ip]]
591 581
592 582 jspectra[ich, :, ind_hei] = jspectra[ich, :,
593 583 ind_hei] - jspc_interf # Corregir indices
594 584
595 585 # Removiendo la interferencia del punto de mayor interferencia
596 586 ListAux = jspc_interf[mask_prof].tolist()
597 587 maxid = ListAux.index(max(ListAux))
598 588
599 589 if cinterfid > 0:
600 590 for ip in range(cinterfid * (interf == 2) - 1):
601 591 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
602 592 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
603 593 cind = len(ind)
604 594
605 595 if (cind > 0):
606 596 jspectra[ich, interfid[ip], ind] = tmp_noise * \
607 597 (1 + (numpy.random.uniform(cind) - 0.5) /
608 598 numpy.sqrt(num_incoh))
609 599
610 600 ind = numpy.array([-2, -1, 1, 2])
611 601 xx = numpy.zeros([4, 4])
612 602
613 603 for id1 in range(4):
614 604 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
615 605
616 606 xx_inv = numpy.linalg.inv(xx)
617 607 xx = xx_inv[:, 0]
618 608 ind = (ind + maxid + num_mask_prof) % num_mask_prof
619 609 yy = jspectra[ich, mask_prof[ind], :]
620 610 jspectra[ich, mask_prof[maxid], :] = numpy.dot(
621 611 yy.transpose(), xx)
622 612
623 613 indAux = (jspectra[ich, :, :] < tmp_noise *
624 614 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
625 615 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
626 616 (1 - 1 / numpy.sqrt(num_incoh))
627 617
628 618 # Remocion de Interferencia en el Cross Spectra
629 619 if jcspectra is None:
630 620 return jspectra, jcspectra
631 621 num_pairs = jcspectra.size / (num_prof * num_hei)
632 622 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
633 623
634 624 for ip in range(num_pairs):
635 625
636 626 #-------------------------------------------
637 627
638 628 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
639 629 cspower = cspower[:, hei_interf]
640 630 cspower = cspower.sum(axis=0)
641 631
642 632 cspsort = cspower.ravel().argsort()
643 633 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
644 634 offhei_interf, nhei_interf + offhei_interf))]]]
645 635 junkcspc_interf = junkcspc_interf.transpose()
646 636 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
647 637
648 638 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
649 639
650 640 median_real = numpy.median(numpy.real(
651 641 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof / 4))]], :]))
652 642 median_imag = numpy.median(numpy.imag(
653 643 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof / 4))]], :]))
654 644 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
655 645 median_real, median_imag)
656 646
657 647 for iprof in range(num_prof):
658 648 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
659 649 jcspc_interf[iprof] = junkcspc_interf[iprof,
660 650 ind[nhei_interf / 2]]
661 651
662 652 # Removiendo la Interferencia
663 653 jcspectra[ip, :, ind_hei] = jcspectra[ip,
664 654 :, ind_hei] - jcspc_interf
665 655
666 656 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
667 657 maxid = ListAux.index(max(ListAux))
668 658
669 659 ind = numpy.array([-2, -1, 1, 2])
670 660 xx = numpy.zeros([4, 4])
671 661
672 662 for id1 in range(4):
673 663 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
674 664
675 665 xx_inv = numpy.linalg.inv(xx)
676 666 xx = xx_inv[:, 0]
677 667
678 668 ind = (ind + maxid + num_mask_prof) % num_mask_prof
679 669 yy = jcspectra[ip, mask_prof[ind], :]
680 670 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
681 671
682 672 # Guardar Resultados
683 673 self.dataOut.data_spc = jspectra
684 674 self.dataOut.data_cspc = jcspectra
685 675
686 676 return 1
687 677
688 678 def setRadarFrequency(self, frequency=None):
689 679
690 680 if frequency != None:
691 681 self.dataOut.frequency = frequency
692 682
693 683 return 1
694 684
695 685 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
696 686 # validacion de rango
697 687 if minHei == None:
698 688 minHei = self.dataOut.heightList[0]
699 689
700 690 if maxHei == None:
701 691 maxHei = self.dataOut.heightList[-1]
702 692
703 693 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
704 694 print('minHei: %.2f is out of the heights range' % (minHei))
705 695 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
706 696 minHei = self.dataOut.heightList[0]
707 697
708 698 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
709 699 print('maxHei: %.2f is out of the heights range' % (maxHei))
710 700 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
711 701 maxHei = self.dataOut.heightList[-1]
712 702
713 703 # validacion de velocidades
714 704 velrange = self.dataOut.getVelRange(1)
715 705
716 706 if minVel == None:
717 707 minVel = velrange[0]
718 708
719 709 if maxVel == None:
720 710 maxVel = velrange[-1]
721 711
722 712 if (minVel < velrange[0]) or (minVel > maxVel):
723 713 print('minVel: %.2f is out of the velocity range' % (minVel))
724 714 print('minVel is setting to %.2f' % (velrange[0]))
725 715 minVel = velrange[0]
726 716
727 717 if (maxVel > velrange[-1]) or (maxVel < minVel):
728 718 print('maxVel: %.2f is out of the velocity range' % (maxVel))
729 719 print('maxVel is setting to %.2f' % (velrange[-1]))
730 720 maxVel = velrange[-1]
731 721
732 722 # seleccion de indices para rango
733 723 minIndex = 0
734 724 maxIndex = 0
735 725 heights = self.dataOut.heightList
736 726
737 727 inda = numpy.where(heights >= minHei)
738 728 indb = numpy.where(heights <= maxHei)
739 729
740 730 try:
741 731 minIndex = inda[0][0]
742 732 except:
743 733 minIndex = 0
744 734
745 735 try:
746 736 maxIndex = indb[0][-1]
747 737 except:
748 738 maxIndex = len(heights)
749 739
750 740 if (minIndex < 0) or (minIndex > maxIndex):
751 741 raise ValueError("some value in (%d,%d) is not valid" % (
752 742 minIndex, maxIndex))
753 743
754 744 if (maxIndex >= self.dataOut.nHeights):
755 745 maxIndex = self.dataOut.nHeights - 1
756 746
757 747 # seleccion de indices para velocidades
758 748 indminvel = numpy.where(velrange >= minVel)
759 749 indmaxvel = numpy.where(velrange <= maxVel)
760 750 try:
761 751 minIndexVel = indminvel[0][0]
762 752 except:
763 753 minIndexVel = 0
764 754
765 755 try:
766 756 maxIndexVel = indmaxvel[0][-1]
767 757 except:
768 758 maxIndexVel = len(velrange)
769 759
770 760 # seleccion del espectro
771 761 data_spc = self.dataOut.data_spc[:,
772 762 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
773 763 # estimacion de ruido
774 764 noise = numpy.zeros(self.dataOut.nChannels)
775 765
776 766 for channel in range(self.dataOut.nChannels):
777 767 daux = data_spc[channel, :, :]
778 768 noise[channel] = hildebrand_sekhon(daux, self.dataOut.nIncohInt)
779 769
780 770 self.dataOut.noise_estimation = noise.copy()
781 771
782 772 return 1
783 773
784 774
785 775 class IncohInt(Operation):
786 776
787 777 __profIndex = 0
788 778 __withOverapping = False
789 779
790 780 __byTime = False
791 781 __initime = None
792 782 __lastdatatime = None
793 783 __integrationtime = None
794 784
795 785 __buffer_spc = None
796 786 __buffer_cspc = None
797 787 __buffer_dc = None
798 788
799 789 __dataReady = False
800 790
801 791 __timeInterval = None
802 792
803 793 n = None
804 794
805 def __init__(self):#, **kwargs):
795 def __init__(self):
806 796
807 Operation.__init__(self)#, **kwargs)
808
809
810 # self.isConfig = False
797 Operation.__init__(self)
811 798
812 799 def setup(self, n=None, timeInterval=None, overlapping=False):
813 800 """
814 801 Set the parameters of the integration class.
815 802
816 803 Inputs:
817 804
818 805 n : Number of coherent integrations
819 806 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
820 807 overlapping :
821 808
822 809 """
823 810
824 811 self.__initime = None
825 812 self.__lastdatatime = 0
826 813
827 814 self.__buffer_spc = 0
828 815 self.__buffer_cspc = 0
829 816 self.__buffer_dc = 0
830 817
831 818 self.__profIndex = 0
832 819 self.__dataReady = False
833 820 self.__byTime = False
834 821
835 822 if n is None and timeInterval is None:
836 823 raise ValueError("n or timeInterval should be specified ...")
837 824
838 825 if n is not None:
839 826 self.n = int(n)
840 827 else:
841 # if (type(timeInterval)!=integer) -> change this line
828
842 829 self.__integrationtime = int(timeInterval)
843 830 self.n = None
844 831 self.__byTime = True
845 832
846 833 def putData(self, data_spc, data_cspc, data_dc):
847 834 """
848 835 Add a profile to the __buffer_spc and increase in one the __profileIndex
849 836
850 837 """
851 838
852 839 self.__buffer_spc += data_spc
853 840
854 841 if data_cspc is None:
855 842 self.__buffer_cspc = None
856 843 else:
857 844 self.__buffer_cspc += data_cspc
858 845
859 846 if data_dc is None:
860 847 self.__buffer_dc = None
861 848 else:
862 849 self.__buffer_dc += data_dc
863 850
864 851 self.__profIndex += 1
865 852
866 853 return
867 854
868 855 def pushData(self):
869 856 """
870 857 Return the sum of the last profiles and the profiles used in the sum.
871 858
872 859 Affected:
873 860
874 861 self.__profileIndex
875 862
876 863 """
877 864
878 865 data_spc = self.__buffer_spc
879 866 data_cspc = self.__buffer_cspc
880 867 data_dc = self.__buffer_dc
881 868 n = self.__profIndex
882 869
883 870 self.__buffer_spc = 0
884 871 self.__buffer_cspc = 0
885 872 self.__buffer_dc = 0
886 873 self.__profIndex = 0
887 874
888 875 return data_spc, data_cspc, data_dc, n
889 876
890 877 def byProfiles(self, *args):
891 878
892 879 self.__dataReady = False
893 880 avgdata_spc = None
894 881 avgdata_cspc = None
895 882 avgdata_dc = None
896 883
897 884 self.putData(*args)
898 885
899 886 if self.__profIndex == self.n:
900 887
901 888 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
902 889 self.n = n
903 890 self.__dataReady = True
904 891
905 892 return avgdata_spc, avgdata_cspc, avgdata_dc
906 893
907 894 def byTime(self, datatime, *args):
908 895
909 896 self.__dataReady = False
910 897 avgdata_spc = None
911 898 avgdata_cspc = None
912 899 avgdata_dc = None
913 900
914 901 self.putData(*args)
915 902
916 903 if (datatime - self.__initime) >= self.__integrationtime:
917 904 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
918 905 self.n = n
919 906 self.__dataReady = True
920 907
921 908 return avgdata_spc, avgdata_cspc, avgdata_dc
922 909
923 910 def integrate(self, datatime, *args):
924 911
925 912 if self.__profIndex == 0:
926 913 self.__initime = datatime
927 914
928 915 if self.__byTime:
929 916 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
930 917 datatime, *args)
931 918 else:
932 919 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
933 920
934 921 if not self.__dataReady:
935 922 return None, None, None, None
936 923
937 924 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
938 925
939 926 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
940 927 if n == 1:
941 928 return
942 929
943 930 dataOut.flagNoData = True
944 931
945 932 if not self.isConfig:
946 933 self.setup(n, timeInterval, overlapping)
947 934 self.isConfig = True
948 935
949 936 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
950 937 dataOut.data_spc,
951 938 dataOut.data_cspc,
952 939 dataOut.data_dc)
953 940
954 941 if self.__dataReady:
955 942
956 943 dataOut.data_spc = avgdata_spc
957 944 dataOut.data_cspc = avgdata_cspc
958 945 dataOut.data_dc = avgdata_dc
959 946
960 947 dataOut.nIncohInt *= self.n
961 948 dataOut.utctime = avgdatatime
962 949 dataOut.flagNoData = False
963 950
964 951 return dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1330 +1,1330
1 1 import sys
2 2 import numpy
3 3 from scipy import interpolate
4 4 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
5 5 from schainpy.model.data.jrodata import Voltage
6 6 from schainpy.utils import log
7 7 from time import time
8 8
9 9
10 10 @MPDecorator
11 11 class VoltageProc(ProcessingUnit):
12 12
13 13 def __init__(self):
14 14
15 15 ProcessingUnit.__init__(self)
16 16
17 17 self.dataOut = Voltage()
18 18 self.flip = 1
19 19 self.setupReq = False
20 20
21 21 def run(self):
22 22
23 23 if self.dataIn.type == 'AMISR':
24 24 self.__updateObjFromAmisrInput()
25 25
26 26 if self.dataIn.type == 'Voltage':
27 27 self.dataOut.copy(self.dataIn)
28 28
29 29 # self.dataOut.copy(self.dataIn)
30 30
31 31 def __updateObjFromAmisrInput(self):
32 32
33 33 self.dataOut.timeZone = self.dataIn.timeZone
34 34 self.dataOut.dstFlag = self.dataIn.dstFlag
35 35 self.dataOut.errorCount = self.dataIn.errorCount
36 36 self.dataOut.useLocalTime = self.dataIn.useLocalTime
37 37
38 38 self.dataOut.flagNoData = self.dataIn.flagNoData
39 39 self.dataOut.data = self.dataIn.data
40 40 self.dataOut.utctime = self.dataIn.utctime
41 41 self.dataOut.channelList = self.dataIn.channelList
42 # self.dataOut.timeInterval = self.dataIn.timeInterval
42 #self.dataOut.timeInterval = self.dataIn.timeInterval
43 43 self.dataOut.heightList = self.dataIn.heightList
44 44 self.dataOut.nProfiles = self.dataIn.nProfiles
45 45
46 46 self.dataOut.nCohInt = self.dataIn.nCohInt
47 47 self.dataOut.ippSeconds = self.dataIn.ippSeconds
48 48 self.dataOut.frequency = self.dataIn.frequency
49 49
50 50 self.dataOut.azimuth = self.dataIn.azimuth
51 51 self.dataOut.zenith = self.dataIn.zenith
52 52
53 53 self.dataOut.beam.codeList = self.dataIn.beam.codeList
54 54 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
55 55 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
56 56 #
57 57 # pass#
58 58 #
59 59 # def init(self):
60 60 #
61 61 #
62 62 # if self.dataIn.type == 'AMISR':
63 63 # self.__updateObjFromAmisrInput()
64 64 #
65 65 # if self.dataIn.type == 'Voltage':
66 66 # self.dataOut.copy(self.dataIn)
67 67 # # No necesita copiar en cada init() los atributos de dataIn
68 68 # # la copia deberia hacerse por cada nuevo bloque de datos
69 69
70 70 def selectChannels(self, channelList):
71 71
72 72 channelIndexList = []
73 73
74 74 for channel in channelList:
75 75 if channel not in self.dataOut.channelList:
76 76 raise ValueError("Channel %d is not in %s" %(channel, str(self.dataOut.channelList)))
77 77
78 78 index = self.dataOut.channelList.index(channel)
79 79 channelIndexList.append(index)
80 80
81 81 self.selectChannelsByIndex(channelIndexList)
82 82
83 83 def selectChannelsByIndex(self, channelIndexList):
84 84 """
85 85 Selecciona un bloque de datos en base a canales segun el channelIndexList
86 86
87 87 Input:
88 88 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
89 89
90 90 Affected:
91 91 self.dataOut.data
92 92 self.dataOut.channelIndexList
93 93 self.dataOut.nChannels
94 94 self.dataOut.m_ProcessingHeader.totalSpectra
95 95 self.dataOut.systemHeaderObj.numChannels
96 96 self.dataOut.m_ProcessingHeader.blockSize
97 97
98 98 Return:
99 99 None
100 100 """
101 101
102 102 for channelIndex in channelIndexList:
103 103 if channelIndex not in self.dataOut.channelIndexList:
104 104 print(channelIndexList)
105 105 raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
106 106
107 107 if self.dataOut.flagDataAsBlock:
108 108 """
109 109 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
110 110 """
111 111 data = self.dataOut.data[channelIndexList,:,:]
112 112 else:
113 113 data = self.dataOut.data[channelIndexList,:]
114 114
115 115 self.dataOut.data = data
116 116 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
117 117 # self.dataOut.nChannels = nChannels
118 118
119 119 return 1
120 120
121 121 def selectHeights(self, minHei=None, maxHei=None):
122 122 """
123 123 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
124 124 minHei <= height <= maxHei
125 125
126 126 Input:
127 127 minHei : valor minimo de altura a considerar
128 128 maxHei : valor maximo de altura a considerar
129 129
130 130 Affected:
131 131 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
132 132
133 133 Return:
134 134 1 si el metodo se ejecuto con exito caso contrario devuelve 0
135 135 """
136 136
137 137 if minHei == None:
138 138 minHei = self.dataOut.heightList[0]
139 139
140 140 if maxHei == None:
141 141 maxHei = self.dataOut.heightList[-1]
142 142
143 143 if (minHei < self.dataOut.heightList[0]):
144 144 minHei = self.dataOut.heightList[0]
145 145
146 146 if (maxHei > self.dataOut.heightList[-1]):
147 147 maxHei = self.dataOut.heightList[-1]
148 148
149 149 minIndex = 0
150 150 maxIndex = 0
151 151 heights = self.dataOut.heightList
152 152
153 153 inda = numpy.where(heights >= minHei)
154 154 indb = numpy.where(heights <= maxHei)
155 155
156 156 try:
157 157 minIndex = inda[0][0]
158 158 except:
159 159 minIndex = 0
160 160
161 161 try:
162 162 maxIndex = indb[0][-1]
163 163 except:
164 164 maxIndex = len(heights)
165 165
166 166 self.selectHeightsByIndex(minIndex, maxIndex)
167 167
168 168 return 1
169 169
170 170
171 171 def selectHeightsByIndex(self, minIndex, maxIndex):
172 172 """
173 173 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
174 174 minIndex <= index <= maxIndex
175 175
176 176 Input:
177 177 minIndex : valor de indice minimo de altura a considerar
178 178 maxIndex : valor de indice maximo de altura a considerar
179 179
180 180 Affected:
181 181 self.dataOut.data
182 182 self.dataOut.heightList
183 183
184 184 Return:
185 185 1 si el metodo se ejecuto con exito caso contrario devuelve 0
186 186 """
187 187
188 188 if (minIndex < 0) or (minIndex > maxIndex):
189 189 raise ValueError("Height index range (%d,%d) is not valid" % (minIndex, maxIndex))
190 190
191 191 if (maxIndex >= self.dataOut.nHeights):
192 192 maxIndex = self.dataOut.nHeights
193 193
194 194 #voltage
195 195 if self.dataOut.flagDataAsBlock:
196 196 """
197 197 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
198 198 """
199 199 data = self.dataOut.data[:,:, minIndex:maxIndex]
200 200 else:
201 201 data = self.dataOut.data[:, minIndex:maxIndex]
202 202
203 203 # firstHeight = self.dataOut.heightList[minIndex]
204 204
205 205 self.dataOut.data = data
206 206 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
207 207
208 208 if self.dataOut.nHeights <= 1:
209 209 raise ValueError("selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights))
210 210
211 211 return 1
212 212
213 213
214 214 def filterByHeights(self, window):
215 215
216 216 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
217 217
218 218 if window == None:
219 219 window = (self.dataOut.radarControllerHeaderObj.txA/self.dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
220 220
221 221 newdelta = deltaHeight * window
222 222 r = self.dataOut.nHeights % window
223 223 newheights = (self.dataOut.nHeights-r)/window
224 224
225 225 if newheights <= 1:
226 226 raise ValueError("filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(self.dataOut.nHeights, window))
227 227
228 228 if self.dataOut.flagDataAsBlock:
229 229 """
230 230 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
231 231 """
232 232 buffer = self.dataOut.data[:, :, 0:self.dataOut.nHeights-r]
233 233 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nProfiles,self.dataOut.nHeights/window,window)
234 234 buffer = numpy.sum(buffer,3)
235 235
236 236 else:
237 237 buffer = self.dataOut.data[:,0:self.dataOut.nHeights-r]
238 238 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nHeights/window,window)
239 239 buffer = numpy.sum(buffer,2)
240 240
241 241 self.dataOut.data = buffer
242 242 self.dataOut.heightList = self.dataOut.heightList[0] + numpy.arange( newheights )*newdelta
243 243 self.dataOut.windowOfFilter = window
244 244
245 245 def setH0(self, h0, deltaHeight = None):
246 246
247 247 if not deltaHeight:
248 248 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
249 249
250 250 nHeights = self.dataOut.nHeights
251 251
252 252 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
253 253
254 254 self.dataOut.heightList = newHeiRange
255 255
256 256 def deFlip(self, channelList = []):
257 257
258 258 data = self.dataOut.data.copy()
259 259
260 260 if self.dataOut.flagDataAsBlock:
261 261 flip = self.flip
262 262 profileList = list(range(self.dataOut.nProfiles))
263 263
264 264 if not channelList:
265 265 for thisProfile in profileList:
266 266 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
267 267 flip *= -1.0
268 268 else:
269 269 for thisChannel in channelList:
270 270 if thisChannel not in self.dataOut.channelList:
271 271 continue
272 272
273 273 for thisProfile in profileList:
274 274 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
275 275 flip *= -1.0
276 276
277 277 self.flip = flip
278 278
279 279 else:
280 280 if not channelList:
281 281 data[:,:] = data[:,:]*self.flip
282 282 else:
283 283 for thisChannel in channelList:
284 284 if thisChannel not in self.dataOut.channelList:
285 285 continue
286 286
287 287 data[thisChannel,:] = data[thisChannel,:]*self.flip
288 288
289 289 self.flip *= -1.
290 290
291 291 self.dataOut.data = data
292 292
293 293 def setRadarFrequency(self, frequency=None):
294 294
295 295 if frequency != None:
296 296 self.dataOut.frequency = frequency
297 297
298 298 return 1
299 299
300 300 def interpolateHeights(self, topLim, botLim):
301 301 #69 al 72 para julia
302 302 #82-84 para meteoros
303 303 if len(numpy.shape(self.dataOut.data))==2:
304 304 sampInterp = (self.dataOut.data[:,botLim-1] + self.dataOut.data[:,topLim+1])/2
305 305 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
306 306 #self.dataOut.data[:,botLim:limSup+1] = sampInterp
307 307 self.dataOut.data[:,botLim:topLim+1] = sampInterp
308 308 else:
309 309 nHeights = self.dataOut.data.shape[2]
310 310 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
311 311 y = self.dataOut.data[:,:,list(range(botLim))+list(range(topLim+1,nHeights))]
312 312 f = interpolate.interp1d(x, y, axis = 2)
313 313 xnew = numpy.arange(botLim,topLim+1)
314 314 ynew = f(xnew)
315 315
316 316 self.dataOut.data[:,:,botLim:topLim+1] = ynew
317 317
318 318 # import collections
319 319
320 320 class CohInt(Operation):
321 321
322 322 isConfig = False
323 323 __profIndex = 0
324 324 __byTime = False
325 325 __initime = None
326 326 __lastdatatime = None
327 327 __integrationtime = None
328 328 __buffer = None
329 329 __bufferStride = []
330 330 __dataReady = False
331 331 __profIndexStride = 0
332 332 __dataToPutStride = False
333 333 n = None
334 334
335 def __init__(self):#, **kwargs):
335 def __init__(self, **kwargs):
336 336
337 Operation.__init__(self)#, **kwargs)
337 Operation.__init__(self, **kwargs)
338 338
339 339 # self.isConfig = False
340 340
341 341 def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False):
342 342 """
343 343 Set the parameters of the integration class.
344 344
345 345 Inputs:
346 346
347 347 n : Number of coherent integrations
348 348 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
349 349 overlapping :
350 350 """
351 351
352 352 self.__initime = None
353 353 self.__lastdatatime = 0
354 354 self.__buffer = None
355 355 self.__dataReady = False
356 356 self.byblock = byblock
357 357 self.stride = stride
358 358
359 359 if n == None and timeInterval == None:
360 360 raise ValueError("n or timeInterval should be specified ...")
361 361
362 362 if n != None:
363 363 self.n = n
364 364 self.__byTime = False
365 365 else:
366 366 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
367 367 self.n = 9999
368 368 self.__byTime = True
369 369
370 370 if overlapping:
371 371 self.__withOverlapping = True
372 372 self.__buffer = None
373 373 else:
374 374 self.__withOverlapping = False
375 375 self.__buffer = 0
376 376
377 377 self.__profIndex = 0
378 378
379 379 def putData(self, data):
380 380
381 381 """
382 382 Add a profile to the __buffer and increase in one the __profileIndex
383 383
384 384 """
385 385
386 386 if not self.__withOverlapping:
387 387 self.__buffer += data.copy()
388 388 self.__profIndex += 1
389 389 return
390 390
391 391 #Overlapping data
392 392 nChannels, nHeis = data.shape
393 393 data = numpy.reshape(data, (1, nChannels, nHeis))
394 394
395 395 #If the buffer is empty then it takes the data value
396 396 if self.__buffer is None:
397 397 self.__buffer = data
398 398 self.__profIndex += 1
399 399 return
400 400
401 401 #If the buffer length is lower than n then stakcing the data value
402 402 if self.__profIndex < self.n:
403 403 self.__buffer = numpy.vstack((self.__buffer, data))
404 404 self.__profIndex += 1
405 405 return
406 406
407 407 #If the buffer length is equal to n then replacing the last buffer value with the data value
408 408 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
409 409 self.__buffer[self.n-1] = data
410 410 self.__profIndex = self.n
411 411 return
412 412
413 413
414 414 def pushData(self):
415 415 """
416 416 Return the sum of the last profiles and the profiles used in the sum.
417 417
418 418 Affected:
419 419
420 420 self.__profileIndex
421 421
422 422 """
423 423
424 424 if not self.__withOverlapping:
425 425 data = self.__buffer
426 426 n = self.__profIndex
427 427
428 428 self.__buffer = 0
429 429 self.__profIndex = 0
430 430
431 431 return data, n
432 432
433 433 #Integration with Overlapping
434 434 data = numpy.sum(self.__buffer, axis=0)
435 435 # print data
436 436 # raise
437 437 n = self.__profIndex
438 438
439 439 return data, n
440 440
441 441 def byProfiles(self, data):
442 442
443 443 self.__dataReady = False
444 444 avgdata = None
445 445 # n = None
446 446 # print data
447 447 # raise
448 448 self.putData(data)
449 449
450 450 if self.__profIndex == self.n:
451 451 avgdata, n = self.pushData()
452 452 self.__dataReady = True
453 453
454 454 return avgdata
455 455
456 456 def byTime(self, data, datatime):
457 457
458 458 self.__dataReady = False
459 459 avgdata = None
460 460 n = None
461 461
462 462 self.putData(data)
463 463
464 464 if (datatime - self.__initime) >= self.__integrationtime:
465 465 avgdata, n = self.pushData()
466 466 self.n = n
467 467 self.__dataReady = True
468 468
469 469 return avgdata
470 470
471 471 def integrateByStride(self, data, datatime):
472 472 # print data
473 473 if self.__profIndex == 0:
474 474 self.__buffer = [[data.copy(), datatime]]
475 475 else:
476 476 self.__buffer.append([data.copy(),datatime])
477 477 self.__profIndex += 1
478 478 self.__dataReady = False
479 479
480 480 if self.__profIndex == self.n * self.stride :
481 481 self.__dataToPutStride = True
482 482 self.__profIndexStride = 0
483 483 self.__profIndex = 0
484 484 self.__bufferStride = []
485 485 for i in range(self.stride):
486 486 current = self.__buffer[i::self.stride]
487 487 data = numpy.sum([t[0] for t in current], axis=0)
488 488 avgdatatime = numpy.average([t[1] for t in current])
489 489 # print data
490 490 self.__bufferStride.append((data, avgdatatime))
491 491
492 492 if self.__dataToPutStride:
493 493 self.__dataReady = True
494 494 self.__profIndexStride += 1
495 495 if self.__profIndexStride == self.stride:
496 496 self.__dataToPutStride = False
497 497 # print self.__bufferStride[self.__profIndexStride - 1]
498 498 # raise
499 499 return self.__bufferStride[self.__profIndexStride - 1]
500 500
501 501
502 502 return None, None
503 503
504 504 def integrate(self, data, datatime=None):
505 505
506 506 if self.__initime == None:
507 507 self.__initime = datatime
508 508
509 509 if self.__byTime:
510 510 avgdata = self.byTime(data, datatime)
511 511 else:
512 512 avgdata = self.byProfiles(data)
513 513
514 514
515 515 self.__lastdatatime = datatime
516 516
517 517 if avgdata is None:
518 518 return None, None
519 519
520 520 avgdatatime = self.__initime
521 521
522 522 deltatime = datatime - self.__lastdatatime
523 523
524 524 if not self.__withOverlapping:
525 525 self.__initime = datatime
526 526 else:
527 527 self.__initime += deltatime
528 528
529 529 return avgdata, avgdatatime
530 530
531 531 def integrateByBlock(self, dataOut):
532 532
533 533 times = int(dataOut.data.shape[1]/self.n)
534 534 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
535 535
536 536 id_min = 0
537 537 id_max = self.n
538 538
539 539 for i in range(times):
540 540 junk = dataOut.data[:,id_min:id_max,:]
541 541 avgdata[:,i,:] = junk.sum(axis=1)
542 542 id_min += self.n
543 543 id_max += self.n
544 544
545 545 timeInterval = dataOut.ippSeconds*self.n
546 546 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
547 547 self.__dataReady = True
548 548 return avgdata, avgdatatime
549 549
550 550 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
551 551
552 552 if not self.isConfig:
553 553 self.setup(n=n, stride=stride, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
554 554 self.isConfig = True
555 555
556 556 if dataOut.flagDataAsBlock:
557 557 """
558 558 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
559 559 """
560 560 avgdata, avgdatatime = self.integrateByBlock(dataOut)
561 561 dataOut.nProfiles /= self.n
562 562 else:
563 563 if stride is None:
564 564 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
565 565 else:
566 566 avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
567 567
568 568
569 569 # dataOut.timeInterval *= n
570 570 dataOut.flagNoData = True
571 571
572 572 if self.__dataReady:
573 573 dataOut.data = avgdata
574 574 dataOut.nCohInt *= self.n
575 575 dataOut.utctime = avgdatatime
576 576 # print avgdata, avgdatatime
577 577 # raise
578 578 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
579 579 dataOut.flagNoData = False
580 580 return dataOut
581 581
582 582 class Decoder(Operation):
583 583
584 584 isConfig = False
585 585 __profIndex = 0
586 586
587 587 code = None
588 588
589 589 nCode = None
590 590 nBaud = None
591 591
592 def __init__(self):#, **kwargs):
592 def __init__(self, **kwargs):
593 593
594 Operation.__init__(self)#, **kwargs)
594 Operation.__init__(self, **kwargs)
595 595
596 596 self.times = None
597 597 self.osamp = None
598 598 # self.__setValues = False
599 # self.isConfig = False
599 self.isConfig = False
600 600 self.setupReq = False
601 601 def setup(self, code, osamp, dataOut):
602 602
603 603 self.__profIndex = 0
604 604
605 605 self.code = code
606 606
607 607 self.nCode = len(code)
608 608 self.nBaud = len(code[0])
609 609
610 610 if (osamp != None) and (osamp >1):
611 611 self.osamp = osamp
612 612 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
613 613 self.nBaud = self.nBaud*self.osamp
614 614
615 615 self.__nChannels = dataOut.nChannels
616 616 self.__nProfiles = dataOut.nProfiles
617 617 self.__nHeis = dataOut.nHeights
618 618
619 619 if self.__nHeis < self.nBaud:
620 620 raise ValueError('Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud))
621 621
622 622 #Frequency
623 623 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
624 624
625 625 __codeBuffer[:,0:self.nBaud] = self.code
626 626
627 627 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
628 628
629 629 if dataOut.flagDataAsBlock:
630 630
631 631 self.ndatadec = self.__nHeis #- self.nBaud + 1
632 632
633 633 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
634 634
635 635 else:
636 636
637 637 #Time
638 638 self.ndatadec = self.__nHeis #- self.nBaud + 1
639 639
640 640 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
641 641
642 642 def __convolutionInFreq(self, data):
643 643
644 644 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
645 645
646 646 fft_data = numpy.fft.fft(data, axis=1)
647 647
648 648 conv = fft_data*fft_code
649 649
650 650 data = numpy.fft.ifft(conv,axis=1)
651 651
652 652 return data
653 653
654 654 def __convolutionInFreqOpt(self, data):
655 655
656 656 raise NotImplementedError
657 657
658 658 def __convolutionInTime(self, data):
659 659
660 660 code = self.code[self.__profIndex]
661 661 for i in range(self.__nChannels):
662 662 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
663 663
664 664 return self.datadecTime
665 665
666 666 def __convolutionByBlockInTime(self, data):
667 667
668 668 repetitions = self.__nProfiles / self.nCode
669 669
670 670 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
671 671 junk = junk.flatten()
672 672 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
673 673 profilesList = range(self.__nProfiles)
674 674
675 675 for i in range(self.__nChannels):
676 676 for j in profilesList:
677 677 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
678 678 return self.datadecTime
679 679
680 680 def __convolutionByBlockInFreq(self, data):
681 681
682 682 raise NotImplementedError("Decoder by frequency fro Blocks not implemented")
683 683
684 684
685 685 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
686 686
687 687 fft_data = numpy.fft.fft(data, axis=2)
688 688
689 689 conv = fft_data*fft_code
690 690
691 691 data = numpy.fft.ifft(conv,axis=2)
692 692
693 693 return data
694 694
695 695
696 696 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
697 697
698 698 if dataOut.flagDecodeData:
699 699 print("This data is already decoded, recoding again ...")
700 700
701 701 if not self.isConfig:
702 702
703 703 if code is None:
704 704 if dataOut.code is None:
705 705 raise ValueError("Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type)
706 706
707 707 code = dataOut.code
708 708 else:
709 709 code = numpy.array(code).reshape(nCode,nBaud)
710 710 self.setup(code, osamp, dataOut)
711 711
712 712 self.isConfig = True
713 713
714 714 if mode == 3:
715 715 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
716 716
717 717 if times != None:
718 718 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
719 719
720 720 if self.code is None:
721 721 print("Fail decoding: Code is not defined.")
722 722 return
723 723
724 724 self.__nProfiles = dataOut.nProfiles
725 725 datadec = None
726 726
727 727 if mode == 3:
728 728 mode = 0
729 729
730 730 if dataOut.flagDataAsBlock:
731 731 """
732 732 Decoding when data have been read as block,
733 733 """
734 734
735 735 if mode == 0:
736 736 datadec = self.__convolutionByBlockInTime(dataOut.data)
737 737 if mode == 1:
738 738 datadec = self.__convolutionByBlockInFreq(dataOut.data)
739 739 else:
740 740 """
741 741 Decoding when data have been read profile by profile
742 742 """
743 743 if mode == 0:
744 744 datadec = self.__convolutionInTime(dataOut.data)
745 745
746 746 if mode == 1:
747 747 datadec = self.__convolutionInFreq(dataOut.data)
748 748
749 749 if mode == 2:
750 750 datadec = self.__convolutionInFreqOpt(dataOut.data)
751 751
752 752 if datadec is None:
753 753 raise ValueError("Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode)
754 754
755 755 dataOut.code = self.code
756 756 dataOut.nCode = self.nCode
757 757 dataOut.nBaud = self.nBaud
758 758
759 759 dataOut.data = datadec
760 760
761 761 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
762 762
763 763 dataOut.flagDecodeData = True #asumo q la data esta decodificada
764 764
765 765 if self.__profIndex == self.nCode-1:
766 766 self.__profIndex = 0
767 767 return dataOut
768 768
769 769 self.__profIndex += 1
770 770
771 771 return dataOut
772 772 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
773 773
774 774
775 775 class ProfileConcat(Operation):
776 776
777 777 isConfig = False
778 778 buffer = None
779 779
780 def __init__(self):#, **kwargs):
780 def __init__(self, **kwargs):
781 781
782 Operation.__init__(self)#, **kwargs)
782 Operation.__init__(self, **kwargs)
783 783 self.profileIndex = 0
784 784
785 785 def reset(self):
786 786 self.buffer = numpy.zeros_like(self.buffer)
787 787 self.start_index = 0
788 788 self.times = 1
789 789
790 790 def setup(self, data, m, n=1):
791 791 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
792 792 self.nHeights = data.shape[1]#.nHeights
793 793 self.start_index = 0
794 794 self.times = 1
795 795
796 796 def concat(self, data):
797 797
798 798 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
799 799 self.start_index = self.start_index + self.nHeights
800 800
801 801 def run(self, dataOut, m):
802 802
803 803 dataOut.flagNoData = True
804 804
805 805 if not self.isConfig:
806 806 self.setup(dataOut.data, m, 1)
807 807 self.isConfig = True
808 808
809 809 if dataOut.flagDataAsBlock:
810 810 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
811 811
812 812 else:
813 813 self.concat(dataOut.data)
814 814 self.times += 1
815 815 if self.times > m:
816 816 dataOut.data = self.buffer
817 817 self.reset()
818 818 dataOut.flagNoData = False
819 819 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
820 820 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
821 821 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
822 822 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
823 823 dataOut.ippSeconds *= m
824 824 return dataOut
825 825
826 826 class ProfileSelector(Operation):
827 827
828 828 profileIndex = None
829 829 # Tamanho total de los perfiles
830 830 nProfiles = None
831 831
832 def __init__(self):#, **kwargs):
832 def __init__(self, **kwargs):
833 833
834 Operation.__init__(self)#, **kwargs)
834 Operation.__init__(self, **kwargs)
835 835 self.profileIndex = 0
836 836
837 837 def incProfileIndex(self):
838 838
839 839 self.profileIndex += 1
840 840
841 841 if self.profileIndex >= self.nProfiles:
842 842 self.profileIndex = 0
843 843
844 844 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
845 845
846 846 if profileIndex < minIndex:
847 847 return False
848 848
849 849 if profileIndex > maxIndex:
850 850 return False
851 851
852 852 return True
853 853
854 854 def isThisProfileInList(self, profileIndex, profileList):
855 855
856 856 if profileIndex not in profileList:
857 857 return False
858 858
859 859 return True
860 860
861 861 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
862 862
863 863 """
864 864 ProfileSelector:
865 865
866 866 Inputs:
867 867 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
868 868
869 869 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
870 870
871 871 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
872 872
873 873 """
874 874
875 875 if rangeList is not None:
876 876 if type(rangeList[0]) not in (tuple, list):
877 877 rangeList = [rangeList]
878 878
879 879 dataOut.flagNoData = True
880 880
881 881 if dataOut.flagDataAsBlock:
882 882 """
883 883 data dimension = [nChannels, nProfiles, nHeis]
884 884 """
885 885 if profileList != None:
886 886 dataOut.data = dataOut.data[:,profileList,:]
887 887
888 888 if profileRangeList != None:
889 889 minIndex = profileRangeList[0]
890 890 maxIndex = profileRangeList[1]
891 891 profileList = list(range(minIndex, maxIndex+1))
892 892
893 893 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
894 894
895 895 if rangeList != None:
896 896
897 897 profileList = []
898 898
899 899 for thisRange in rangeList:
900 900 minIndex = thisRange[0]
901 901 maxIndex = thisRange[1]
902 902
903 903 profileList.extend(list(range(minIndex, maxIndex+1)))
904 904
905 905 dataOut.data = dataOut.data[:,profileList,:]
906 906
907 907 dataOut.nProfiles = len(profileList)
908 908 dataOut.profileIndex = dataOut.nProfiles - 1
909 909 dataOut.flagNoData = False
910 910
911 911 return True
912 912
913 913 """
914 914 data dimension = [nChannels, nHeis]
915 915 """
916 916
917 917 if profileList != None:
918 918
919 919 if self.isThisProfileInList(dataOut.profileIndex, profileList):
920 920
921 921 self.nProfiles = len(profileList)
922 922 dataOut.nProfiles = self.nProfiles
923 923 dataOut.profileIndex = self.profileIndex
924 924 dataOut.flagNoData = False
925 925
926 926 self.incProfileIndex()
927 927 return True
928 928
929 929 if profileRangeList != None:
930 930
931 931 minIndex = profileRangeList[0]
932 932 maxIndex = profileRangeList[1]
933 933
934 934 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
935 935
936 936 self.nProfiles = maxIndex - minIndex + 1
937 937 dataOut.nProfiles = self.nProfiles
938 938 dataOut.profileIndex = self.profileIndex
939 939 dataOut.flagNoData = False
940 940
941 941 self.incProfileIndex()
942 942 return True
943 943
944 944 if rangeList != None:
945 945
946 946 nProfiles = 0
947 947
948 948 for thisRange in rangeList:
949 949 minIndex = thisRange[0]
950 950 maxIndex = thisRange[1]
951 951
952 952 nProfiles += maxIndex - minIndex + 1
953 953
954 954 for thisRange in rangeList:
955 955
956 956 minIndex = thisRange[0]
957 957 maxIndex = thisRange[1]
958 958
959 959 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
960 960
961 961 self.nProfiles = nProfiles
962 962 dataOut.nProfiles = self.nProfiles
963 963 dataOut.profileIndex = self.profileIndex
964 964 dataOut.flagNoData = False
965 965
966 966 self.incProfileIndex()
967 967
968 968 break
969 969
970 970 return True
971 971
972 972
973 973 if beam != None: #beam is only for AMISR data
974 974 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
975 975 dataOut.flagNoData = False
976 976 dataOut.profileIndex = self.profileIndex
977 977
978 978 self.incProfileIndex()
979 979
980 980 return True
981 981
982 982 raise ValueError("ProfileSelector needs profileList, profileRangeList or rangeList parameter")
983 983
984 984 #return False
985 985 return dataOut
986 986
987 987 class Reshaper(Operation):
988 988
989 def __init__(self):#, **kwargs):
989 def __init__(self, **kwargs):
990 990
991 Operation.__init__(self)#, **kwargs)
991 Operation.__init__(self, **kwargs)
992 992
993 993 self.__buffer = None
994 994 self.__nitems = 0
995 995
996 996 def __appendProfile(self, dataOut, nTxs):
997 997
998 998 if self.__buffer is None:
999 999 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
1000 1000 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
1001 1001
1002 1002 ini = dataOut.nHeights * self.__nitems
1003 1003 end = ini + dataOut.nHeights
1004 1004
1005 1005 self.__buffer[:, ini:end] = dataOut.data
1006 1006
1007 1007 self.__nitems += 1
1008 1008
1009 1009 return int(self.__nitems*nTxs)
1010 1010
1011 1011 def __getBuffer(self):
1012 1012
1013 1013 if self.__nitems == int(1./self.__nTxs):
1014 1014
1015 1015 self.__nitems = 0
1016 1016
1017 1017 return self.__buffer.copy()
1018 1018
1019 1019 return None
1020 1020
1021 1021 def __checkInputs(self, dataOut, shape, nTxs):
1022 1022
1023 1023 if shape is None and nTxs is None:
1024 1024 raise ValueError("Reshaper: shape of factor should be defined")
1025 1025
1026 1026 if nTxs:
1027 1027 if nTxs < 0:
1028 1028 raise ValueError("nTxs should be greater than 0")
1029 1029
1030 1030 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
1031 1031 raise ValueError("nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs)))
1032 1032
1033 1033 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
1034 1034
1035 1035 return shape, nTxs
1036 1036
1037 1037 if len(shape) != 2 and len(shape) != 3:
1038 1038 raise ValueError("shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights))
1039 1039
1040 1040 if len(shape) == 2:
1041 1041 shape_tuple = [dataOut.nChannels]
1042 1042 shape_tuple.extend(shape)
1043 1043 else:
1044 1044 shape_tuple = list(shape)
1045 1045
1046 1046 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1047 1047
1048 1048 return shape_tuple, nTxs
1049 1049
1050 1050 def run(self, dataOut, shape=None, nTxs=None):
1051 1051
1052 1052 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1053 1053
1054 1054 dataOut.flagNoData = True
1055 1055 profileIndex = None
1056 1056
1057 1057 if dataOut.flagDataAsBlock:
1058 1058
1059 1059 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1060 1060 dataOut.flagNoData = False
1061 1061
1062 1062 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1063 1063
1064 1064 else:
1065 1065
1066 1066 if self.__nTxs < 1:
1067 1067
1068 1068 self.__appendProfile(dataOut, self.__nTxs)
1069 1069 new_data = self.__getBuffer()
1070 1070
1071 1071 if new_data is not None:
1072 1072 dataOut.data = new_data
1073 1073 dataOut.flagNoData = False
1074 1074
1075 1075 profileIndex = dataOut.profileIndex*nTxs
1076 1076
1077 1077 else:
1078 1078 raise ValueError("nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)")
1079 1079
1080 1080 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1081 1081
1082 1082 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1083 1083
1084 1084 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1085 1085
1086 1086 dataOut.profileIndex = profileIndex
1087 1087
1088 1088 dataOut.ippSeconds /= self.__nTxs
1089 1089
1090 1090 return dataOut
1091 1091
1092 1092 class SplitProfiles(Operation):
1093 1093
1094 def __init__(self):#, **kwargs):
1094 def __init__(self, **kwargs):
1095 1095
1096 Operation.__init__(self)#, **kwargs)
1096 Operation.__init__(self, **kwargs)
1097 1097
1098 1098 def run(self, dataOut, n):
1099 1099
1100 1100 dataOut.flagNoData = True
1101 1101 profileIndex = None
1102 1102
1103 1103 if dataOut.flagDataAsBlock:
1104 1104
1105 1105 #nchannels, nprofiles, nsamples
1106 1106 shape = dataOut.data.shape
1107 1107
1108 1108 if shape[2] % n != 0:
1109 1109 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
1110 1110
1111 1111 new_shape = shape[0], shape[1]*n, int(shape[2]/n)
1112 1112
1113 1113 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1114 1114 dataOut.flagNoData = False
1115 1115
1116 1116 profileIndex = int(dataOut.nProfiles/n) - 1
1117 1117
1118 1118 else:
1119 1119
1120 1120 raise ValueError("Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)")
1121 1121
1122 1122 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1123 1123
1124 1124 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1125 1125
1126 1126 dataOut.nProfiles = int(dataOut.nProfiles*n)
1127 1127
1128 1128 dataOut.profileIndex = profileIndex
1129 1129
1130 1130 dataOut.ippSeconds /= n
1131 1131
1132 1132 return dataOut
1133 1133
1134 1134 class CombineProfiles(Operation):
1135 def __init__(self):#, **kwargs):
1135 def __init__(self, **kwargs):
1136 1136
1137 Operation.__init__(self)#, **kwargs)
1137 Operation.__init__(self, **kwargs)
1138 1138
1139 1139 self.__remData = None
1140 1140 self.__profileIndex = 0
1141 1141
1142 1142 def run(self, dataOut, n):
1143 1143
1144 1144 dataOut.flagNoData = True
1145 1145 profileIndex = None
1146 1146
1147 1147 if dataOut.flagDataAsBlock:
1148 1148
1149 1149 #nchannels, nprofiles, nsamples
1150 1150 shape = dataOut.data.shape
1151 1151 new_shape = shape[0], shape[1]/n, shape[2]*n
1152 1152
1153 1153 if shape[1] % n != 0:
1154 1154 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[1]))
1155 1155
1156 1156 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1157 1157 dataOut.flagNoData = False
1158 1158
1159 1159 profileIndex = int(dataOut.nProfiles*n) - 1
1160 1160
1161 1161 else:
1162 1162
1163 1163 #nchannels, nsamples
1164 1164 if self.__remData is None:
1165 1165 newData = dataOut.data
1166 1166 else:
1167 1167 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1168 1168
1169 1169 self.__profileIndex += 1
1170 1170
1171 1171 if self.__profileIndex < n:
1172 1172 self.__remData = newData
1173 1173 #continue
1174 1174 return
1175 1175
1176 1176 self.__profileIndex = 0
1177 1177 self.__remData = None
1178 1178
1179 1179 dataOut.data = newData
1180 1180 dataOut.flagNoData = False
1181 1181
1182 1182 profileIndex = dataOut.profileIndex/n
1183 1183
1184 1184
1185 1185 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1186 1186
1187 1187 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1188 1188
1189 1189 dataOut.nProfiles = int(dataOut.nProfiles/n)
1190 1190
1191 1191 dataOut.profileIndex = profileIndex
1192 1192
1193 1193 dataOut.ippSeconds *= n
1194 1194
1195 1195 return dataOut
1196 1196 # import collections
1197 1197 # from scipy.stats import mode
1198 1198 #
1199 1199 # class Synchronize(Operation):
1200 1200 #
1201 1201 # isConfig = False
1202 1202 # __profIndex = 0
1203 1203 #
1204 1204 # def __init__(self, **kwargs):
1205 1205 #
1206 1206 # Operation.__init__(self, **kwargs)
1207 1207 # # self.isConfig = False
1208 1208 # self.__powBuffer = None
1209 1209 # self.__startIndex = 0
1210 1210 # self.__pulseFound = False
1211 1211 #
1212 1212 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1213 1213 #
1214 1214 # #Read data
1215 1215 #
1216 1216 # powerdB = dataOut.getPower(channel = channel)
1217 1217 # noisedB = dataOut.getNoise(channel = channel)[0]
1218 1218 #
1219 1219 # self.__powBuffer.extend(powerdB.flatten())
1220 1220 #
1221 1221 # dataArray = numpy.array(self.__powBuffer)
1222 1222 #
1223 1223 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1224 1224 #
1225 1225 # maxValue = numpy.nanmax(filteredPower)
1226 1226 #
1227 1227 # if maxValue < noisedB + 10:
1228 1228 # #No se encuentra ningun pulso de transmision
1229 1229 # return None
1230 1230 #
1231 1231 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1232 1232 #
1233 1233 # if len(maxValuesIndex) < 2:
1234 1234 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1235 1235 # return None
1236 1236 #
1237 1237 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1238 1238 #
1239 1239 # #Seleccionar solo valores con un espaciamiento de nSamples
1240 1240 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1241 1241 #
1242 1242 # if len(pulseIndex) < 2:
1243 1243 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1244 1244 # return None
1245 1245 #
1246 1246 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1247 1247 #
1248 1248 # #remover senales que se distancien menos de 10 unidades o muestras
1249 1249 # #(No deberian existir IPP menor a 10 unidades)
1250 1250 #
1251 1251 # realIndex = numpy.where(spacing > 10 )[0]
1252 1252 #
1253 1253 # if len(realIndex) < 2:
1254 1254 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1255 1255 # return None
1256 1256 #
1257 1257 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1258 1258 # realPulseIndex = pulseIndex[realIndex]
1259 1259 #
1260 1260 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1261 1261 #
1262 1262 # print "IPP = %d samples" %period
1263 1263 #
1264 1264 # self.__newNSamples = dataOut.nHeights #int(period)
1265 1265 # self.__startIndex = int(realPulseIndex[0])
1266 1266 #
1267 1267 # return 1
1268 1268 #
1269 1269 #
1270 1270 # def setup(self, nSamples, nChannels, buffer_size = 4):
1271 1271 #
1272 1272 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1273 1273 # maxlen = buffer_size*nSamples)
1274 1274 #
1275 1275 # bufferList = []
1276 1276 #
1277 1277 # for i in range(nChannels):
1278 1278 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1279 1279 # maxlen = buffer_size*nSamples)
1280 1280 #
1281 1281 # bufferList.append(bufferByChannel)
1282 1282 #
1283 1283 # self.__nSamples = nSamples
1284 1284 # self.__nChannels = nChannels
1285 1285 # self.__bufferList = bufferList
1286 1286 #
1287 1287 # def run(self, dataOut, channel = 0):
1288 1288 #
1289 1289 # if not self.isConfig:
1290 1290 # nSamples = dataOut.nHeights
1291 1291 # nChannels = dataOut.nChannels
1292 1292 # self.setup(nSamples, nChannels)
1293 1293 # self.isConfig = True
1294 1294 #
1295 1295 # #Append new data to internal buffer
1296 1296 # for thisChannel in range(self.__nChannels):
1297 1297 # bufferByChannel = self.__bufferList[thisChannel]
1298 1298 # bufferByChannel.extend(dataOut.data[thisChannel])
1299 1299 #
1300 1300 # if self.__pulseFound:
1301 1301 # self.__startIndex -= self.__nSamples
1302 1302 #
1303 1303 # #Finding Tx Pulse
1304 1304 # if not self.__pulseFound:
1305 1305 # indexFound = self.__findTxPulse(dataOut, channel)
1306 1306 #
1307 1307 # if indexFound == None:
1308 1308 # dataOut.flagNoData = True
1309 1309 # return
1310 1310 #
1311 1311 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1312 1312 # self.__pulseFound = True
1313 1313 # self.__startIndex = indexFound
1314 1314 #
1315 1315 # #If pulse was found ...
1316 1316 # for thisChannel in range(self.__nChannels):
1317 1317 # bufferByChannel = self.__bufferList[thisChannel]
1318 1318 # #print self.__startIndex
1319 1319 # x = numpy.array(bufferByChannel)
1320 1320 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1321 1321 #
1322 1322 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1323 1323 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1324 1324 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1325 1325 #
1326 1326 # dataOut.data = self.__arrayBuffer
1327 1327 #
1328 1328 # self.__startIndex += self.__newNSamples
1329 1329 #
1330 1330 # return
General Comments 0
You need to be logged in to leave comments. Login now