##// END OF EJS Templates
schain
RSS
Merge branch 'master' of http://jro-dev.igp.gob.pe/rhodecode/schain
Danny Scipión -
r1346:a455424d10b0 merge
parent child
Show More
Side by Side Unified
Context file:
r1346:a455424d10b0 Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,8 +1,8
1 1 """Signal chain python package"""
2 2
3 3 try:
4 4 from schainpy.controller import Project
5 5 except:
6 6 pass
7 7
8 __version__ = '3.0.0b5'
8 __version__ = '3.0.0b6'
Show file before | Show file after | Hide comments
@@ -1,1193 +1,1066
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Definition of diferent Data objects for different types of data
6 6
7 7 Here you will find the diferent data objects for the different types
8 8 of data, this data objects must be used as dataIn or dataOut objects in
9 9 processing units and operations. Currently the supported data objects are:
10 10 Voltage, Spectra, SpectraHeis, Fits, Correlation and Parameters
11 11 """
12 12
13 13 import copy
14 14 import numpy
15 15 import datetime
16 16 import json
17 17
18 18 import schainpy.admin
19 19 from schainpy.utils import log
20 20 from .jroheaderIO import SystemHeader, RadarControllerHeader
21 21 from schainpy.model.data import _noise
22 22
23 23
24 24 def getNumpyDtype(dataTypeCode):
25 25
26 26 if dataTypeCode == 0:
27 27 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
28 28 elif dataTypeCode == 1:
29 29 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
30 30 elif dataTypeCode == 2:
31 31 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
32 32 elif dataTypeCode == 3:
33 33 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
34 34 elif dataTypeCode == 4:
35 35 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
36 36 elif dataTypeCode == 5:
37 37 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
38 38 else:
39 39 raise ValueError('dataTypeCode was not defined')
40 40
41 41 return numpyDtype
42 42
43 43
44 44 def getDataTypeCode(numpyDtype):
45 45
46 46 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
47 47 datatype = 0
48 48 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
49 49 datatype = 1
50 50 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
51 51 datatype = 2
52 52 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
53 53 datatype = 3
54 54 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
55 55 datatype = 4
56 56 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
57 57 datatype = 5
58 58 else:
59 59 datatype = None
60 60
61 61 return datatype
62 62
63 63
64 64 def hildebrand_sekhon(data, navg):
65 65 """
66 66 This method is for the objective determination of the noise level in Doppler spectra. This
67 67 implementation technique is based on the fact that the standard deviation of the spectral
68 68 densities is equal to the mean spectral density for white Gaussian noise
69 69
70 70 Inputs:
71 71 Data : heights
72 72 navg : numbers of averages
73 73
74 74 Return:
75 75 mean : noise's level
76 76 """
77 77
78 78 sortdata = numpy.sort(data, axis=None)
79 79 '''
80 80 lenOfData = len(sortdata)
81 81 nums_min = lenOfData*0.2
82 82
83 83 if nums_min <= 5:
84 84
85 85 nums_min = 5
86 86
87 87 sump = 0.
88 88 sumq = 0.
89 89
90 90 j = 0
91 91 cont = 1
92 92
93 93 while((cont == 1)and(j < lenOfData)):
94 94
95 95 sump += sortdata[j]
96 96 sumq += sortdata[j]**2
97 97
98 98 if j > nums_min:
99 99 rtest = float(j)/(j-1) + 1.0/navg
100 100 if ((sumq*j) > (rtest*sump**2)):
101 101 j = j - 1
102 102 sump = sump - sortdata[j]
103 103 sumq = sumq - sortdata[j]**2
104 104 cont = 0
105 105
106 106 j += 1
107 107
108 108 lnoise = sump / j
109 109 '''
110 110 return _noise.hildebrand_sekhon(sortdata, navg)
111 111
112 112
113 113 class Beam:
114 114
115 115 def __init__(self):
116 116 self.codeList = []
117 117 self.azimuthList = []
118 118 self.zenithList = []
119 119
120 120
121 121 class GenericData(object):
122 122
123 123 flagNoData = True
124 124
125 125 def copy(self, inputObj=None):
126 126
127 127 if inputObj == None:
128 128 return copy.deepcopy(self)
129 129
130 130 for key in list(inputObj.__dict__.keys()):
131 131
132 132 attribute = inputObj.__dict__[key]
133 133
134 134 # If this attribute is a tuple or list
135 135 if type(inputObj.__dict__[key]) in (tuple, list):
136 136 self.__dict__[key] = attribute[:]
137 137 continue
138 138
139 139 # If this attribute is another object or instance
140 140 if hasattr(attribute, '__dict__'):
141 141 self.__dict__[key] = attribute.copy()
142 142 continue
143 143
144 144 self.__dict__[key] = inputObj.__dict__[key]
145 145
146 146 def deepcopy(self):
147 147
148 148 return copy.deepcopy(self)
149 149
150 150 def isEmpty(self):
151 151
152 152 return self.flagNoData
153 153
154 154 def isReady(self):
155 155
156 156 return not self.flagNoData
157 157
158 158
159 159 class JROData(GenericData):
160 160
161 161 systemHeaderObj = SystemHeader()
162 162 radarControllerHeaderObj = RadarControllerHeader()
163 163 type = None
164 164 datatype = None # dtype but in string
165 165 nProfiles = None
166 166 heightList = None
167 167 channelList = None
168 168 flagDiscontinuousBlock = False
169 169 useLocalTime = False
170 170 utctime = None
171 171 timeZone = None
172 172 dstFlag = None
173 173 errorCount = None
174 174 blocksize = None
175 175 flagDecodeData = False # asumo q la data no esta decodificada
176 176 flagDeflipData = False # asumo q la data no esta sin flip
177 177 flagShiftFFT = False
178 178 nCohInt = None
179 179 windowOfFilter = 1
180 180 C = 3e8
181 181 frequency = 49.92e6
182 182 realtime = False
183 183 beacon_heiIndexList = None
184 184 last_block = None
185 185 blocknow = None
186 186 azimuth = None
187 187 zenith = None
188 188 beam = Beam()
189 189 profileIndex = None
190 190 error = None
191 191 data = None
192 192 nmodes = None
193 193 metadata_list = ['heightList', 'timeZone', 'type']
194 194
195 195 def __str__(self):
196 196
197 197 return '{} - {}'.format(self.type, self.datatime())
198 198
199 199 def getNoise(self):
200 200
201 201 raise NotImplementedError
202 202
203 203 @property
204 204 def nChannels(self):
205 205
206 206 return len(self.channelList)
207 207
208 208 @property
209 209 def channelIndexList(self):
210 210
211 211 return list(range(self.nChannels))
212 212
213 213 @property
214 214 def nHeights(self):
215 215
216 216 return len(self.heightList)
217 217
218 218 def getDeltaH(self):
219 219
220 220 return self.heightList[1] - self.heightList[0]
221 221
222 222 @property
223 223 def ltctime(self):
224 224
225 225 if self.useLocalTime:
226 226 return self.utctime - self.timeZone * 60
227 227
228 228 return self.utctime
229 229
230 230 @property
231 231 def datatime(self):
232 232
233 233 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
234 234 return datatimeValue
235 235
236 236 def getTimeRange(self):
237 237
238 238 datatime = []
239 239
240 240 datatime.append(self.ltctime)
241 241 datatime.append(self.ltctime + self.timeInterval + 1)
242 242
243 243 datatime = numpy.array(datatime)
244 244
245 245 return datatime
246 246
247 247 def getFmaxTimeResponse(self):
248 248
249 249 period = (10**-6) * self.getDeltaH() / (0.15)
250 250
251 251 PRF = 1. / (period * self.nCohInt)
252 252
253 253 fmax = PRF
254 254
255 255 return fmax
256 256
257 257 def getFmax(self):
258 258 PRF = 1. / (self.ippSeconds * self.nCohInt)
259 259
260 260 fmax = PRF
261 261 return fmax
262 262
263 263 def getVmax(self):
264 264
265 265 _lambda = self.C / self.frequency
266 266
267 267 vmax = self.getFmax() * _lambda / 2
268 268
269 269 return vmax
270 270
271 271 @property
272 272 def ippSeconds(self):
273 273 '''
274 274 '''
275 275 return self.radarControllerHeaderObj.ippSeconds
276 276
277 277 @ippSeconds.setter
278 278 def ippSeconds(self, ippSeconds):
279 279 '''
280 280 '''
281 281 self.radarControllerHeaderObj.ippSeconds = ippSeconds
282 282
283 283 @property
284 284 def code(self):
285 285 '''
286 286 '''
287 287 return self.radarControllerHeaderObj.code
288 288
289 289 @code.setter
290 290 def code(self, code):
291 291 '''
292 292 '''
293 293 self.radarControllerHeaderObj.code = code
294 294
295 295 @property
296 296 def nCode(self):
297 297 '''
298 298 '''
299 299 return self.radarControllerHeaderObj.nCode
300 300
301 301 @nCode.setter
302 302 def nCode(self, ncode):
303 303 '''
304 304 '''
305 305 self.radarControllerHeaderObj.nCode = ncode
306 306
307 307 @property
308 308 def nBaud(self):
309 309 '''
310 310 '''
311 311 return self.radarControllerHeaderObj.nBaud
312 312
313 313 @nBaud.setter
314 314 def nBaud(self, nbaud):
315 315 '''
316 316 '''
317 317 self.radarControllerHeaderObj.nBaud = nbaud
318 318
319 319 @property
320 320 def ipp(self):
321 321 '''
322 322 '''
323 323 return self.radarControllerHeaderObj.ipp
324 324
325 325 @ipp.setter
326 326 def ipp(self, ipp):
327 327 '''
328 328 '''
329 329 self.radarControllerHeaderObj.ipp = ipp
330 330
331 331 @property
332 332 def metadata(self):
333 333 '''
334 334 '''
335 335
336 336 return {attr: getattr(self, attr) for attr in self.metadata_list}
337 337
338 338
339 339 class Voltage(JROData):
340 340
341 341 dataPP_POW = None
342 342 dataPP_DOP = None
343 343 dataPP_WIDTH = None
344 344 dataPP_SNR = None
345 345
346 346 def __init__(self):
347 347 '''
348 348 Constructor
349 349 '''
350 350
351 351 self.useLocalTime = True
352 352 self.radarControllerHeaderObj = RadarControllerHeader()
353 353 self.systemHeaderObj = SystemHeader()
354 354 self.type = "Voltage"
355 355 self.data = None
356 356 self.nProfiles = None
357 357 self.heightList = None
358 358 self.channelList = None
359 359 self.flagNoData = True
360 360 self.flagDiscontinuousBlock = False
361 361 self.utctime = None
362 362 self.timeZone = 0
363 363 self.dstFlag = None
364 364 self.errorCount = None
365 365 self.nCohInt = None
366 366 self.blocksize = None
367 367 self.flagCohInt = False
368 368 self.flagDecodeData = False # asumo q la data no esta decodificada
369 369 self.flagDeflipData = False # asumo q la data no esta sin flip
370 370 self.flagShiftFFT = False
371 371 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
372 372 self.profileIndex = 0
373 373 self.metadata_list = ['type', 'heightList', 'timeZone', 'nProfiles', 'channelList', 'nCohInt',
374 374 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp']
375 375
376 376 def getNoisebyHildebrand(self, channel=None):
377 377 """
378 378 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
379 379
380 380 Return:
381 381 noiselevel
382 382 """
383 383
384 384 if channel != None:
385 385 data = self.data[channel]
386 386 nChannels = 1
387 387 else:
388 388 data = self.data
389 389 nChannels = self.nChannels
390 390
391 391 noise = numpy.zeros(nChannels)
392 392 power = data * numpy.conjugate(data)
393 393
394 394 for thisChannel in range(nChannels):
395 395 if nChannels == 1:
396 396 daux = power[:].real
397 397 else:
398 398 daux = power[thisChannel, :].real
399 399 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
400 400
401 401 return noise
402 402
403 403 def getNoise(self, type=1, channel=None):
404 404
405 405 if type == 1:
406 406 noise = self.getNoisebyHildebrand(channel)
407 407
408 408 return noise
409 409
410 410 def getPower(self, channel=None):
411 411
412 412 if channel != None:
413 413 data = self.data[channel]
414 414 else:
415 415 data = self.data
416 416
417 417 power = data * numpy.conjugate(data)
418 418 powerdB = 10 * numpy.log10(power.real)
419 419 powerdB = numpy.squeeze(powerdB)
420 420
421 421 return powerdB
422 422
423 423 @property
424 424 def timeInterval(self):
425 425
426 426 return self.ippSeconds * self.nCohInt
427 427
428 428 noise = property(getNoise, "I'm the 'nHeights' property.")
429 429
430 430
431 431 class Spectra(JROData):
432 432
433 433 def __init__(self):
434 434 '''
435 435 Constructor
436 436 '''
437 437
438 438 self.useLocalTime = True
439 439 self.radarControllerHeaderObj = RadarControllerHeader()
440 440 self.systemHeaderObj = SystemHeader()
441 441 self.type = "Spectra"
442 442 self.timeZone = 0
443 443 self.nProfiles = None
444 444 self.heightList = None
445 445 self.channelList = None
446 446 self.pairsList = None
447 447 self.flagNoData = True
448 448 self.flagDiscontinuousBlock = False
449 449 self.utctime = None
450 450 self.nCohInt = None
451 451 self.nIncohInt = None
452 452 self.blocksize = None
453 453 self.nFFTPoints = None
454 454 self.wavelength = None
455 455 self.flagDecodeData = False # asumo q la data no esta decodificada
456 456 self.flagDeflipData = False # asumo q la data no esta sin flip
457 457 self.flagShiftFFT = False
458 458 self.ippFactor = 1
459 459 self.beacon_heiIndexList = []
460 460 self.noise_estimation = None
461 461 self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt',
462 462 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp','nIncohInt', 'nFFTPoints', 'nProfiles']
463 463
464 464 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
465 465 """
466 466 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
467 467
468 468 Return:
469 469 noiselevel
470 470 """
471 471
472 472 noise = numpy.zeros(self.nChannels)
473 473
474 474 for channel in range(self.nChannels):
475 475 daux = self.data_spc[channel,
476 476 xmin_index:xmax_index, ymin_index:ymax_index]
477 477 noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
478 478
479 479 return noise
480 480
481 481 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
482 482
483 483 if self.noise_estimation is not None:
484 484 # this was estimated by getNoise Operation defined in jroproc_spectra.py
485 485 return self.noise_estimation
486 486 else:
487 487 noise = self.getNoisebyHildebrand(
488 488 xmin_index, xmax_index, ymin_index, ymax_index)
489 489 return noise
490 490
491 491 def getFreqRangeTimeResponse(self, extrapoints=0):
492 492
493 493 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
494 494 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
495 495
496 496 return freqrange
497 497
498 498 def getAcfRange(self, extrapoints=0):
499 499
500 500 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
501 501 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
502 502
503 503 return freqrange
504 504
505 505 def getFreqRange(self, extrapoints=0):
506 506
507 507 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
508 508 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
509 509
510 510 return freqrange
511 511
512 512 def getVelRange(self, extrapoints=0):
513 513
514 514 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
515 515 velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
516 516
517 517 if self.nmodes:
518 518 return velrange/self.nmodes
519 519 else:
520 520 return velrange
521 521
522 522 @property
523 523 def nPairs(self):
524 524
525 525 return len(self.pairsList)
526 526
527 527 @property
528 528 def pairsIndexList(self):
529 529
530 530 return list(range(self.nPairs))
531 531
532 532 @property
533 533 def normFactor(self):
534 534
535 535 pwcode = 1
536 536
537 537 if self.flagDecodeData:
538 538 pwcode = numpy.sum(self.code[0]**2)
539 539 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
540 540 normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
541 541
542 542 return normFactor
543 543
544 544 @property
545 545 def flag_cspc(self):
546 546
547 547 if self.data_cspc is None:
548 548 return True
549 549
550 550 return False
551 551
552 552 @property
553 553 def flag_dc(self):
554 554
555 555 if self.data_dc is None:
556 556 return True
557 557
558 558 return False
559 559
560 560 @property
561 561 def timeInterval(self):
562 562
563 563 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
564 564 if self.nmodes:
565 565 return self.nmodes*timeInterval
566 566 else:
567 567 return timeInterval
568 568
569 569 def getPower(self):
570 570
571 571 factor = self.normFactor
572 572 z = self.data_spc / factor
573 573 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
574 574 avg = numpy.average(z, axis=1)
575 575
576 576 return 10 * numpy.log10(avg)
577 577
578 578 def getCoherence(self, pairsList=None, phase=False):
579 579
580 580 z = []
581 581 if pairsList is None:
582 582 pairsIndexList = self.pairsIndexList
583 583 else:
584 584 pairsIndexList = []
585 585 for pair in pairsList:
586 586 if pair not in self.pairsList:
587 587 raise ValueError("Pair %s is not in dataOut.pairsList" % (
588 588 pair))
589 589 pairsIndexList.append(self.pairsList.index(pair))
590 590 for i in range(len(pairsIndexList)):
591 591 pair = self.pairsList[pairsIndexList[i]]
592 592 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
593 593 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
594 594 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
595 595 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
596 596 if phase:
597 597 data = numpy.arctan2(avgcoherenceComplex.imag,
598 598 avgcoherenceComplex.real) * 180 / numpy.pi
599 599 else:
600 600 data = numpy.abs(avgcoherenceComplex)
601 601
602 602 z.append(data)
603 603
604 604 return numpy.array(z)
605 605
606 606 def setValue(self, value):
607 607
608 608 print("This property should not be initialized")
609 609
610 610 return
611 611
612 612 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
613 613
614 614
615 615 class SpectraHeis(Spectra):
616 616
617 617 def __init__(self):
618 618
619 619 self.radarControllerHeaderObj = RadarControllerHeader()
620 620 self.systemHeaderObj = SystemHeader()
621 621 self.type = "SpectraHeis"
622 622 self.nProfiles = None
623 623 self.heightList = None
624 624 self.channelList = None
625 625 self.flagNoData = True
626 626 self.flagDiscontinuousBlock = False
627 627 self.utctime = None
628 628 self.blocksize = None
629 629 self.profileIndex = 0
630 630 self.nCohInt = 1
631 631 self.nIncohInt = 1
632 632
633 633 @property
634 634 def normFactor(self):
635 635 pwcode = 1
636 636 if self.flagDecodeData:
637 637 pwcode = numpy.sum(self.code[0]**2)
638 638
639 639 normFactor = self.nIncohInt * self.nCohInt * pwcode
640 640
641 641 return normFactor
642 642
643 643 @property
644 644 def timeInterval(self):
645 645
646 646 return self.ippSeconds * self.nCohInt * self.nIncohInt
647 647
648 648
649 649 class Fits(JROData):
650 650
651 651 def __init__(self):
652 652
653 653 self.type = "Fits"
654 654 self.nProfiles = None
655 655 self.heightList = None
656 656 self.channelList = None
657 657 self.flagNoData = True
658 658 self.utctime = None
659 659 self.nCohInt = 1
660 660 self.nIncohInt = 1
661 661 self.useLocalTime = True
662 662 self.profileIndex = 0
663 663 self.timeZone = 0
664 664
665 665 def getTimeRange(self):
666 666
667 667 datatime = []
668 668
669 669 datatime.append(self.ltctime)
670 670 datatime.append(self.ltctime + self.timeInterval)
671 671
672 672 datatime = numpy.array(datatime)
673 673
674 674 return datatime
675 675
676 676 def getChannelIndexList(self):
677 677
678 678 return list(range(self.nChannels))
679 679
680 680 def getNoise(self, type=1):
681 681
682 682
683 683 if type == 1:
684 684 noise = self.getNoisebyHildebrand()
685 685
686 686 if type == 2:
687 687 noise = self.getNoisebySort()
688 688
689 689 if type == 3:
690 690 noise = self.getNoisebyWindow()
691 691
692 692 return noise
693 693
694 694 @property
695 695 def timeInterval(self):
696 696
697 697 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
698 698
699 699 return timeInterval
700 700
701 701 @property
702 702 def ippSeconds(self):
703 703 '''
704 704 '''
705 705 return self.ipp_sec
706 706
707 707 noise = property(getNoise, "I'm the 'nHeights' property.")
708 708
709 709
710 710 class Correlation(JROData):
711 711
712 712 def __init__(self):
713 713 '''
714 714 Constructor
715 715 '''
716 716 self.radarControllerHeaderObj = RadarControllerHeader()
717 717 self.systemHeaderObj = SystemHeader()
718 718 self.type = "Correlation"
719 719 self.data = None
720 720 self.dtype = None
721 721 self.nProfiles = None
722 722 self.heightList = None
723 723 self.channelList = None
724 724 self.flagNoData = True
725 725 self.flagDiscontinuousBlock = False
726 726 self.utctime = None
727 727 self.timeZone = 0
728 728 self.dstFlag = None
729 729 self.errorCount = None
730 730 self.blocksize = None
731 731 self.flagDecodeData = False # asumo q la data no esta decodificada
732 732 self.flagDeflipData = False # asumo q la data no esta sin flip
733 733 self.pairsList = None
734 734 self.nPoints = None
735 735
736 736 def getPairsList(self):
737 737
738 738 return self.pairsList
739 739
740 740 def getNoise(self, mode=2):
741 741
742 742 indR = numpy.where(self.lagR == 0)[0][0]
743 743 indT = numpy.where(self.lagT == 0)[0][0]
744 744
745 745 jspectra0 = self.data_corr[:, :, indR, :]
746 746 jspectra = copy.copy(jspectra0)
747 747
748 748 num_chan = jspectra.shape[0]
749 749 num_hei = jspectra.shape[2]
750 750
751 751 freq_dc = jspectra.shape[1] / 2
752 752 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
753 753
754 754 if ind_vel[0] < 0:
755 755 ind_vel[list(range(0, 1))] = ind_vel[list(
756 756 range(0, 1))] + self.num_prof
757 757
758 758 if mode == 1:
759 759 jspectra[:, freq_dc, :] = (
760 760 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
761 761
762 762 if mode == 2:
763 763
764 764 vel = numpy.array([-2, -1, 1, 2])
765 765 xx = numpy.zeros([4, 4])
766 766
767 767 for fil in range(4):
768 768 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
769 769
770 770 xx_inv = numpy.linalg.inv(xx)
771 771 xx_aux = xx_inv[0, :]
772 772
773 773 for ich in range(num_chan):
774 774 yy = jspectra[ich, ind_vel, :]
775 775 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
776 776
777 777 junkid = jspectra[ich, freq_dc, :] <= 0
778 778 cjunkid = sum(junkid)
779 779
780 780 if cjunkid.any():
781 781 jspectra[ich, freq_dc, junkid.nonzero()] = (
782 782 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
783 783
784 784 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
785 785
786 786 return noise
787 787
788 788 @property
789 789 def timeInterval(self):
790 790
791 791 return self.ippSeconds * self.nCohInt * self.nProfiles
792 792
793 793 def splitFunctions(self):
794 794
795 795 pairsList = self.pairsList
796 796 ccf_pairs = []
797 797 acf_pairs = []
798 798 ccf_ind = []
799 799 acf_ind = []
800 800 for l in range(len(pairsList)):
801 801 chan0 = pairsList[l][0]
802 802 chan1 = pairsList[l][1]
803 803
804 804 # Obteniendo pares de Autocorrelacion
805 805 if chan0 == chan1:
806 806 acf_pairs.append(chan0)
807 807 acf_ind.append(l)
808 808 else:
809 809 ccf_pairs.append(pairsList[l])
810 810 ccf_ind.append(l)
811 811
812 812 data_acf = self.data_cf[acf_ind]
813 813 data_ccf = self.data_cf[ccf_ind]
814 814
815 815 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
816 816
817 817 @property
818 818 def normFactor(self):
819 819 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
820 820 acf_pairs = numpy.array(acf_pairs)
821 821 normFactor = numpy.zeros((self.nPairs, self.nHeights))
822 822
823 823 for p in range(self.nPairs):
824 824 pair = self.pairsList[p]
825 825
826 826 ch0 = pair[0]
827 827 ch1 = pair[1]
828 828
829 829 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
830 830 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
831 831 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
832 832
833 833 return normFactor
834 834
835 835
836 836 class Parameters(Spectra):
837 837
838 838 groupList = None # List of Pairs, Groups, etc
839 839 data_param = None # Parameters obtained
840 840 data_pre = None # Data Pre Parametrization
841 841 data_SNR = None # Signal to Noise Ratio
842 842 abscissaList = None # Abscissa, can be velocities, lags or time
843 843 utctimeInit = None # Initial UTC time
844 844 paramInterval = None # Time interval to calculate Parameters in seconds
845 845 useLocalTime = True
846 846 # Fitting
847 847 data_error = None # Error of the estimation
848 848 constants = None
849 849 library = None
850 850 # Output signal
851 851 outputInterval = None # Time interval to calculate output signal in seconds
852 852 data_output = None # Out signal
853 853 nAvg = None
854 854 noise_estimation = None
855 855 GauSPC = None # Fit gaussian SPC
856 856
857 857 def __init__(self):
858 858 '''
859 859 Constructor
860 860 '''
861 861 self.radarControllerHeaderObj = RadarControllerHeader()
862 862 self.systemHeaderObj = SystemHeader()
863 863 self.type = "Parameters"
864 864 self.timeZone = 0
865 865
866 866 def getTimeRange1(self, interval):
867 867
868 868 datatime = []
869 869
870 870 if self.useLocalTime:
871 871 time1 = self.utctimeInit - self.timeZone * 60
872 872 else:
873 873 time1 = self.utctimeInit
874 874
875 875 datatime.append(time1)
876 876 datatime.append(time1 + interval)
877 877 datatime = numpy.array(datatime)
878 878
879 879 return datatime
880 880
881 881 @property
882 882 def timeInterval(self):
883 883
884 884 if hasattr(self, 'timeInterval1'):
885 885 return self.timeInterval1
886 886 else:
887 887 return self.paramInterval
888 888
889 889 def setValue(self, value):
890 890
891 891 print("This property should not be initialized")
892 892
893 893 return
894 894
895 895 def getNoise(self):
896 896
897 897 return self.spc_noise
898 898
899 899 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
900 900
901 901
902 902 class PlotterData(object):
903 903 '''
904 904 Object to hold data to be plotted
905 905 '''
906 906
907 907 MAXNUMX = 200
908 908 MAXNUMY = 200
909 909
910 def __init__(self, code, throttle_value, exp_code, localtime=True, buffering=True, snr=False):
910 def __init__(self, code, exp_code, localtime=True):
911 911
912 912 self.key = code
913 self.throttle = throttle_value
914 913 self.exp_code = exp_code
915 self.buffering = buffering
916 914 self.ready = False
917 915 self.flagNoData = False
918 916 self.localtime = localtime
919 917 self.data = {}
920 918 self.meta = {}
921 919 self.__heights = []
922 920
923 if 'snr' in code:
924 self.plottypes = ['snr']
925 elif code == 'spc':
926 self.plottypes = ['spc', 'noise', 'rti']
927 elif code == 'cspc':
928 self.plottypes = ['cspc', 'spc', 'noise', 'rti']
929 elif code == 'rti':
930 self.plottypes = ['noise', 'rti']
931 else:
932 self.plottypes = [code]
933
934 if 'snr' not in self.plottypes and snr:
935 self.plottypes.append('snr')
936
937 for plot in self.plottypes:
938 self.data[plot] = {}
939
940 921 def __str__(self):
941 922 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
942 923 return 'Data[{}][{}]'.format(';'.join(dum), len(self.times))
943 924
944 925 def __len__(self):
945 return len(self.data[self.key])
926 return len(self.data)
946 927
947 928 def __getitem__(self, key):
948
949 if key not in self.data:
950 raise KeyError(log.error('Missing key: {}'.format(key)))
951 if 'spc' in key or not self.buffering:
952 ret = self.data[key][self.tm]
953 elif 'scope' in key:
954 ret = numpy.array(self.data[key][float(self.tm)])
955 else:
956 ret = numpy.array([self.data[key][x] for x in self.times])
929 if isinstance(key, int):
930 return self.data[self.times[key]]
931 elif isinstance(key, str):
932 ret = numpy.array([self.data[x][key] for x in self.times])
957 933 if ret.ndim > 1:
958 934 ret = numpy.swapaxes(ret, 0, 1)
959 return ret
935 return ret
960 936
961 937 def __contains__(self, key):
962 return key in self.data
938 return key in self.data[self.min_time]
963 939
964 940 def setup(self):
965 941 '''
966 942 Configure object
967 943 '''
968 944 self.type = ''
969 945 self.ready = False
970 946 del self.data
971 947 self.data = {}
972 948 self.__heights = []
973 949 self.__all_heights = set()
974 for plot in self.plottypes:
975 if 'snr' in plot:
976 plot = 'snr'
977 elif 'spc_moments' == plot:
978 plot = 'moments'
979 self.data[plot] = {}
980
981 if 'spc' in self.data or 'rti' in self.data or 'cspc' in self.data or 'moments' in self.data:
982 self.data['noise'] = {}
983 self.data['rti'] = {}
984 if 'noise' not in self.plottypes:
985 self.plottypes.append('noise')
986 if 'rti' not in self.plottypes:
987 self.plottypes.append('rti')
988 950
989 951 def shape(self, key):
990 952 '''
991 953 Get the shape of the one-element data for the given key
992 954 '''
993 955
994 if len(self.data[key]):
995 if 'spc' in key or not self.buffering:
996 return self.data[key].shape
997 return self.data[key][self.times[0]].shape
956 if len(self.data[self.min_time][key]):
957 return self.data[self.min_time][key].shape
998 958 return (0,)
999 959
1000 def update(self, dataOut, tm):
960 def update(self, data, tm, meta={}):
1001 961 '''
1002 962 Update data object with new dataOut
1003 963 '''
1004 964
1005 self.profileIndex = dataOut.profileIndex
1006 self.tm = tm
1007 self.type = dataOut.type
1008 self.parameters = getattr(dataOut, 'parameters', [])
1009
1010 if hasattr(dataOut, 'meta'):
1011 self.meta.update(dataOut.meta)
1012
1013 if hasattr(dataOut, 'pairsList'):
1014 self.pairs = dataOut.pairsList
1015
1016 self.interval = dataOut.timeInterval
1017 if True in ['spc' in ptype for ptype in self.plottypes]:
1018 self.xrange = (dataOut.getFreqRange(1)/1000.,
1019 dataOut.getAcfRange(1), dataOut.getVelRange(1))
1020 self.__heights.append(dataOut.heightList)
1021 self.__all_heights.update(dataOut.heightList)
1022
1023 for plot in self.plottypes:
1024 if plot in ('spc', 'spc_moments', 'spc_cut'):
1025 z = dataOut.data_spc/dataOut.normFactor
1026 buffer = 10*numpy.log10(z)
1027 if plot == 'cspc':
1028 buffer = (dataOut.data_spc, dataOut.data_cspc)
1029 if plot == 'noise':
1030 buffer = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
1031 if plot in ('rti', 'spcprofile'):
1032 buffer = dataOut.getPower()
1033 if plot == 'snr_db':
1034 buffer = dataOut.data_SNR
1035 if plot == 'snr':
1036 buffer = 10*numpy.log10(dataOut.data_SNR)
1037 if plot == 'dop':
1038 buffer = dataOut.data_DOP
1039 if plot == 'pow':
1040 buffer = 10*numpy.log10(dataOut.data_POW)
1041 if plot == 'width':
1042 buffer = dataOut.data_WIDTH
1043 if plot == 'coh':
1044 buffer = dataOut.getCoherence()
1045 if plot == 'phase':
1046 buffer = dataOut.getCoherence(phase=True)
1047 if plot == 'output':
1048 buffer = dataOut.data_output
1049 if plot == 'param':
1050 buffer = dataOut.data_param
1051 if plot == 'scope':
1052 buffer = dataOut.data
1053 self.flagDataAsBlock = dataOut.flagDataAsBlock
1054 self.nProfiles = dataOut.nProfiles
1055 if plot == 'pp_power':
1056 buffer = dataOut.dataPP_POWER
1057 self.flagDataAsBlock = dataOut.flagDataAsBlock
1058 self.nProfiles = dataOut.nProfiles
1059 if plot == 'pp_signal':
1060 buffer = dataOut.dataPP_POW
1061 self.flagDataAsBlock = dataOut.flagDataAsBlock
1062 self.nProfiles = dataOut.nProfiles
1063 if plot == 'pp_velocity':
1064 buffer = dataOut.dataPP_DOP
1065 self.flagDataAsBlock = dataOut.flagDataAsBlock
1066 self.nProfiles = dataOut.nProfiles
1067 if plot == 'pp_specwidth':
1068 buffer = dataOut.dataPP_WIDTH
1069 self.flagDataAsBlock = dataOut.flagDataAsBlock
1070 self.nProfiles = dataOut.nProfiles
1071
1072 if plot == 'spc':
1073 self.data['spc'][tm] = buffer
1074 elif plot == 'cspc':
1075 self.data['cspc'][tm] = buffer
1076 elif plot == 'spc_moments':
1077 self.data['spc'][tm] = buffer
1078 self.data['moments'][tm] = dataOut.moments
1079 else:
1080 if self.buffering:
1081 self.data[plot][tm] = buffer
1082 else:
1083 self.data[plot][tm] = buffer
1084
1085 if dataOut.channelList is None:
1086 self.channels = range(buffer.shape[0])
1087 else:
1088 self.channels = dataOut.channelList
1089
1090 if buffer is None:
1091 self.flagNoData = True
1092 raise schainpy.admin.SchainWarning('Attribute data_{} is empty'.format(self.key))
965 self.data[tm] = data
966
967 for key, value in meta.items():
968 setattr(self, key, value)
1093 969
1094 970 def normalize_heights(self):
1095 971 '''
1096 972 Ensure same-dimension of the data for different heighList
1097 973 '''
1098 974
1099 975 H = numpy.array(list(self.__all_heights))
1100 976 H.sort()
1101 977 for key in self.data:
1102 978 shape = self.shape(key)[:-1] + H.shape
1103 979 for tm, obj in list(self.data[key].items()):
1104 980 h = self.__heights[self.times.tolist().index(tm)]
1105 981 if H.size == h.size:
1106 982 continue
1107 983 index = numpy.where(numpy.in1d(H, h))[0]
1108 984 dummy = numpy.zeros(shape) + numpy.nan
1109 985 if len(shape) == 2:
1110 986 dummy[:, index] = obj
1111 987 else:
1112 988 dummy[index] = obj
1113 989 self.data[key][tm] = dummy
1114 990
1115 991 self.__heights = [H for tm in self.times]
1116 992
1117 993 def jsonify(self, tm, plot_name, plot_type, decimate=False):
1118 994 '''
1119 995 Convert data to json
1120 996 '''
1121 997
1122 dy = int(self.heights.size/self.MAXNUMY) + 1
1123 if self.key in ('spc', 'cspc'):
1124 dx = int(self.data[self.key][tm].shape[1]/self.MAXNUMX) + 1
998 meta = {}
999 meta['xrange'] = []
1000 dy = int(len(self.yrange)/self.MAXNUMY) + 1
1001 tmp = self.data[tm][self.key]
1002 shape = tmp.shape
1003 if len(shape) == 2:
1004 data = self.roundFloats(self.data[tm][self.key][::, ::dy].tolist())
1005 elif len(shape) == 3:
1006 dx = int(self.data[tm][self.key].shape[1]/self.MAXNUMX) + 1
1125 1007 data = self.roundFloats(
1126 self.data[self.key][tm][::, ::dx, ::dy].tolist())
1008 self.data[tm][self.key][::, ::dx, ::dy].tolist())
1009 meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1127 1010 else:
1128 if self.key is 'noise':
1129 data = [[x] for x in self.roundFloats(self.data[self.key][tm].tolist())]
1130 else:
1131 data = self.roundFloats(self.data[self.key][tm][::, ::dy].tolist())
1132
1133 meta = {}
1011 data = self.roundFloats(self.data[tm][self.key].tolist())
1012
1134 1013 ret = {
1135 1014 'plot': plot_name,
1136 1015 'code': self.exp_code,
1137 1016 'time': float(tm),
1138 1017 'data': data,
1139 1018 }
1140 1019 meta['type'] = plot_type
1141 1020 meta['interval'] = float(self.interval)
1142 1021 meta['localtime'] = self.localtime
1143 meta['yrange'] = self.roundFloats(self.heights[::dy].tolist())
1144 if 'spc' in self.data or 'cspc' in self.data:
1145 meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1146 else:
1147 meta['xrange'] = []
1148
1022 meta['yrange'] = self.roundFloats(self.yrange[::dy].tolist())
1149 1023 meta.update(self.meta)
1150 1024 ret['metadata'] = meta
1151 1025 return json.dumps(ret)
1152 1026
1153 1027 @property
1154 1028 def times(self):
1155 1029 '''
1156 1030 Return the list of times of the current data
1157 1031 '''
1158 1032
1159 ret = numpy.array([t for t in self.data[self.key]])
1160 if self:
1161 ret.sort()
1162 return ret
1033 ret = [t for t in self.data]
1034 ret.sort()
1035 return numpy.array(ret)
1163 1036
1164 1037 @property
1165 1038 def min_time(self):
1166 1039 '''
1167 1040 Return the minimun time value
1168 1041 '''
1169 1042
1170 1043 return self.times[0]
1171 1044
1172 1045 @property
1173 1046 def max_time(self):
1174 1047 '''
1175 1048 Return the maximun time value
1176 1049 '''
1177 1050
1178 1051 return self.times[-1]
1179 1052
1180 @property
1181 def heights(self):
1182 '''
1183 Return the list of heights of the current data
1184 '''
1053 # @property
1054 # def heights(self):
1055 # '''
1056 # Return the list of heights of the current data
1057 # '''
1185 1058
1186 return numpy.array(self.__heights[-1])
1059 # return numpy.array(self.__heights[-1])
1187 1060
1188 1061 @staticmethod
1189 1062 def roundFloats(obj):
1190 1063 if isinstance(obj, list):
1191 1064 return list(map(PlotterData.roundFloats, obj))
1192 1065 elif isinstance(obj, float):
1193 1066 return round(obj, 2)
Show file before | Show file after | Hide comments
@@ -1,906 +1,906
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
5 5 '''
6 6 import sys
7 7 import numpy
8 8 import copy
9 9 import datetime
10 10 import inspect
11 11 from schainpy.utils import log
12 12
13 13 SPEED_OF_LIGHT = 299792458
14 14 SPEED_OF_LIGHT = 3e8
15 15
16 16 BASIC_STRUCTURE = numpy.dtype([
17 17 ('nSize', '<u4'),
18 18 ('nVersion', '<u2'),
19 19 ('nDataBlockId', '<u4'),
20 20 ('nUtime', '<u4'),
21 21 ('nMilsec', '<u2'),
22 22 ('nTimezone', '<i2'),
23 23 ('nDstflag', '<i2'),
24 24 ('nErrorCount', '<u4')
25 25 ])
26 26
27 27 SYSTEM_STRUCTURE = numpy.dtype([
28 28 ('nSize', '<u4'),
29 29 ('nNumSamples', '<u4'),
30 30 ('nNumProfiles', '<u4'),
31 31 ('nNumChannels', '<u4'),
32 32 ('nADCResolution', '<u4'),
33 33 ('nPCDIOBusWidth', '<u4'),
34 34 ])
35 35
36 36 RADAR_STRUCTURE = numpy.dtype([
37 37 ('nSize', '<u4'),
38 38 ('nExpType', '<u4'),
39 39 ('nNTx', '<u4'),
40 40 ('fIpp', '<f4'),
41 41 ('fTxA', '<f4'),
42 42 ('fTxB', '<f4'),
43 43 ('nNumWindows', '<u4'),
44 44 ('nNumTaus', '<u4'),
45 45 ('nCodeType', '<u4'),
46 46 ('nLine6Function', '<u4'),
47 47 ('nLine5Function', '<u4'),
48 48 ('fClock', '<f4'),
49 49 ('nPrePulseBefore', '<u4'),
50 50 ('nPrePulseAfter', '<u4'),
51 51 ('sRangeIPP', '<a20'),
52 52 ('sRangeTxA', '<a20'),
53 53 ('sRangeTxB', '<a20'),
54 54 ])
55 55
56 56 SAMPLING_STRUCTURE = numpy.dtype(
57 57 [('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')])
58 58
59 59
60 60 PROCESSING_STRUCTURE = numpy.dtype([
61 61 ('nSize', '<u4'),
62 62 ('nDataType', '<u4'),
63 63 ('nSizeOfDataBlock', '<u4'),
64 64 ('nProfilesperBlock', '<u4'),
65 65 ('nDataBlocksperFile', '<u4'),
66 66 ('nNumWindows', '<u4'),
67 67 ('nProcessFlags', '<u4'),
68 68 ('nCoherentIntegrations', '<u4'),
69 69 ('nIncoherentIntegrations', '<u4'),
70 70 ('nTotalSpectra', '<u4')
71 71 ])
72 72
73 73
74 74 class Header(object):
75 75
76 76 def __init__(self):
77 77 raise NotImplementedError
78 78
79 79 def copy(self):
80 80 return copy.deepcopy(self)
81 81
82 82 def read(self):
83 83
84 84 raise NotImplementedError
85 85
86 86 def write(self):
87 87
88 88 raise NotImplementedError
89 89
90 90 def getAllowedArgs(self):
91 91 args = inspect.getargspec(self.__init__).args
92 92 try:
93 93 args.remove('self')
94 94 except:
95 95 pass
96 96 return args
97 97
98 98 def getAsDict(self):
99 99 args = self.getAllowedArgs()
100 100 asDict = {}
101 101 for x in args:
102 102 asDict[x] = self[x]
103 103 return asDict
104 104
105 105 def __getitem__(self, name):
106 106 return getattr(self, name)
107 107
108 108 def printInfo(self):
109 109
110 110 message = "#" * 50 + "\n"
111 111 message += self.__class__.__name__.upper() + "\n"
112 112 message += "#" * 50 + "\n"
113 113
114 114 keyList = list(self.__dict__.keys())
115 115 keyList.sort()
116 116
117 117 for key in keyList:
118 118 message += "%s = %s" % (key, self.__dict__[key]) + "\n"
119 119
120 120 if "size" not in keyList:
121 121 attr = getattr(self, "size")
122 122
123 123 if attr:
124 124 message += "%s = %s" % ("size", attr) + "\n"
125 125
126 126 print(message)
127 127
128 128
129 129 class BasicHeader(Header):
130 130
131 131 size = None
132 132 version = None
133 133 dataBlock = None
134 134 utc = None
135 135 ltc = None
136 136 miliSecond = None
137 137 timeZone = None
138 138 dstFlag = None
139 139 errorCount = None
140 140 datatime = None
141 141 structure = BASIC_STRUCTURE
142 142 __LOCALTIME = None
143 143
144 144 def __init__(self, useLocalTime=True):
145 145
146 146 self.size = 24
147 147 self.version = 0
148 148 self.dataBlock = 0
149 149 self.utc = 0
150 150 self.miliSecond = 0
151 151 self.timeZone = 0
152 152 self.dstFlag = 0
153 153 self.errorCount = 0
154 154
155 155 self.useLocalTime = useLocalTime
156 156
157 157 def read(self, fp):
158 158
159 159 self.length = 0
160 160 try:
161 161 if hasattr(fp, 'read'):
162 162 header = numpy.fromfile(fp, BASIC_STRUCTURE, 1)
163 163 else:
164 164 header = numpy.fromstring(fp, BASIC_STRUCTURE, 1)
165 165 except Exception as e:
166 166 print("BasicHeader: ")
167 167 print(e)
168 168 return 0
169 169
170 170 self.size = int(header['nSize'][0])
171 171 self.version = int(header['nVersion'][0])
172 172 self.dataBlock = int(header['nDataBlockId'][0])
173 173 self.utc = int(header['nUtime'][0])
174 174 self.miliSecond = int(header['nMilsec'][0])
175 175 self.timeZone = int(header['nTimezone'][0])
176 176 self.dstFlag = int(header['nDstflag'][0])
177 177 self.errorCount = int(header['nErrorCount'][0])
178 178
179 179 if self.size < 24:
180 180 return 0
181 181
182 182 self.length = header.nbytes
183 183 return 1
184 184
185 185 def write(self, fp):
186 186
187 187 headerTuple = (self.size, self.version, self.dataBlock, self.utc,
188 188 self.miliSecond, self.timeZone, self.dstFlag, self.errorCount)
189 189 header = numpy.array(headerTuple, BASIC_STRUCTURE)
190 190 header.tofile(fp)
191 191
192 192 return 1
193 193
194 194 def get_ltc(self):
195 195
196 196 return self.utc - self.timeZone * 60
197 197
198 198 def set_ltc(self, value):
199 199
200 200 self.utc = value + self.timeZone * 60
201 201
202 202 def get_datatime(self):
203 203
204 204 return datetime.datetime.utcfromtimestamp(self.ltc)
205 205
206 206 ltc = property(get_ltc, set_ltc)
207 207 datatime = property(get_datatime)
208 208
209 209
210 210 class SystemHeader(Header):
211 211
212 212 size = None
213 213 nSamples = None
214 214 nProfiles = None
215 215 nChannels = None
216 216 adcResolution = None
217 217 pciDioBusWidth = None
218 218 structure = SYSTEM_STRUCTURE
219 219
220 220 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWidth=0):
221 221
222 222 self.size = 24
223 223 self.nSamples = nSamples
224 224 self.nProfiles = nProfiles
225 225 self.nChannels = nChannels
226 226 self.adcResolution = adcResolution
227 227 self.pciDioBusWidth = pciDioBusWidth
228 228
229 229 def read(self, fp):
230 230 self.length = 0
231 231 try:
232 232 startFp = fp.tell()
233 233 except Exception as e:
234 234 startFp = None
235 235 pass
236 236
237 237 try:
238 238 if hasattr(fp, 'read'):
239 239 header = numpy.fromfile(fp, SYSTEM_STRUCTURE, 1)
240 240 else:
241 241 header = numpy.fromstring(fp, SYSTEM_STRUCTURE, 1)
242 242 except Exception as e:
243 243 print("System Header: " + str(e))
244 244 return 0
245 245
246 246 self.size = header['nSize'][0]
247 247 self.nSamples = header['nNumSamples'][0]
248 248 self.nProfiles = header['nNumProfiles'][0]
249 249 self.nChannels = header['nNumChannels'][0]
250 250 self.adcResolution = header['nADCResolution'][0]
251 251 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
252 252
253 253 if startFp is not None:
254 254 endFp = self.size + startFp
255 255
256 256 if fp.tell() > endFp:
257 257 sys.stderr.write(
258 258 "Warning %s: Size value read from System Header is lower than it has to be\n" % fp.name)
259 259 return 0
260 260
261 261 if fp.tell() < endFp:
262 262 sys.stderr.write(
263 263 "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp.name)
264 264 return 0
265 265
266 266 self.length = header.nbytes
267 267 return 1
268 268
269 269 def write(self, fp):
270 270
271 271 headerTuple = (self.size, self.nSamples, self.nProfiles,
272 272 self.nChannels, self.adcResolution, self.pciDioBusWidth)
273 273 header = numpy.array(headerTuple, SYSTEM_STRUCTURE)
274 274 header.tofile(fp)
275 275
276 276 return 1
277 277
278 278
279 279 class RadarControllerHeader(Header):
280 280
281 281 expType = None
282 282 nTx = None
283 283 ipp = None
284 284 txA = None
285 285 txB = None
286 286 nWindows = None
287 287 numTaus = None
288 288 codeType = None
289 289 line6Function = None
290 290 line5Function = None
291 291 fClock = None
292 292 prePulseBefore = None
293 293 prePulseAfter = None
294 294 rangeIpp = None
295 295 rangeTxA = None
296 296 rangeTxB = None
297 297 structure = RADAR_STRUCTURE
298 298 __size = None
299 299
300 300 def __init__(self, expType=2, nTx=1,
301 301 ipp=None, txA=0, txB=0,
302 302 nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
303 303 numTaus=0, line6Function=0, line5Function=0, fClock=None,
304 304 prePulseBefore=0, prePulseAfter=0,
305 codeType=0, nCode=0, nBaud=0, code=None,
305 codeType=0, nCode=0, nBaud=0, code=[],
306 306 flip1=0, flip2=0):
307 307
308 308 # self.size = 116
309 309 self.expType = expType
310 310 self.nTx = nTx
311 311 self.ipp = ipp
312 312 self.txA = txA
313 313 self.txB = txB
314 314 self.rangeIpp = ipp
315 315 self.rangeTxA = txA
316 316 self.rangeTxB = txB
317 317
318 318 self.nWindows = nWindows
319 319 self.numTaus = numTaus
320 320 self.codeType = codeType
321 321 self.line6Function = line6Function
322 322 self.line5Function = line5Function
323 323 self.fClock = fClock
324 324 self.prePulseBefore = prePulseBefore
325 325 self.prePulseAfter = prePulseAfter
326 326
327 327 self.nHeights = nHeights
328 328 self.firstHeight = firstHeight
329 329 self.deltaHeight = deltaHeight
330 330 self.samplesWin = nHeights
331 331
332 332 self.nCode = nCode
333 333 self.nBaud = nBaud
334 334 self.code = code
335 335 self.flip1 = flip1
336 336 self.flip2 = flip2
337 337
338 338 self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
339 339 # self.dynamic = numpy.array([],numpy.dtype('byte'))
340 340
341 341 if self.fClock is None and self.deltaHeight is not None:
342 342 self.fClock = 0.15 / (deltaHeight * 1e-6) # 0.15Km / (height * 1u)
343 343
344 344 def read(self, fp):
345 345 self.length = 0
346 346 try:
347 347 startFp = fp.tell()
348 348 except Exception as e:
349 349 startFp = None
350 350 pass
351 351
352 352 try:
353 353 if hasattr(fp, 'read'):
354 354 header = numpy.fromfile(fp, RADAR_STRUCTURE, 1)
355 355 else:
356 356 header = numpy.fromstring(fp, RADAR_STRUCTURE, 1)
357 357 self.length += header.nbytes
358 358 except Exception as e:
359 359 print("RadarControllerHeader: " + str(e))
360 360 return 0
361 361
362 362 size = int(header['nSize'][0])
363 363 self.expType = int(header['nExpType'][0])
364 364 self.nTx = int(header['nNTx'][0])
365 365 self.ipp = float(header['fIpp'][0])
366 366 self.txA = float(header['fTxA'][0])
367 367 self.txB = float(header['fTxB'][0])
368 368 self.nWindows = int(header['nNumWindows'][0])
369 369 self.numTaus = int(header['nNumTaus'][0])
370 370 self.codeType = int(header['nCodeType'][0])
371 371 self.line6Function = int(header['nLine6Function'][0])
372 372 self.line5Function = int(header['nLine5Function'][0])
373 373 self.fClock = float(header['fClock'][0])
374 374 self.prePulseBefore = int(header['nPrePulseBefore'][0])
375 375 self.prePulseAfter = int(header['nPrePulseAfter'][0])
376 376 self.rangeIpp = header['sRangeIPP'][0]
377 377 self.rangeTxA = header['sRangeTxA'][0]
378 378 self.rangeTxB = header['sRangeTxB'][0]
379 379
380 380 try:
381 381 if hasattr(fp, 'read'):
382 382 samplingWindow = numpy.fromfile(
383 383 fp, SAMPLING_STRUCTURE, self.nWindows)
384 384 else:
385 385 samplingWindow = numpy.fromstring(
386 386 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
387 387 self.length += samplingWindow.nbytes
388 388 except Exception as e:
389 389 print("RadarControllerHeader: " + str(e))
390 390 return 0
391 391 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
392 392 self.firstHeight = samplingWindow['h0']
393 393 self.deltaHeight = samplingWindow['dh']
394 394 self.samplesWin = samplingWindow['nsa']
395 395
396 396 try:
397 397 if hasattr(fp, 'read'):
398 398 self.Taus = numpy.fromfile(fp, '<f4', self.numTaus)
399 399 else:
400 400 self.Taus = numpy.fromstring(
401 401 fp[self.length:], '<f4', self.numTaus)
402 402 self.length += self.Taus.nbytes
403 403 except Exception as e:
404 404 print("RadarControllerHeader: " + str(e))
405 405 return 0
406 406
407 407 self.code_size = 0
408 408 if self.codeType != 0:
409 409
410 410 try:
411 411 if hasattr(fp, 'read'):
412 412 self.nCode = numpy.fromfile(fp, '<u4', 1)[0]
413 413 self.length += self.nCode.nbytes
414 414 self.nBaud = numpy.fromfile(fp, '<u4', 1)[0]
415 415 self.length += self.nBaud.nbytes
416 416 else:
417 417 self.nCode = numpy.fromstring(
418 418 fp[self.length:], '<u4', 1)[0]
419 419 self.length += self.nCode.nbytes
420 420 self.nBaud = numpy.fromstring(
421 421 fp[self.length:], '<u4', 1)[0]
422 422 self.length += self.nBaud.nbytes
423 423 except Exception as e:
424 424 print("RadarControllerHeader: " + str(e))
425 425 return 0
426 426 code = numpy.empty([self.nCode, self.nBaud], dtype='i1')
427 427
428 428 for ic in range(self.nCode):
429 429 try:
430 430 if hasattr(fp, 'read'):
431 431 temp = numpy.fromfile(fp, 'u4', int(
432 432 numpy.ceil(self.nBaud / 32.)))
433 433 else:
434 434 temp = numpy.fromstring(
435 435 fp, 'u4', int(numpy.ceil(self.nBaud / 32.)))
436 436 self.length += temp.nbytes
437 437 except Exception as e:
438 438 print("RadarControllerHeader: " + str(e))
439 439 return 0
440 440
441 441 for ib in range(self.nBaud - 1, -1, -1):
442 442 code[ic, ib] = temp[int(ib / 32)] % 2
443 443 temp[int(ib / 32)] = temp[int(ib / 32)] / 2
444 444
445 445 self.code = 2.0 * code - 1.0
446 446 self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
447 447
448 448 # if self.line5Function == RCfunction.FLIP:
449 449 # self.flip1 = numpy.fromfile(fp,'<u4',1)
450 450 #
451 451 # if self.line6Function == RCfunction.FLIP:
452 452 # self.flip2 = numpy.fromfile(fp,'<u4',1)
453 453 if startFp is not None:
454 454 endFp = size + startFp
455 455
456 456 if fp.tell() != endFp:
457 457 # fp.seek(endFp)
458 458 print("%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" % (fp.name, fp.tell() - startFp, size))
459 459 # return 0
460 460
461 461 if fp.tell() > endFp:
462 462 sys.stderr.write(
463 463 "Warning %s: Size value read from Radar Controller header is lower than it has to be\n" % fp.name)
464 464 # return 0
465 465
466 466 if fp.tell() < endFp:
467 467 sys.stderr.write(
468 468 "Warning %s: Size value read from Radar Controller header is greater than it has to be\n" % fp.name)
469 469
470 470 return 1
471 471
472 472 def write(self, fp):
473 473
474 474 headerTuple = (self.size,
475 475 self.expType,
476 476 self.nTx,
477 477 self.ipp,
478 478 self.txA,
479 479 self.txB,
480 480 self.nWindows,
481 481 self.numTaus,
482 482 self.codeType,
483 483 self.line6Function,
484 484 self.line5Function,
485 485 self.fClock,
486 486 self.prePulseBefore,
487 487 self.prePulseAfter,
488 488 self.rangeIpp,
489 489 self.rangeTxA,
490 490 self.rangeTxB)
491 491
492 492 header = numpy.array(headerTuple, RADAR_STRUCTURE)
493 493 header.tofile(fp)
494 494
495 495 sampleWindowTuple = (
496 496 self.firstHeight, self.deltaHeight, self.samplesWin)
497 497 samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
498 498 samplingWindow.tofile(fp)
499 499
500 500 if self.numTaus > 0:
501 501 self.Taus.tofile(fp)
502 502
503 503 if self.codeType != 0:
504 504 nCode = numpy.array(self.nCode, '<u4')
505 505 nCode.tofile(fp)
506 506 nBaud = numpy.array(self.nBaud, '<u4')
507 507 nBaud.tofile(fp)
508 508 code1 = (self.code + 1.0) / 2.
509 509
510 510 for ic in range(self.nCode):
511 511 tempx = numpy.zeros(int(numpy.ceil(self.nBaud / 32.)))
512 512 start = 0
513 513 end = 32
514 514 for i in range(len(tempx)):
515 515 code_selected = code1[ic, start:end]
516 516 for j in range(len(code_selected) - 1, -1, -1):
517 517 if code_selected[j] == 1:
518 518 tempx[i] = tempx[i] + \
519 519 2**(len(code_selected) - 1 - j)
520 520 start = start + 32
521 521 end = end + 32
522 522
523 523 tempx = tempx.astype('u4')
524 524 tempx.tofile(fp)
525 525
526 526 # if self.line5Function == RCfunction.FLIP:
527 527 # self.flip1.tofile(fp)
528 528 #
529 529 # if self.line6Function == RCfunction.FLIP:
530 530 # self.flip2.tofile(fp)
531 531
532 532 return 1
533 533
534 534 def get_ippSeconds(self):
535 535 '''
536 536 '''
537 537 ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT
538 538
539 539 return ippSeconds
540 540
541 541 def set_ippSeconds(self, ippSeconds):
542 542 '''
543 543 '''
544 544
545 545 self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0 * 1000)
546 546
547 547 return
548 548
549 549 def get_size(self):
550 550
551 551 self.__size = 116 + 12 * self.nWindows + 4 * self.numTaus
552 552
553 553 if self.codeType != 0:
554 554 self.__size += 4 + 4 + 4 * self.nCode * \
555 555 numpy.ceil(self.nBaud / 32.)
556 556
557 557 return self.__size
558 558
559 559 def set_size(self, value):
560 560
561 561 raise IOError("size is a property and it cannot be set, just read")
562 562
563 563 return
564 564
565 565 ippSeconds = property(get_ippSeconds, set_ippSeconds)
566 566 size = property(get_size, set_size)
567 567
568 568
569 569 class ProcessingHeader(Header):
570 570
571 571 # size = None
572 572 dtype = None
573 573 blockSize = None
574 574 profilesPerBlock = None
575 575 dataBlocksPerFile = None
576 576 nWindows = None
577 577 processFlags = None
578 578 nCohInt = None
579 579 nIncohInt = None
580 580 totalSpectra = None
581 581 structure = PROCESSING_STRUCTURE
582 582 flag_dc = None
583 583 flag_cspc = None
584 584
585 585 def __init__(self, dtype=0, blockSize=0, profilesPerBlock=0, dataBlocksPerFile=0, nWindows=0, processFlags=0, nCohInt=0,
586 586 nIncohInt=0, totalSpectra=0, nHeights=0, firstHeight=0, deltaHeight=0, samplesWin=0, spectraComb=0, nCode=0,
587 587 code=0, nBaud=None, shif_fft=False, flag_dc=False, flag_cspc=False, flag_decode=False, flag_deflip=False
588 588 ):
589 589
590 590 # self.size = 0
591 591 self.dtype = dtype
592 592 self.blockSize = blockSize
593 593 self.profilesPerBlock = 0
594 594 self.dataBlocksPerFile = 0
595 595 self.nWindows = 0
596 596 self.processFlags = 0
597 597 self.nCohInt = 0
598 598 self.nIncohInt = 0
599 599 self.totalSpectra = 0
600 600
601 601 self.nHeights = 0
602 602 self.firstHeight = 0
603 603 self.deltaHeight = 0
604 604 self.samplesWin = 0
605 605 self.spectraComb = 0
606 606 self.nCode = None
607 607 self.code = None
608 608 self.nBaud = None
609 609
610 610 self.shif_fft = False
611 611 self.flag_dc = False
612 612 self.flag_cspc = False
613 613 self.flag_decode = False
614 614 self.flag_deflip = False
615 615 self.length = 0
616 616
617 617 def read(self, fp):
618 618 self.length = 0
619 619 try:
620 620 startFp = fp.tell()
621 621 except Exception as e:
622 622 startFp = None
623 623 pass
624 624
625 625 try:
626 626 if hasattr(fp, 'read'):
627 627 header = numpy.fromfile(fp, PROCESSING_STRUCTURE, 1)
628 628 else:
629 629 header = numpy.fromstring(fp, PROCESSING_STRUCTURE, 1)
630 630 self.length += header.nbytes
631 631 except Exception as e:
632 632 print("ProcessingHeader: " + str(e))
633 633 return 0
634 634
635 635 size = int(header['nSize'][0])
636 636 self.dtype = int(header['nDataType'][0])
637 637 self.blockSize = int(header['nSizeOfDataBlock'][0])
638 638 self.profilesPerBlock = int(header['nProfilesperBlock'][0])
639 639 self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
640 640 self.nWindows = int(header['nNumWindows'][0])
641 641 self.processFlags = header['nProcessFlags']
642 642 self.nCohInt = int(header['nCoherentIntegrations'][0])
643 643 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
644 644 self.totalSpectra = int(header['nTotalSpectra'][0])
645 645
646 646 try:
647 647 if hasattr(fp, 'read'):
648 648 samplingWindow = numpy.fromfile(
649 649 fp, SAMPLING_STRUCTURE, self.nWindows)
650 650 else:
651 651 samplingWindow = numpy.fromstring(
652 652 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
653 653 self.length += samplingWindow.nbytes
654 654 except Exception as e:
655 655 print("ProcessingHeader: " + str(e))
656 656 return 0
657 657
658 658 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
659 659 self.firstHeight = float(samplingWindow['h0'][0])
660 660 self.deltaHeight = float(samplingWindow['dh'][0])
661 661 self.samplesWin = samplingWindow['nsa'][0]
662 662
663 663 try:
664 664 if hasattr(fp, 'read'):
665 665 self.spectraComb = numpy.fromfile(
666 666 fp, 'u1', 2 * self.totalSpectra)
667 667 else:
668 668 self.spectraComb = numpy.fromstring(
669 669 fp[self.length:], 'u1', 2 * self.totalSpectra)
670 670 self.length += self.spectraComb.nbytes
671 671 except Exception as e:
672 672 print("ProcessingHeader: " + str(e))
673 673 return 0
674 674
675 675 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
676 676 self.nCode = int(numpy.fromfile(fp, '<u4', 1))
677 677 self.nBaud = int(numpy.fromfile(fp, '<u4', 1))
678 678 self.code = numpy.fromfile(
679 679 fp, '<f4', self.nCode * self.nBaud).reshape(self.nCode, self.nBaud)
680 680
681 681 if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP):
682 682 exp_name_len = int(numpy.fromfile(fp, '<u4', 1))
683 683 exp_name = numpy.fromfile(fp, 'u1', exp_name_len + 1)
684 684
685 685 if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
686 686 self.shif_fft = True
687 687 else:
688 688 self.shif_fft = False
689 689
690 690 if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
691 691 self.flag_dc = True
692 692 else:
693 693 self.flag_dc = False
694 694
695 695 if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA):
696 696 self.flag_decode = True
697 697 else:
698 698 self.flag_decode = False
699 699
700 700 if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA):
701 701 self.flag_deflip = True
702 702 else:
703 703 self.flag_deflip = False
704 704
705 705 nChannels = 0
706 706 nPairs = 0
707 707 pairList = []
708 708
709 709 for i in range(0, self.totalSpectra * 2, 2):
710 710 if self.spectraComb[i] == self.spectraComb[i + 1]:
711 711 nChannels = nChannels + 1 # par de canales iguales
712 712 else:
713 713 nPairs = nPairs + 1 # par de canales diferentes
714 714 pairList.append((self.spectraComb[i], self.spectraComb[i + 1]))
715 715
716 716 self.flag_cspc = False
717 717 if nPairs > 0:
718 718 self.flag_cspc = True
719 719
720 720 if startFp is not None:
721 721 endFp = size + startFp
722 722 if fp.tell() > endFp:
723 723 sys.stderr.write(
724 724 "Warning: Processing header size is lower than it has to be")
725 725 return 0
726 726
727 727 if fp.tell() < endFp:
728 728 sys.stderr.write(
729 729 "Warning: Processing header size is greater than it is considered")
730 730
731 731 return 1
732 732
733 733 def write(self, fp):
734 734 # Clear DEFINE_PROCESS_CODE
735 735 self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE)
736 736
737 737 headerTuple = (self.size,
738 738 self.dtype,
739 739 self.blockSize,
740 740 self.profilesPerBlock,
741 741 self.dataBlocksPerFile,
742 742 self.nWindows,
743 743 self.processFlags,
744 744 self.nCohInt,
745 745 self.nIncohInt,
746 746 self.totalSpectra)
747 747
748 748 header = numpy.array(headerTuple, PROCESSING_STRUCTURE)
749 749 header.tofile(fp)
750 750
751 751 if self.nWindows != 0:
752 752 sampleWindowTuple = (
753 753 self.firstHeight, self.deltaHeight, self.samplesWin)
754 754 samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
755 755 samplingWindow.tofile(fp)
756 756
757 757 if self.totalSpectra != 0:
758 758 # spectraComb = numpy.array([],numpy.dtype('u1'))
759 759 spectraComb = self.spectraComb
760 760 spectraComb.tofile(fp)
761 761
762 762 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
763 763 # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
764 764 # nCode.tofile(fp)
765 765 #
766 766 # nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
767 767 # nBaud.tofile(fp)
768 768 #
769 769 # code = self.code.reshape(self.nCode*self.nBaud)
770 770 # code = code.astype(numpy.dtype('<f4'))
771 771 # code.tofile(fp)
772 772
773 773 return 1
774 774
775 775 def get_size(self):
776 776
777 777 self.__size = 40 + 12 * self.nWindows + 2 * self.totalSpectra
778 778
779 779 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
780 780 # self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
781 781 # self.__size += 4 + 4 + 4 * self.nCode * self.nBaud
782 782
783 783 return self.__size
784 784
785 785 def set_size(self, value):
786 786
787 787 raise IOError("size is a property and it cannot be set, just read")
788 788
789 789 return
790 790
791 791 size = property(get_size, set_size)
792 792
793 793
794 794 class RCfunction:
795 795 NONE = 0
796 796 FLIP = 1
797 797 CODE = 2
798 798 SAMPLING = 3
799 799 LIN6DIV256 = 4
800 800 SYNCHRO = 5
801 801
802 802
803 803 class nCodeType:
804 804 NONE = 0
805 805 USERDEFINE = 1
806 806 BARKER2 = 2
807 807 BARKER3 = 3
808 808 BARKER4 = 4
809 809 BARKER5 = 5
810 810 BARKER7 = 6
811 811 BARKER11 = 7
812 812 BARKER13 = 8
813 813 AC128 = 9
814 814 COMPLEMENTARYCODE2 = 10
815 815 COMPLEMENTARYCODE4 = 11
816 816 COMPLEMENTARYCODE8 = 12
817 817 COMPLEMENTARYCODE16 = 13
818 818 COMPLEMENTARYCODE32 = 14
819 819 COMPLEMENTARYCODE64 = 15
820 820 COMPLEMENTARYCODE128 = 16
821 821 CODE_BINARY28 = 17
822 822
823 823
824 824 class PROCFLAG:
825 825
826 826 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
827 827 DECODE_DATA = numpy.uint32(0x00000002)
828 828 SPECTRA_CALC = numpy.uint32(0x00000004)
829 829 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
830 830 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
831 831 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
832 832
833 833 DATATYPE_CHAR = numpy.uint32(0x00000040)
834 834 DATATYPE_SHORT = numpy.uint32(0x00000080)
835 835 DATATYPE_LONG = numpy.uint32(0x00000100)
836 836 DATATYPE_INT64 = numpy.uint32(0x00000200)
837 837 DATATYPE_FLOAT = numpy.uint32(0x00000400)
838 838 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
839 839
840 840 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
841 841 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
842 842 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
843 843
844 844 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
845 845 DEFLIP_DATA = numpy.uint32(0x00010000)
846 846 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
847 847
848 848 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
849 849 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
850 850 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
851 851 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
852 852 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
853 853
854 854 EXP_NAME_ESP = numpy.uint32(0x00200000)
855 855 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
856 856
857 857 OPERATION_MASK = numpy.uint32(0x0000003F)
858 858 DATATYPE_MASK = numpy.uint32(0x00000FC0)
859 859 DATAARRANGE_MASK = numpy.uint32(0x00007000)
860 860 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
861 861
862 862
863 863 dtype0 = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
864 864 dtype1 = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
865 865 dtype2 = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
866 866 dtype3 = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
867 867 dtype4 = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
868 868 dtype5 = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
869 869
870 870 NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
871 871
872 872 PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR,
873 873 PROCFLAG.DATATYPE_SHORT,
874 874 PROCFLAG.DATATYPE_LONG,
875 875 PROCFLAG.DATATYPE_INT64,
876 876 PROCFLAG.DATATYPE_FLOAT,
877 877 PROCFLAG.DATATYPE_DOUBLE]
878 878
879 879 DTYPE_WIDTH = [1, 2, 4, 8, 4, 8]
880 880
881 881
882 882 def get_dtype_index(numpy_dtype):
883 883
884 884 index = None
885 885
886 886 for i in range(len(NUMPY_DTYPE_LIST)):
887 887 if numpy_dtype == NUMPY_DTYPE_LIST[i]:
888 888 index = i
889 889 break
890 890
891 891 return index
892 892
893 893
894 894 def get_numpy_dtype(index):
895 895
896 896 return NUMPY_DTYPE_LIST[index]
897 897
898 898
899 899 def get_procflag_dtype(index):
900 900
901 901 return PROCFLAG_DTYPE_LIST[index]
902 902
903 903
904 904 def get_dtype_width(index):
905 905
906 906 return DTYPE_WIDTH[index] No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,665 +1,688
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Base class to create plot operations
6 6
7 7 """
8 8
9 9 import os
10 10 import sys
11 11 import zmq
12 12 import time
13 13 import numpy
14 14 import datetime
15 from multiprocessing import Queue
15 from collections import deque
16 16 from functools import wraps
17 17 from threading import Thread
18 18 import matplotlib
19 19
20 20 if 'BACKEND' in os.environ:
21 21 matplotlib.use(os.environ['BACKEND'])
22 22 elif 'linux' in sys.platform:
23 23 matplotlib.use("TkAgg")
24 24 elif 'darwin' in sys.platform:
25 matplotlib.use('WxAgg')
25 matplotlib.use('MacOSX')
26 26 else:
27 27 from schainpy.utils import log
28 28 log.warning('Using default Backend="Agg"', 'INFO')
29 29 matplotlib.use('Agg')
30 30
31 31 import matplotlib.pyplot as plt
32 32 from matplotlib.patches import Polygon
33 33 from mpl_toolkits.axes_grid1 import make_axes_locatable
34 34 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
35 35
36 36 from schainpy.model.data.jrodata import PlotterData
37 37 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
38 38 from schainpy.utils import log
39 39
40 40 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
41 41 blu_values = matplotlib.pyplot.get_cmap(
42 42 'seismic_r', 20)(numpy.arange(20))[10:15]
43 43 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
44 44 'jro', numpy.vstack((blu_values, jet_values)))
45 45 matplotlib.pyplot.register_cmap(cmap=ncmap)
46 46
47 47 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis',
48 48 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
49 49
50 50 EARTH_RADIUS = 6.3710e3
51 51
52 52 def ll2xy(lat1, lon1, lat2, lon2):
53 53
54 54 p = 0.017453292519943295
55 55 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
56 56 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
57 57 r = 12742 * numpy.arcsin(numpy.sqrt(a))
58 58 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
59 59 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
60 60 theta = -theta + numpy.pi/2
61 61 return r*numpy.cos(theta), r*numpy.sin(theta)
62 62
63 63
64 64 def km2deg(km):
65 65 '''
66 66 Convert distance in km to degrees
67 67 '''
68 68
69 69 return numpy.rad2deg(km/EARTH_RADIUS)
70 70
71 71
72 72 def figpause(interval):
73 73 backend = plt.rcParams['backend']
74 74 if backend in matplotlib.rcsetup.interactive_bk:
75 75 figManager = matplotlib._pylab_helpers.Gcf.get_active()
76 76 if figManager is not None:
77 77 canvas = figManager.canvas
78 78 if canvas.figure.stale:
79 79 canvas.draw()
80 80 try:
81 81 canvas.start_event_loop(interval)
82 82 except:
83 83 pass
84 84 return
85 85
86
87 86 def popup(message):
88 87 '''
89 88 '''
90 89
91 90 fig = plt.figure(figsize=(12, 8), facecolor='r')
92 91 text = '\n'.join([s.strip() for s in message.split(':')])
93 92 fig.text(0.01, 0.5, text, ha='left', va='center',
94 93 size='20', weight='heavy', color='w')
95 94 fig.show()
96 95 figpause(1000)
97 96
98 97
99 98 class Throttle(object):
100 99 '''
101 100 Decorator that prevents a function from being called more than once every
102 101 time period.
103 102 To create a function that cannot be called more than once a minute, but
104 103 will sleep until it can be called:
105 104 @Throttle(minutes=1)
106 105 def foo():
107 106 pass
108 107
109 108 for i in range(10):
110 109 foo()
111 110 print "This function has run %s times." % i
112 111 '''
113 112
114 113 def __init__(self, seconds=0, minutes=0, hours=0):
115 114 self.throttle_period = datetime.timedelta(
116 115 seconds=seconds, minutes=minutes, hours=hours
117 116 )
118 117
119 118 self.time_of_last_call = datetime.datetime.min
120 119
121 120 def __call__(self, fn):
122 121 @wraps(fn)
123 122 def wrapper(*args, **kwargs):
124 123 coerce = kwargs.pop('coerce', None)
125 124 if coerce:
126 125 self.time_of_last_call = datetime.datetime.now()
127 126 return fn(*args, **kwargs)
128 127 else:
129 128 now = datetime.datetime.now()
130 129 time_since_last_call = now - self.time_of_last_call
131 130 time_left = self.throttle_period - time_since_last_call
132 131
133 132 if time_left > datetime.timedelta(seconds=0):
134 133 return
135 134
136 135 self.time_of_last_call = datetime.datetime.now()
137 136 return fn(*args, **kwargs)
138 137
139 138 return wrapper
140 139
141 140 def apply_throttle(value):
142 141
143 142 @Throttle(seconds=value)
144 143 def fnThrottled(fn):
145 144 fn()
146 145
147 146 return fnThrottled
148 147
149 148
150 149 @MPDecorator
151 150 class Plot(Operation):
152 151 """Base class for Schain plotting operations
153 152
154 153 This class should never be use directtly you must subclass a new operation,
155 154 children classes must be defined as follow:
156 155
157 156 ExamplePlot(Plot):
158 157
159 158 CODE = 'code'
160 159 colormap = 'jet'
161 160 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
162 161
163 162 def setup(self):
164 163 pass
165 164
166 165 def plot(self):
167 166 pass
168 167
169 168 """
170 169
171 170 CODE = 'Figure'
172 171 colormap = 'jet'
173 172 bgcolor = 'white'
174 173 buffering = True
175 174 __missing = 1E30
176 175
177 176 __attrs__ = ['show', 'save', 'ymin', 'ymax', 'zmin', 'zmax', 'title',
178 177 'showprofile']
179 178
180 179 def __init__(self):
181 180
182 181 Operation.__init__(self)
183 182 self.isConfig = False
184 183 self.isPlotConfig = False
185 184 self.save_time = 0
186 185 self.sender_time = 0
187 186 self.data = None
188 187 self.firsttime = True
189 self.sender_queue = Queue(maxsize=60)
188 self.sender_queue = deque(maxlen=10)
190 189 self.plots_adjust = {'left': 0.125, 'right': 0.9, 'bottom': 0.15, 'top': 0.9, 'wspace': 0.2, 'hspace': 0.2}
191 190
192 191 def __fmtTime(self, x, pos):
193 192 '''
194 193 '''
195 194
196 195 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
197 196
198 197 def __setup(self, **kwargs):
199 198 '''
200 199 Initialize variables
201 200 '''
202 201
203 202 self.figures = []
204 203 self.axes = []
205 204 self.cb_axes = []
206 205 self.localtime = kwargs.pop('localtime', True)
207 206 self.show = kwargs.get('show', True)
208 207 self.save = kwargs.get('save', False)
209 208 self.save_period = kwargs.get('save_period', 0)
210 209 self.colormap = kwargs.get('colormap', self.colormap)
211 210 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
212 211 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
213 212 self.colormaps = kwargs.get('colormaps', None)
214 213 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
215 214 self.showprofile = kwargs.get('showprofile', False)
216 215 self.title = kwargs.get('wintitle', self.CODE.upper())
217 216 self.cb_label = kwargs.get('cb_label', None)
218 217 self.cb_labels = kwargs.get('cb_labels', None)
219 218 self.labels = kwargs.get('labels', None)
220 219 self.xaxis = kwargs.get('xaxis', 'frequency')
221 220 self.zmin = kwargs.get('zmin', None)
222 221 self.zmax = kwargs.get('zmax', None)
223 222 self.zlimits = kwargs.get('zlimits', None)
224 223 self.xmin = kwargs.get('xmin', None)
225 224 self.xmax = kwargs.get('xmax', None)
226 225 self.xrange = kwargs.get('xrange', 12)
227 226 self.xscale = kwargs.get('xscale', None)
228 227 self.ymin = kwargs.get('ymin', None)
229 228 self.ymax = kwargs.get('ymax', None)
230 229 self.yscale = kwargs.get('yscale', None)
231 230 self.xlabel = kwargs.get('xlabel', None)
232 231 self.attr_time = kwargs.get('attr_time', 'utctime')
232 self.attr_data = kwargs.get('attr_data', 'data_param')
233 233 self.decimation = kwargs.get('decimation', None)
234 234 self.showSNR = kwargs.get('showSNR', False)
235 235 self.oneFigure = kwargs.get('oneFigure', True)
236 236 self.width = kwargs.get('width', None)
237 237 self.height = kwargs.get('height', None)
238 238 self.colorbar = kwargs.get('colorbar', True)
239 239 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
240 240 self.channels = kwargs.get('channels', None)
241 241 self.titles = kwargs.get('titles', [])
242 242 self.polar = False
243 243 self.type = kwargs.get('type', 'iq')
244 244 self.grid = kwargs.get('grid', False)
245 245 self.pause = kwargs.get('pause', False)
246 246 self.save_code = kwargs.get('save_code', self.CODE)
247 247 self.throttle = kwargs.get('throttle', 0)
248 248 self.exp_code = kwargs.get('exp_code', None)
249 249 self.server = kwargs.get('server', False)
250 250 self.sender_period = kwargs.get('sender_period', 60)
251 251 self.tag = kwargs.get('tag', '')
252 252 self.height_index = kwargs.get('height_index', None)
253 253 self.__throttle_plot = apply_throttle(self.throttle)
254 self.data = PlotterData(
255 self.CODE, self.throttle, self.exp_code, self.localtime, self.buffering, snr=self.showSNR)
254 code = self.attr_data if self.attr_data else self.CODE
255 self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
256 256
257 257 if self.server:
258 258 if not self.server.startswith('tcp://'):
259 259 self.server = 'tcp://{}'.format(self.server)
260 260 log.success(
261 261 'Sending to server: {}'.format(self.server),
262 262 self.name
263 263 )
264 264
265 265 def __setup_plot(self):
266 266 '''
267 267 Common setup for all figures, here figures and axes are created
268 268 '''
269 269
270 270 self.setup()
271 271
272 272 self.time_label = 'LT' if self.localtime else 'UTC'
273 273
274 274 if self.width is None:
275 275 self.width = 8
276 276
277 277 self.figures = []
278 278 self.axes = []
279 279 self.cb_axes = []
280 280 self.pf_axes = []
281 281 self.cmaps = []
282 282
283 283 size = '15%' if self.ncols == 1 else '30%'
284 284 pad = '4%' if self.ncols == 1 else '8%'
285 285
286 286 if self.oneFigure:
287 287 if self.height is None:
288 288 self.height = 1.4 * self.nrows + 1
289 289 fig = plt.figure(figsize=(self.width, self.height),
290 290 edgecolor='k',
291 291 facecolor='w')
292 292 self.figures.append(fig)
293 293 for n in range(self.nplots):
294 294 ax = fig.add_subplot(self.nrows, self.ncols,
295 295 n + 1, polar=self.polar)
296 296 ax.tick_params(labelsize=8)
297 297 ax.firsttime = True
298 298 ax.index = 0
299 299 ax.press = None
300 300 self.axes.append(ax)
301 301 if self.showprofile:
302 302 cax = self.__add_axes(ax, size=size, pad=pad)
303 303 cax.tick_params(labelsize=8)
304 304 self.pf_axes.append(cax)
305 305 else:
306 306 if self.height is None:
307 307 self.height = 3
308 308 for n in range(self.nplots):
309 309 fig = plt.figure(figsize=(self.width, self.height),
310 310 edgecolor='k',
311 311 facecolor='w')
312 312 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
313 313 ax.tick_params(labelsize=8)
314 314 ax.firsttime = True
315 315 ax.index = 0
316 316 ax.press = None
317 317 self.figures.append(fig)
318 318 self.axes.append(ax)
319 319 if self.showprofile:
320 320 cax = self.__add_axes(ax, size=size, pad=pad)
321 321 cax.tick_params(labelsize=8)
322 322 self.pf_axes.append(cax)
323 323
324 324 for n in range(self.nrows):
325 325 if self.colormaps is not None:
326 326 cmap = plt.get_cmap(self.colormaps[n])
327 327 else:
328 328 cmap = plt.get_cmap(self.colormap)
329 329 cmap.set_bad(self.bgcolor, 1.)
330 330 self.cmaps.append(cmap)
331 331
332 332 def __add_axes(self, ax, size='30%', pad='8%'):
333 333 '''
334 334 Add new axes to the given figure
335 335 '''
336 336 divider = make_axes_locatable(ax)
337 337 nax = divider.new_horizontal(size=size, pad=pad)
338 338 ax.figure.add_axes(nax)
339 339 return nax
340 340
341 341 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
342 342 '''
343 343 Create a masked array for missing data
344 344 '''
345 345 if x_buffer.shape[0] < 2:
346 346 return x_buffer, y_buffer, z_buffer
347 347
348 348 deltas = x_buffer[1:] - x_buffer[0:-1]
349 349 x_median = numpy.median(deltas)
350 350
351 351 index = numpy.where(deltas > 5 * x_median)
352 352
353 353 if len(index[0]) != 0:
354 354 z_buffer[::, index[0], ::] = self.__missing
355 355 z_buffer = numpy.ma.masked_inside(z_buffer,
356 356 0.99 * self.__missing,
357 357 1.01 * self.__missing)
358 358
359 359 return x_buffer, y_buffer, z_buffer
360 360
361 361 def decimate(self):
362 362
363 363 # dx = int(len(self.x)/self.__MAXNUMX) + 1
364 364 dy = int(len(self.y) / self.decimation) + 1
365 365
366 366 # x = self.x[::dx]
367 367 x = self.x
368 368 y = self.y[::dy]
369 369 z = self.z[::, ::, ::dy]
370 370
371 371 return x, y, z
372 372
373 373 def format(self):
374 374 '''
375 375 Set min and max values, labels, ticks and titles
376 376 '''
377 377
378 378 for n, ax in enumerate(self.axes):
379 379 if ax.firsttime:
380 380 if self.xaxis != 'time':
381 381 xmin = self.xmin
382 382 xmax = self.xmax
383 383 else:
384 384 xmin = self.tmin
385 385 xmax = self.tmin + self.xrange*60*60
386 386 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
387 387 ax.xaxis.set_major_locator(LinearLocator(9))
388 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
389 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
388 ymin = self.ymin if self.ymin is not None else numpy.nanmin(self.y[numpy.isfinite(self.y)])
389 ymax = self.ymax if self.ymax is not None else numpy.nanmax(self.y[numpy.isfinite(self.y)])
390 390 ax.set_facecolor(self.bgcolor)
391 391 if self.xscale:
392 392 ax.xaxis.set_major_formatter(FuncFormatter(
393 393 lambda x, pos: '{0:g}'.format(x*self.xscale)))
394 394 if self.yscale:
395 395 ax.yaxis.set_major_formatter(FuncFormatter(
396 396 lambda x, pos: '{0:g}'.format(x*self.yscale)))
397 397 if self.xlabel is not None:
398 398 ax.set_xlabel(self.xlabel)
399 399 if self.ylabel is not None:
400 400 ax.set_ylabel(self.ylabel)
401 401 if self.showprofile:
402 402 self.pf_axes[n].set_ylim(ymin, ymax)
403 403 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
404 404 self.pf_axes[n].set_xlabel('dB')
405 405 self.pf_axes[n].grid(b=True, axis='x')
406 406 [tick.set_visible(False)
407 407 for tick in self.pf_axes[n].get_yticklabels()]
408 408 if self.colorbar:
409 409 ax.cbar = plt.colorbar(
410 410 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
411 411 ax.cbar.ax.tick_params(labelsize=8)
412 412 ax.cbar.ax.press = None
413 413 if self.cb_label:
414 414 ax.cbar.set_label(self.cb_label, size=8)
415 415 elif self.cb_labels:
416 416 ax.cbar.set_label(self.cb_labels[n], size=8)
417 417 else:
418 418 ax.cbar = None
419 419 ax.set_xlim(xmin, xmax)
420 420 ax.set_ylim(ymin, ymax)
421 421 ax.firsttime = False
422 422 if self.grid:
423 423 ax.grid(True)
424 424 if not self.polar:
425 425 ax.set_title('{} {} {}'.format(
426 426 self.titles[n],
427 427 self.getDateTime(self.data.max_time).strftime(
428 428 '%Y-%m-%d %H:%M:%S'),
429 429 self.time_label),
430 430 size=8)
431 431 else:
432 432 ax.set_title('{}'.format(self.titles[n]), size=8)
433 433 ax.set_ylim(0, 90)
434 434 ax.set_yticks(numpy.arange(0, 90, 20))
435 435 ax.yaxis.labelpad = 40
436 436
437 437 if self.firsttime:
438 438 for n, fig in enumerate(self.figures):
439 439 fig.subplots_adjust(**self.plots_adjust)
440 440 self.firsttime = False
441 441
442 442 def clear_figures(self):
443 443 '''
444 444 Reset axes for redraw plots
445 445 '''
446 446
447 447 for ax in self.axes+self.pf_axes+self.cb_axes:
448 448 ax.clear()
449 449 ax.firsttime = True
450 450 if hasattr(ax, 'cbar') and ax.cbar:
451 451 ax.cbar.remove()
452 452
453 453 def __plot(self):
454 454 '''
455 455 Main function to plot, format and save figures
456 456 '''
457 457
458 458 self.plot()
459 459 self.format()
460 460
461 461 for n, fig in enumerate(self.figures):
462 462 if self.nrows == 0 or self.nplots == 0:
463 463 log.warning('No data', self.name)
464 464 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
465 465 fig.canvas.manager.set_window_title(self.CODE)
466 466 continue
467 467
468 468 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
469 469 self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
470 470 fig.canvas.draw()
471 471 if self.show:
472 472 fig.show()
473 473 figpause(0.01)
474 474
475 475 if self.save:
476 476 self.save_figure(n)
477 477
478 478 if self.server:
479 479 self.send_to_server()
480 480
481 def __update(self, dataOut, timestamp):
482 '''
483 '''
484
485 metadata = {
486 'yrange': dataOut.heightList,
487 'interval': dataOut.timeInterval,
488 'channels': dataOut.channelList
489 }
490
491 data, meta = self.update(dataOut)
492 metadata.update(meta)
493 self.data.update(data, timestamp, metadata)
494
481 495 def save_figure(self, n):
482 496 '''
483 497 '''
484 498
485 if (self.data.tm - self.save_time) <= self.save_period:
499 if (self.data.max_time - self.save_time) <= self.save_period:
486 500 return
487 501
488 self.save_time = self.data.tm
502 self.save_time = self.data.max_time
489 503
490 504 fig = self.figures[n]
491 505
492 506 figname = os.path.join(
493 507 self.save,
494 508 self.save_code,
495 509 '{}_{}.png'.format(
496 510 self.save_code,
497 511 self.getDateTime(self.data.max_time).strftime(
498 512 '%Y%m%d_%H%M%S'
499 513 ),
500 514 )
501 515 )
502 516 log.log('Saving figure: {}'.format(figname), self.name)
503 517 if not os.path.isdir(os.path.dirname(figname)):
504 518 os.makedirs(os.path.dirname(figname))
505 519 fig.savefig(figname)
506 520
507 521 if self.throttle == 0:
508 522 figname = os.path.join(
509 523 self.save,
510 524 '{}_{}.png'.format(
511 525 self.save_code,
512 526 self.getDateTime(self.data.min_time).strftime(
513 527 '%Y%m%d'
514 528 ),
515 529 )
516 530 )
517 531 fig.savefig(figname)
518 532
519 533 def send_to_server(self):
520 534 '''
521 535 '''
522 536
523 interval = self.data.tm - self.sender_time
537 if self.exp_code == None:
538 log.warning('Missing `exp_code` skipping sending to server...')
539
540 last_time = self.data.max_time
541 interval = last_time - self.sender_time
524 542 if interval < self.sender_period:
525 543 return
526 544
527 self.sender_time = self.data.tm
545 self.sender_time = last_time
528 546
529 547 attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
530 548 for attr in attrs:
531 549 value = getattr(self, attr)
532 550 if value:
533 551 if isinstance(value, (numpy.float32, numpy.float64)):
534 552 value = round(float(value), 2)
535 553 self.data.meta[attr] = value
536 554 if self.colormap == 'jet':
537 555 self.data.meta['colormap'] = 'Jet'
538 556 elif 'RdBu' in self.colormap:
539 557 self.data.meta['colormap'] = 'RdBu'
540 558 else:
541 559 self.data.meta['colormap'] = 'Viridis'
542 560 self.data.meta['interval'] = int(interval)
543 561
544 try:
545 self.sender_queue.put(self.data.tm, block=False)
546 except:
547 tm = self.sender_queue.get()
548 self.sender_queue.put(self.data.tm)
562 self.sender_queue.append(last_time)
549 563
550 564 while True:
551 if self.sender_queue.empty():
552 break
553 tm = self.sender_queue.get()
554 565 try:
555 msg = self.data.jsonify(tm, self.save_code, self.plot_type)
556 except:
557 continue
566 tm = self.sender_queue.popleft()
567 except IndexError:
568 break
569 msg = self.data.jsonify(tm, self.save_code, self.plot_type)
558 570 self.socket.send_string(msg)
559 socks = dict(self.poll.poll(5000))
571 socks = dict(self.poll.poll(2000))
560 572 if socks.get(self.socket) == zmq.POLLIN:
561 573 reply = self.socket.recv_string()
562 574 if reply == 'ok':
563 575 log.log("Response from server ok", self.name)
564 time.sleep(0.2)
576 time.sleep(0.1)
565 577 continue
566 578 else:
567 579 log.warning(
568 580 "Malformed reply from server: {}".format(reply), self.name)
569 581 else:
570 582 log.warning(
571 583 "No response from server, retrying...", self.name)
572 self.sender_queue.put(self.data.tm)
584 self.sender_queue.appendleft(tm)
573 585 self.socket.setsockopt(zmq.LINGER, 0)
574 586 self.socket.close()
575 587 self.poll.unregister(self.socket)
576 time.sleep(0.1)
577 588 self.socket = self.context.socket(zmq.REQ)
578 589 self.socket.connect(self.server)
579 590 self.poll.register(self.socket, zmq.POLLIN)
580 591 break
581 592
582 593 def setup(self):
583 594 '''
584 595 This method should be implemented in the child class, the following
585 596 attributes should be set:
586 597
587 598 self.nrows: number of rows
588 599 self.ncols: number of cols
589 600 self.nplots: number of plots (channels or pairs)
590 601 self.ylabel: label for Y axes
591 602 self.titles: list of axes title
592 603
593 604 '''
594 605 raise NotImplementedError
595 606
596 607 def plot(self):
597 608 '''
598 Must be defined in the child class
609 Must be defined in the child class, the actual plotting method
599 610 '''
600 611 raise NotImplementedError
612
613 def update(self, dataOut):
614 '''
615 Must be defined in the child class, update self.data with new data
616 '''
617
618 data = {
619 self.CODE: getattr(dataOut, 'data_{}'.format(self.CODE))
620 }
621 meta = {}
622
623 return data, meta
601 624
602 625 def run(self, dataOut, **kwargs):
603 626 '''
604 627 Main plotting routine
605 628 '''
606 629
607 630 if self.isConfig is False:
608 631 self.__setup(**kwargs)
609 632
610 633 if self.localtime:
611 634 self.getDateTime = datetime.datetime.fromtimestamp
612 635 else:
613 636 self.getDateTime = datetime.datetime.utcfromtimestamp
614 637
615 638 self.data.setup()
616 639 self.isConfig = True
617 640 if self.server:
618 641 self.context = zmq.Context()
619 642 self.socket = self.context.socket(zmq.REQ)
620 643 self.socket.connect(self.server)
621 644 self.poll = zmq.Poller()
622 645 self.poll.register(self.socket, zmq.POLLIN)
623 646
624 647 tm = getattr(dataOut, self.attr_time)
625 648
626 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
649 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
627 650 self.save_time = tm
628 651 self.__plot()
629 652 self.tmin += self.xrange*60*60
630 653 self.data.setup()
631 654 self.clear_figures()
632 655
633 self.data.update(dataOut, tm)
656 self.__update(dataOut, tm)
634 657
635 658 if self.isPlotConfig is False:
636 659 self.__setup_plot()
637 660 self.isPlotConfig = True
638 661 if self.xaxis == 'time':
639 662 dt = self.getDateTime(tm)
640 663 if self.xmin is None:
641 664 self.tmin = tm
642 665 self.xmin = dt.hour
643 666 minutes = (self.xmin-int(self.xmin)) * 60
644 667 seconds = (minutes - int(minutes)) * 60
645 668 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
646 669 datetime.datetime(1970, 1, 1)).total_seconds()
647 670 if self.localtime:
648 671 self.tmin += time.timezone
649 672
650 673 if self.xmin is not None and self.xmax is not None:
651 674 self.xrange = self.xmax - self.xmin
652 675
653 676 if self.throttle == 0:
654 677 self.__plot()
655 678 else:
656 679 self.__throttle_plot(self.__plot)#, coerce=coerce)
657 680
658 681 def close(self):
659 682
660 683 if self.data and not self.data.flagNoData:
661 self.save_time = self.data.tm
684 self.save_time = self.data.max_time
662 685 self.__plot()
663 686 if self.data and not self.data.flagNoData and self.pause:
664 687 figpause(10)
665 688
Show file before | Show file after | Hide comments
@@ -1,342 +1,101
1 '''
2 Created on Jul 9, 2014
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
3 #
4 # Distributed under the terms of the BSD 3-clause license.
5 """Classes to plo Specra Heis data
3 6
4 @author: roj-idl71
5 '''
6 import os
7 import datetime
8 import numpy
9
10 from schainpy.model.graphics.jroplot_base import Plot
11
12
13 class SpectraHeisScope(Plot):
14
15
16 isConfig = None
17 __nsubplots = None
18
19 WIDTHPROF = None
20 HEIGHTPROF = None
21 PREFIX = 'spc'
22
23 def __init__(self):#, **kwargs):
24
25 Plot.__init__(self)#, **kwargs)
26 self.isConfig = False
27 self.__nsubplots = 1
28
29 self.WIDTH = 230
30 self.HEIGHT = 250
31 self.WIDTHPROF = 120
32 self.HEIGHTPROF = 0
33 self.counter_imagwr = 0
34
35 self.PLOT_CODE = SPEC_CODE
36
37 def getSubplots(self):
38
39 ncol = int(numpy.sqrt(self.nplots)+0.9)
40 nrow = int(self.nplots*1./ncol + 0.9)
41
42 return nrow, ncol
43
44 def setup(self, id, nplots, wintitle, show):
45
46 showprofile = False
47 self.__showprofile = showprofile
48 self.nplots = nplots
49
50 ncolspan = 1
51 colspan = 1
52 if showprofile:
53 ncolspan = 3
54 colspan = 2
55 self.__nsubplots = 2
56
57 self.createFigure(id = id,
58 wintitle = wintitle,
59 widthplot = self.WIDTH + self.WIDTHPROF,
60 heightplot = self.HEIGHT + self.HEIGHTPROF,
61 show = show)
62
63 nrow, ncol = self.getSubplots()
64
65 counter = 0
66 for y in range(nrow):
67 for x in range(ncol):
68
69 if counter >= self.nplots:
70 break
71
72 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
73
74 if showprofile:
75 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
76
77 counter += 1
78
79
80 def run(self, dataOut, id, wintitle="", channelList=None,
81 xmin=None, xmax=None, ymin=None, ymax=None, save=False,
82 figpath='./', figfile=None, ftp=False, wr_period=1, show=True,
83 server=None, folder=None, username=None, password=None,
84 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
85
86 """
87
88 Input:
89 dataOut :
90 id :
91 wintitle :
92 channelList :
93 xmin : None,
94 xmax : None,
95 ymin : None,
96 ymax : None,
97 """
98
99 if dataOut.flagNoData:
100 return dataOut
101
102 if dataOut.realtime:
103 if not(isRealtime(utcdatatime = dataOut.utctime)):
104 print('Skipping this plot function')
105 return
106
107 if channelList == None:
108 channelIndexList = dataOut.channelIndexList
109 else:
110 channelIndexList = []
111 for channel in channelList:
112 if channel not in dataOut.channelList:
113 raise ValueError("Channel %d is not in dataOut.channelList")
114 channelIndexList.append(dataOut.channelList.index(channel))
115
116 # x = dataOut.heightList
117 c = 3E8
118 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
119 #deberia cambiar para el caso de 1Mhz y 100KHz
120 x = numpy.arange(-1*dataOut.nHeights/2.,dataOut.nHeights/2.)*(c/(2*deltaHeight*dataOut.nHeights*1000))
121 #para 1Mhz descomentar la siguiente linea
122 #x= x/(10000.0)
123 # y = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
124 # y = y.real
125 factor = dataOut.normFactor
126 data = dataOut.data_spc / factor
127 datadB = 10.*numpy.log10(data)
128 y = datadB
129
130 #thisDatetime = dataOut.datatime
131 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
132 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
133 xlabel = ""
134 #para 1Mhz descomentar la siguiente linea
135 #xlabel = "Frequency x 10000"
136 ylabel = "Intensity (dB)"
137
138 if not self.isConfig:
139 nplots = len(channelIndexList)
140
141 self.setup(id=id,
142 nplots=nplots,
143 wintitle=wintitle,
144 show=show)
145
146 if xmin == None: xmin = numpy.nanmin(x)
147 if xmax == None: xmax = numpy.nanmax(x)
148 if ymin == None: ymin = numpy.nanmin(y)
149 if ymax == None: ymax = numpy.nanmax(y)
150
151 self.FTP_WEI = ftp_wei
152 self.EXP_CODE = exp_code
153 self.SUB_EXP_CODE = sub_exp_code
154 self.PLOT_POS = plot_pos
155
156 self.isConfig = True
157
158 self.setWinTitle(title)
159
160 for i in range(len(self.axesList)):
161 ychannel = y[i,:]
162 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
163 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[channelIndexList[i]], numpy.max(ychannel), str_datetime)
164 axes = self.axesList[i]
165 axes.pline(x, ychannel,
166 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
167 xlabel=xlabel, ylabel=ylabel, title=title, grid='both')
168
169
170 self.draw()
171
172 self.save(figpath=figpath,
173 figfile=figfile,
174 save=save,
175 ftp=ftp,
176 wr_period=wr_period,
177 thisDatetime=thisDatetime)
178
179 return dataOut
180
181
182 class RTIfromSpectraHeis(Plot):
183
184 isConfig = None
185 __nsubplots = None
186
187 PREFIX = 'rtinoise'
188
189 def __init__(self):#, **kwargs):
190 Plot.__init__(self)#, **kwargs)
191 self.timerange = 24*60*60
192 self.isConfig = False
193 self.__nsubplots = 1
194
195 self.WIDTH = 820
196 self.HEIGHT = 200
197 self.WIDTHPROF = 120
198 self.HEIGHTPROF = 0
199 self.counter_imagwr = 0
200 self.xdata = None
201 self.ydata = None
202 self.figfile = None
203
204 self.PLOT_CODE = RTI_CODE
205
206 def getSubplots(self):
207
208 ncol = 1
209 nrow = 1
210
211 return nrow, ncol
212
213 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
214
215 self.__showprofile = showprofile
216 self.nplots = nplots
217
218 ncolspan = 7
219 colspan = 6
220 self.__nsubplots = 2
221
222 self.createFigure(id = id,
223 wintitle = wintitle,
224 widthplot = self.WIDTH+self.WIDTHPROF,
225 heightplot = self.HEIGHT+self.HEIGHTPROF,
226 show = show)
227
228 nrow, ncol = self.getSubplots()
229
230 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
7 """
231 8
9 import numpy
232 10
233 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
234 xmin=None, xmax=None, ymin=None, ymax=None,
235 timerange=None,
236 save=False, figpath='./', figfile=None, ftp=False, wr_period=1, show=True,
237 server=None, folder=None, username=None, password=None,
238 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
11 from schainpy.model.graphics.jroplot_base import Plot, plt
239 12
240 if dataOut.flagNoData:
241 return dataOut
242 13
14 class SpectraHeisPlot(Plot):
243 15
244 if channelList == None:
245 channelIndexList = dataOut.channelIndexList
246 channelList = dataOut.channelList
247 else:
248 channelIndexList = []
249 for channel in channelList:
250 if channel not in dataOut.channelList:
251 raise ValueError("Channel %d is not in dataOut.channelList")
252 channelIndexList.append(dataOut.channelList.index(channel))
16 CODE = 'spc_heis'
253 17
254 if timerange != None:
255 self.timerange = timerange
18 def setup(self):
256 19
257 x = dataOut.getTimeRange()
258 y = dataOut.heightList
20 self.nplots = len(self.data.channels)
21 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
22 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
23 self.height = 2.6 * self.nrows
24 self.width = 3.5 * self.ncols
25 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.95, 'bottom': 0.08})
26 self.ylabel = 'Intensity [dB]'
27 self.xlabel = 'Frequency [KHz]'
28 self.colorbar = False
259 29
260 factor = dataOut.normFactor
261 data = dataOut.data_spc / factor
262 data = numpy.average(data,axis=1)
263 datadB = 10*numpy.log10(data)
30 def update(self, dataOut):
264 31
265 # factor = dataOut.normFactor
266 # noise = dataOut.getNoise()/factor
267 # noisedB = 10*numpy.log10(noise)
32 data = {}
33 meta = {}
34 spc = 10*numpy.log10(dataOut.data_spc / dataOut.normFactor)
35 data['spc_heis'] = spc
36
37 return data, meta
268 38
269 #thisDatetime = dataOut.datatime
270 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
271 title = wintitle + " RTI: %s" %(thisDatetime.strftime("%d-%b-%Y"))
272 xlabel = "Local Time"
273 ylabel = "Intensity (dB)"
39 def plot(self):
274 40
275 if not self.isConfig:
41 c = 3E8
42 deltaHeight = self.data.yrange[1] - self.data.yrange[0]
43 x = numpy.arange(-1*len(self.data.yrange)/2., len(self.data.yrange)/2.)*(c/(2*deltaHeight*len(self.data.yrange)*1000))
44 self.y = self.data[-1]['spc_heis']
45 self.titles = []
276 46
277 nplots = 1
47 for n, ax in enumerate(self.axes):
48 ychannel = self.y[n,:]
49 if ax.firsttime:
50 self.xmin = min(x) if self.xmin is None else self.xmin
51 self.xmax = max(x) if self.xmax is None else self.xmax
52 ax.plt = ax.plot(x, ychannel, lw=1, color='b')[0]
53 else:
54 ax.plt.set_data(x, ychannel)
278 55
279 self.setup(id=id,
280 nplots=nplots,
281 wintitle=wintitle,
282 showprofile=showprofile,
283 show=show)
56 self.titles.append("Channel {}: {:4.2f}dB".format(n, numpy.max(ychannel)))
284 57
285 self.tmin, self.tmax = self.getTimeLim(x, xmin, xmax)
286 58
287 if ymin == None: ymin = numpy.nanmin(datadB)
288 if ymax == None: ymax = numpy.nanmax(datadB)
59 class RTIHeisPlot(Plot):
289 60
290 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
291 self.isConfig = True
292 self.figfile = figfile
293 self.xdata = numpy.array([])
294 self.ydata = numpy.array([])
61 CODE = 'rti_heis'
295 62
296 self.FTP_WEI = ftp_wei
297 self.EXP_CODE = exp_code
298 self.SUB_EXP_CODE = sub_exp_code
299 self.PLOT_POS = plot_pos
63 def setup(self):
300 64
301 self.setWinTitle(title)
65 self.xaxis = 'time'
66 self.ncols = 1
67 self.nrows = 1
68 self.nplots = 1
69 self.ylabel = 'Intensity [dB]'
70 self.xlabel = 'Time'
71 self.titles = ['RTI']
72 self.colorbar = False
73 self.height = 4
74 self.plots_adjust.update({'right': 0.85 })
302 75
76 def update(self, dataOut):
303 77
304 # title = "RTI %s" %(thisDatetime.strftime("%d-%b-%Y"))
305 title = "RTI - %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
78 data = {}
79 meta = {}
80 spc = dataOut.data_spc / dataOut.normFactor
81 spc = 10*numpy.log10(numpy.average(spc, axis=1))
82 data['rti_heis'] = spc
83
84 return data, meta
306 85
307 legendlabels = ["channel %d"%idchannel for idchannel in channelList]
308 axes = self.axesList[0]
86 def plot(self):
309 87
310 self.xdata = numpy.hstack((self.xdata, x[0:1]))
88 x = self.data.times
89 Y = self.data['rti_heis']
311 90
312 if len(self.ydata)==0:
313 self.ydata = datadB[channelIndexList].reshape(-1,1)
91 if self.axes[0].firsttime:
92 self.ymin = numpy.nanmin(Y) - 5 if self.ymin == None else self.ymin
93 self.ymax = numpy.nanmax(Y) + 5 if self.ymax == None else self.ymax
94 for ch in self.data.channels:
95 y = Y[ch]
96 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
97 plt.legend(bbox_to_anchor=(1.18, 1.0))
314 98 else:
315 self.ydata = numpy.hstack((self.ydata, datadB[channelIndexList].reshape(-1,1)))
316
317
318 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
319 xmin=self.tmin, xmax=self.tmax, ymin=ymin, ymax=ymax,
320 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='.', markersize=8, linestyle="solid", grid='both',
321 XAxisAsTime=True
322 )
323
324 self.draw()
325
326 update_figfile = False
327
328 if dataOut.ltctime >= self.tmax:
329 self.counter_imagwr = wr_period
330 self.isConfig = False
331 update_figfile = True
332
333 self.save(figpath=figpath,
334 figfile=figfile,
335 save=save,
336 ftp=ftp,
337 wr_period=wr_period,
338 thisDatetime=thisDatetime,
339 update_figfile=update_figfile)
340
341
342 return dataOut No newline at end of file
99 for ch in self.data.channels:
100 y = Y[ch]
101 self.axes[0].lines[ch].set_data(x, y)
Show file before | Show file after | Hide comments
@@ -1,339 +1,358
1 1 import os
2 2 import datetime
3 3 import numpy
4 4
5 5 from schainpy.model.graphics.jroplot_base import Plot, plt
6 6 from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot
7 7 from schainpy.utils import log
8 8
9 9 EARTH_RADIUS = 6.3710e3
10 10
11 11
12 12 def ll2xy(lat1, lon1, lat2, lon2):
13 13
14 14 p = 0.017453292519943295
15 15 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
16 16 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
17 17 r = 12742 * numpy.arcsin(numpy.sqrt(a))
18 18 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
19 19 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
20 20 theta = -theta + numpy.pi/2
21 21 return r*numpy.cos(theta), r*numpy.sin(theta)
22 22
23 23
24 24 def km2deg(km):
25 25 '''
26 26 Convert distance in km to degrees
27 27 '''
28 28
29 29 return numpy.rad2deg(km/EARTH_RADIUS)
30 30
31 31
32 32
33 33 class SpectralMomentsPlot(SpectraPlot):
34 34 '''
35 35 Plot for Spectral Moments
36 36 '''
37 37 CODE = 'spc_moments'
38 38 colormap = 'jet'
39 39 plot_type = 'pcolor'
40 40
41 41
42 42 class SnrPlot(RTIPlot):
43 43 '''
44 44 Plot for SNR Data
45 45 '''
46 46
47 47 CODE = 'snr'
48 48 colormap = 'jet'
49 49
50 def update(self, dataOut):
51
52 data = {
53 'snr': 10*numpy.log10(dataOut.data_snr)
54 }
55
56 return data, {}
50 57
51 58 class DopplerPlot(RTIPlot):
52 59 '''
53 60 Plot for DOPPLER Data (1st moment)
54 61 '''
55 62
56 63 CODE = 'dop'
57 64 colormap = 'jet'
58 65
66 def update(self, dataOut):
67
68 data = {
69 'dop': 10*numpy.log10(dataOut.data_dop)
70 }
71
72 return data, {}
59 73
60 74 class PowerPlot(RTIPlot):
61 75 '''
62 76 Plot for Power Data (0 moment)
63 77 '''
64 78
65 79 CODE = 'pow'
66 80 colormap = 'jet'
67 81
82 def update(self, dataOut):
83
84 data = {
85 'pow': 10*numpy.log10(dataOut.data_pow)
86 }
87
88 return data, {}
68 89
69 90 class SpectralWidthPlot(RTIPlot):
70 91 '''
71 92 Plot for Spectral Width Data (2nd moment)
72 93 '''
73 94
74 95 CODE = 'width'
75 96 colormap = 'jet'
76 97
98 def update(self, dataOut):
99
100 data = {
101 'width': dataOut.data_width
102 }
103
104 return data, {}
77 105
78 106 class SkyMapPlot(Plot):
79 107 '''
80 108 Plot for meteors detection data
81 109 '''
82 110
83 111 CODE = 'param'
84 112
85 113 def setup(self):
86 114
87 115 self.ncols = 1
88 116 self.nrows = 1
89 117 self.width = 7.2
90 118 self.height = 7.2
91 119 self.nplots = 1
92 120 self.xlabel = 'Zonal Zenith Angle (deg)'
93 121 self.ylabel = 'Meridional Zenith Angle (deg)'
94 122 self.polar = True
95 123 self.ymin = -180
96 124 self.ymax = 180
97 125 self.colorbar = False
98 126
99 127 def plot(self):
100 128
101 129 arrayParameters = numpy.concatenate(self.data['param'])
102 130 error = arrayParameters[:, -1]
103 131 indValid = numpy.where(error == 0)[0]
104 132 finalMeteor = arrayParameters[indValid, :]
105 133 finalAzimuth = finalMeteor[:, 3]
106 134 finalZenith = finalMeteor[:, 4]
107 135
108 136 x = finalAzimuth * numpy.pi / 180
109 137 y = finalZenith
110 138
111 139 ax = self.axes[0]
112 140
113 141 if ax.firsttime:
114 142 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
115 143 else:
116 144 ax.plot.set_data(x, y)
117 145
118 146 dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
119 147 dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
120 148 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
121 149 dt2,
122 150 len(x))
123 151 self.titles[0] = title
124 152
125 153
126 class ParametersPlot(RTIPlot):
154 class GenericRTIPlot(Plot):
127 155 '''
128 Plot for data_param object
156 Plot for data_xxxx object
129 157 '''
130 158
131 159 CODE = 'param'
132 colormap = 'seismic'
160 colormap = 'viridis'
161 plot_type = 'pcolorbuffer'
133 162
134 163 def setup(self):
135 164 self.xaxis = 'time'
136 165 self.ncols = 1
137 self.nrows = self.data.shape(self.CODE)[0]
166 self.nrows = self.data.shape(self.attr_data)[0]
138 167 self.nplots = self.nrows
139 168 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
140 169
141 170 if not self.xlabel:
142 171 self.xlabel = 'Time'
143
144 if self.showSNR:
145 self.nrows += 1
146 self.nplots += 1
147 172
148 173 self.ylabel = 'Height [km]'
149 174 if not self.titles:
150 175 self.titles = self.data.parameters \
151 176 if self.data.parameters else ['Param {}'.format(x) for x in range(self.nrows)]
152 if self.showSNR:
153 self.titles.append('SNR')
154 177
178 def update(self, dataOut):
179
180 data = {
181 self.attr_data : getattr(dataOut, self.attr_data)
182 }
183
184 meta = {}
185
186 return data, meta
187
155 188 def plot(self):
156 self.data.normalize_heights()
189 # self.data.normalize_heights()
157 190 self.x = self.data.times
158 self.y = self.data.heights
159 if self.showSNR:
160 self.z = numpy.concatenate(
161 (self.data[self.CODE], self.data['snr'])
162 )
163 else:
164 self.z = self.data[self.CODE]
191 self.y = self.data.yrange
192 self.z = self.data[self.attr_data]
165 193
166 194 self.z = numpy.ma.masked_invalid(self.z)
167 195
168 196 if self.decimation is None:
169 197 x, y, z = self.fill_gaps(self.x, self.y, self.z)
170 198 else:
171 199 x, y, z = self.fill_gaps(*self.decimate())
172 200
173 201 for n, ax in enumerate(self.axes):
174 202
175 203 self.zmax = self.zmax if self.zmax is not None else numpy.max(
176 204 self.z[n])
177 205 self.zmin = self.zmin if self.zmin is not None else numpy.min(
178 206 self.z[n])
179 207
180 208 if ax.firsttime:
181 209 if self.zlimits is not None:
182 210 self.zmin, self.zmax = self.zlimits[n]
183 211
184 212 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
185 213 vmin=self.zmin,
186 214 vmax=self.zmax,
187 215 cmap=self.cmaps[n]
188 216 )
189 217 else:
190 218 if self.zlimits is not None:
191 219 self.zmin, self.zmax = self.zlimits[n]
192 220 ax.collections.remove(ax.collections[0])
193 221 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
194 222 vmin=self.zmin,
195 223 vmax=self.zmax,
196 224 cmap=self.cmaps[n]
197 225 )
198 226
199 227
200 class OutputPlot(ParametersPlot):
201 '''
202 Plot data_output object
203 '''
204
205 CODE = 'output'
206 colormap = 'seismic'
207
208
209 228 class PolarMapPlot(Plot):
210 229 '''
211 230 Plot for weather radar
212 231 '''
213 232
214 233 CODE = 'param'
215 234 colormap = 'seismic'
216 235
217 236 def setup(self):
218 237 self.ncols = 1
219 238 self.nrows = 1
220 239 self.width = 9
221 240 self.height = 8
222 241 self.mode = self.data.meta['mode']
223 242 if self.channels is not None:
224 243 self.nplots = len(self.channels)
225 244 self.nrows = len(self.channels)
226 245 else:
227 246 self.nplots = self.data.shape(self.CODE)[0]
228 247 self.nrows = self.nplots
229 248 self.channels = list(range(self.nplots))
230 249 if self.mode == 'E':
231 250 self.xlabel = 'Longitude'
232 251 self.ylabel = 'Latitude'
233 252 else:
234 253 self.xlabel = 'Range (km)'
235 254 self.ylabel = 'Height (km)'
236 255 self.bgcolor = 'white'
237 256 self.cb_labels = self.data.meta['units']
238 257 self.lat = self.data.meta['latitude']
239 258 self.lon = self.data.meta['longitude']
240 259 self.xmin, self.xmax = float(
241 260 km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
242 261 self.ymin, self.ymax = float(
243 262 km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
244 263 # self.polar = True
245 264
246 265 def plot(self):
247 266
248 267 for n, ax in enumerate(self.axes):
249 268 data = self.data['param'][self.channels[n]]
250 269
251 270 zeniths = numpy.linspace(
252 271 0, self.data.meta['max_range'], data.shape[1])
253 272 if self.mode == 'E':
254 azimuths = -numpy.radians(self.data.heights)+numpy.pi/2
273 azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
255 274 r, theta = numpy.meshgrid(zeniths, azimuths)
256 275 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
257 276 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
258 277 x = km2deg(x) + self.lon
259 278 y = km2deg(y) + self.lat
260 279 else:
261 azimuths = numpy.radians(self.data.heights)
280 azimuths = numpy.radians(self.data.yrange)
262 281 r, theta = numpy.meshgrid(zeniths, azimuths)
263 282 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
264 283 self.y = zeniths
265 284
266 285 if ax.firsttime:
267 286 if self.zlimits is not None:
268 287 self.zmin, self.zmax = self.zlimits[n]
269 288 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
270 289 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
271 290 vmin=self.zmin,
272 291 vmax=self.zmax,
273 292 cmap=self.cmaps[n])
274 293 else:
275 294 if self.zlimits is not None:
276 295 self.zmin, self.zmax = self.zlimits[n]
277 296 ax.collections.remove(ax.collections[0])
278 297 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
279 298 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
280 299 vmin=self.zmin,
281 300 vmax=self.zmax,
282 301 cmap=self.cmaps[n])
283 302
284 303 if self.mode == 'A':
285 304 continue
286 305
287 306 # plot district names
288 307 f = open('/data/workspace/schain_scripts/distrito.csv')
289 308 for line in f:
290 309 label, lon, lat = [s.strip() for s in line.split(',') if s]
291 310 lat = float(lat)
292 311 lon = float(lon)
293 312 # ax.plot(lon, lat, '.b', ms=2)
294 313 ax.text(lon, lat, label.decode('utf8'), ha='center',
295 314 va='bottom', size='8', color='black')
296 315
297 316 # plot limites
298 317 limites = []
299 318 tmp = []
300 319 for line in open('/data/workspace/schain_scripts/lima.csv'):
301 320 if '#' in line:
302 321 if tmp:
303 322 limites.append(tmp)
304 323 tmp = []
305 324 continue
306 325 values = line.strip().split(',')
307 326 tmp.append((float(values[0]), float(values[1])))
308 327 for points in limites:
309 328 ax.add_patch(
310 329 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
311 330
312 331 # plot Cuencas
313 332 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
314 333 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
315 334 values = [line.strip().split(',') for line in f]
316 335 points = [(float(s[0]), float(s[1])) for s in values]
317 336 ax.add_patch(Polygon(points, ec='b', fc='none'))
318 337
319 338 # plot grid
320 339 for r in (15, 30, 45, 60):
321 340 ax.add_artist(plt.Circle((self.lon, self.lat),
322 341 km2deg(r), color='0.6', fill=False, lw=0.2))
323 342 ax.text(
324 343 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
325 344 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
326 345 '{}km'.format(r),
327 346 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
328 347
329 348 if self.mode == 'E':
330 349 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
331 350 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
332 351 else:
333 352 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
334 353 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
335 354
336 355 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
337 356 self.titles = ['{} {}'.format(
338 357 self.data.parameters[x], title) for x in self.channels]
339 358
Show file before | Show file after | Hide comments
@@ -1,641 +1,702
1 '''
2 Created on Jul 9, 2014
3 Modified on May 10, 2020
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
3 #
4 # Distributed under the terms of the BSD 3-clause license.
5 """Classes to plot Spectra data
4 6
5 @author: Juan C. Espinoza
6 '''
7 """
7 8
8 9 import os
9 import datetime
10 10 import numpy
11 11
12 from schainpy.model.graphics.jroplot_base import Plot, plt
12 from schainpy.model.graphics.jroplot_base import Plot, plt, log
13 13
14 14
15 15 class SpectraPlot(Plot):
16 16 '''
17 17 Plot for Spectra data
18 18 '''
19 19
20 20 CODE = 'spc'
21 21 colormap = 'jet'
22 22 plot_type = 'pcolor'
23 buffering = False
23 24
24 25 def setup(self):
25 26 self.nplots = len(self.data.channels)
26 27 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
27 28 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
28 29 self.height = 2.6 * self.nrows
29 30 self.cb_label = 'dB'
30 31 if self.showprofile:
31 32 self.width = 4 * self.ncols
32 33 else:
33 34 self.width = 3.5 * self.ncols
34 35 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
35 36 self.ylabel = 'Range [km]'
36 37
38 def update(self, dataOut):
39
40 data = {}
41 meta = {}
42 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
43 data['spc'] = spc
44 data['rti'] = dataOut.getPower()
45 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
46 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
47 if self.CODE == 'spc_moments':
48 data['moments'] = dataOut.moments
49
50 return data, meta
51
37 52 def plot(self):
38 53 if self.xaxis == "frequency":
39 54 x = self.data.xrange[0]
40 55 self.xlabel = "Frequency (kHz)"
41 56 elif self.xaxis == "time":
42 57 x = self.data.xrange[1]
43 58 self.xlabel = "Time (ms)"
44 59 else:
45 60 x = self.data.xrange[2]
46 61 self.xlabel = "Velocity (m/s)"
47 62
48 63 if self.CODE == 'spc_moments':
49 64 x = self.data.xrange[2]
50 65 self.xlabel = "Velocity (m/s)"
51 66
52 67 self.titles = []
53 68
54 y = self.data.heights
69 y = self.data.yrange
55 70 self.y = y
56 z = self.data['spc']
71
72 data = self.data[-1]
73 z = data['spc']
57 74
58 75 for n, ax in enumerate(self.axes):
59 noise = self.data['noise'][n][-1]
76 noise = data['noise'][n]
60 77 if self.CODE == 'spc_moments':
61 mean = self.data['moments'][n, :, 1, :][-1]
78 mean = data['moments'][n, 2]
62 79 if ax.firsttime:
63 80 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
64 81 self.xmin = self.xmin if self.xmin else -self.xmax
65 82 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
66 83 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
67 84 ax.plt = ax.pcolormesh(x, y, z[n].T,
68 85 vmin=self.zmin,
69 86 vmax=self.zmax,
70 87 cmap=plt.get_cmap(self.colormap)
71 88 )
72 89
73 90 if self.showprofile:
74 91 ax.plt_profile = self.pf_axes[n].plot(
75 self.data['rti'][n][-1], y)[0]
92 data['rti'][n], y)[0]
76 93 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
77 94 color="k", linestyle="dashed", lw=1)[0]
78 95 if self.CODE == 'spc_moments':
79 96 ax.plt_mean = ax.plot(mean, y, color='k')[0]
80 97 else:
81 98 ax.plt.set_array(z[n].T.ravel())
82 99 if self.showprofile:
83 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
100 ax.plt_profile.set_data(data['rti'][n], y)
84 101 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
85 102 if self.CODE == 'spc_moments':
86 103 ax.plt_mean.set_data(mean, y)
87 104 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
88 105
89 106
90 107 class CrossSpectraPlot(Plot):
91 108
92 109 CODE = 'cspc'
93 110 colormap = 'jet'
94 111 plot_type = 'pcolor'
95 112 zmin_coh = None
96 113 zmax_coh = None
97 114 zmin_phase = None
98 115 zmax_phase = None
99 116
100 117 def setup(self):
101 118
102 119 self.ncols = 4
103 self.nrows = len(self.data.pairs)
104 self.nplots = self.nrows * 4
120 self.nplots = len(self.data.pairs) * 2
121 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
105 122 self.width = 3.1 * self.ncols
106 123 self.height = 2.6 * self.nrows
107 124 self.ylabel = 'Range [km]'
108 125 self.showprofile = False
109 126 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
110 127
128 def update(self, dataOut):
129
130 data = {}
131 meta = {}
132
133 spc = dataOut.data_spc
134 cspc = dataOut.data_cspc
135 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
136 meta['pairs'] = dataOut.pairsList
137
138 tmp = []
139
140 for n, pair in enumerate(meta['pairs']):
141 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
142 coh = numpy.abs(out)
143 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
144 tmp.append(coh)
145 tmp.append(phase)
146
147 data['cspc'] = numpy.array(tmp)
148
149 return data, meta
150
111 151 def plot(self):
112 152
113 153 if self.xaxis == "frequency":
114 154 x = self.data.xrange[0]
115 155 self.xlabel = "Frequency (kHz)"
116 156 elif self.xaxis == "time":
117 157 x = self.data.xrange[1]
118 158 self.xlabel = "Time (ms)"
119 159 else:
120 160 x = self.data.xrange[2]
121 161 self.xlabel = "Velocity (m/s)"
122 162
123 163 self.titles = []
124 164
125 y = self.data.heights
165 y = self.data.yrange
126 166 self.y = y
127 nspc = self.data['spc']
128 spc = self.data['cspc'][0]
129 cspc = self.data['cspc'][1]
130 167
131 for n in range(self.nrows):
132 noise = self.data['noise'][:,-1]
133 pair = self.data.pairs[n]
134 ax = self.axes[4 * n]
135 if ax.firsttime:
136 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
137 self.xmin = self.xmin if self.xmin else -self.xmax
138 self.zmin = self.zmin if self.zmin else numpy.nanmin(nspc)
139 self.zmax = self.zmax if self.zmax else numpy.nanmax(nspc)
140 ax.plt = ax.pcolormesh(x , y , nspc[pair[0]].T,
141 vmin=self.zmin,
142 vmax=self.zmax,
143 cmap=plt.get_cmap(self.colormap)
144 )
145 else:
146 ax.plt.set_array(nspc[pair[0]].T.ravel())
147 self.titles.append('CH {}: {:3.2f}dB'.format(pair[0], noise[pair[0]]))
148
149 ax = self.axes[4 * n + 1]
150 if ax.firsttime:
151 ax.plt = ax.pcolormesh(x , y, nspc[pair[1]].T,
152 vmin=self.zmin,
153 vmax=self.zmax,
154 cmap=plt.get_cmap(self.colormap)
155 )
156 else:
157 ax.plt.set_array(nspc[pair[1]].T.ravel())
158 self.titles.append('CH {}: {:3.2f}dB'.format(pair[1], noise[pair[1]]))
159
160 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
161 coh = numpy.abs(out)
162 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
168 data = self.data[-1]
169 cspc = data['cspc']
163 170
164 ax = self.axes[4 * n + 2]
171 for n in range(len(self.data.pairs)):
172 pair = self.data.pairs[n]
173 coh = cspc[n*2]
174 phase = cspc[n*2+1]
175 ax = self.axes[2 * n]
165 176 if ax.firsttime:
166 177 ax.plt = ax.pcolormesh(x, y, coh.T,
167 178 vmin=0,
168 179 vmax=1,
169 180 cmap=plt.get_cmap(self.colormap_coh)
170 181 )
171 182 else:
172 183 ax.plt.set_array(coh.T.ravel())
173 184 self.titles.append(
174 185 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
175 186
176 ax = self.axes[4 * n + 3]
187 ax = self.axes[2 * n + 1]
177 188 if ax.firsttime:
178 189 ax.plt = ax.pcolormesh(x, y, phase.T,
179 190 vmin=-180,
180 191 vmax=180,
181 192 cmap=plt.get_cmap(self.colormap_phase)
182 193 )
183 194 else:
184 195 ax.plt.set_array(phase.T.ravel())
185 196 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
186 197
187 198
188 199 class RTIPlot(Plot):
189 200 '''
190 201 Plot for RTI data
191 202 '''
192 203
193 204 CODE = 'rti'
194 205 colormap = 'jet'
195 206 plot_type = 'pcolorbuffer'
196 207
197 208 def setup(self):
198 209 self.xaxis = 'time'
199 210 self.ncols = 1
200 211 self.nrows = len(self.data.channels)
201 212 self.nplots = len(self.data.channels)
202 213 self.ylabel = 'Range [km]'
203 214 self.xlabel = 'Time'
204 215 self.cb_label = 'dB'
205 216 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95})
206 217 self.titles = ['{} Channel {}'.format(
207 218 self.CODE.upper(), x) for x in range(self.nrows)]
208 219
220 def update(self, dataOut):
221
222 data = {}
223 meta = {}
224 data['rti'] = dataOut.getPower()
225 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
226
227 return data, meta
228
209 229 def plot(self):
210 230 self.x = self.data.times
211 self.y = self.data.heights
231 self.y = self.data.yrange
212 232 self.z = self.data[self.CODE]
213 233 self.z = numpy.ma.masked_invalid(self.z)
214 234
215 235 if self.decimation is None:
216 236 x, y, z = self.fill_gaps(self.x, self.y, self.z)
217 237 else:
218 238 x, y, z = self.fill_gaps(*self.decimate())
219 239
220 240 for n, ax in enumerate(self.axes):
221 241 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
222 242 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
243 data = self.data[-1]
223 244 if ax.firsttime:
224 245 ax.plt = ax.pcolormesh(x, y, z[n].T,
225 246 vmin=self.zmin,
226 247 vmax=self.zmax,
227 248 cmap=plt.get_cmap(self.colormap)
228 249 )
229 250 if self.showprofile:
230 251 ax.plot_profile = self.pf_axes[n].plot(
231 self.data['rti'][n][-1], self.y)[0]
232 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
252 data['rti'][n], self.y)[0]
253 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
233 254 color="k", linestyle="dashed", lw=1)[0]
234 255 else:
235 256 ax.collections.remove(ax.collections[0])
236 257 ax.plt = ax.pcolormesh(x, y, z[n].T,
237 258 vmin=self.zmin,
238 259 vmax=self.zmax,
239 260 cmap=plt.get_cmap(self.colormap)
240 261 )
241 262 if self.showprofile:
242 ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
263 ax.plot_profile.set_data(data['rti'][n], self.y)
243 264 ax.plot_noise.set_data(numpy.repeat(
244 self.data['noise'][n][-1], len(self.y)), self.y)
265 data['noise'][n], len(self.y)), self.y)
245 266
246 267
247 268 class CoherencePlot(RTIPlot):
248 269 '''
249 270 Plot for Coherence data
250 271 '''
251 272
252 273 CODE = 'coh'
253 274
254 275 def setup(self):
255 276 self.xaxis = 'time'
256 277 self.ncols = 1
257 278 self.nrows = len(self.data.pairs)
258 279 self.nplots = len(self.data.pairs)
259 280 self.ylabel = 'Range [km]'
260 281 self.xlabel = 'Time'
261 282 self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1,'right':0.95})
262 283 if self.CODE == 'coh':
263 284 self.cb_label = ''
264 285 self.titles = [
265 286 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
266 287 else:
267 288 self.cb_label = 'Degrees'
268 289 self.titles = [
269 290 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
270 291
292 def update(self, dataOut):
293
294 data = {}
295 meta = {}
296 data['coh'] = dataOut.getCoherence()
297 meta['pairs'] = dataOut.pairsList
298
299 return data, meta
271 300
272 301 class PhasePlot(CoherencePlot):
273 302 '''
274 303 Plot for Phase map data
275 304 '''
276 305
277 306 CODE = 'phase'
278 307 colormap = 'seismic'
279 308
309 def update(self, dataOut):
310
311 data = {}
312 meta = {}
313 data['phase'] = dataOut.getCoherence(phase=True)
314 meta['pairs'] = dataOut.pairsList
315
316 return data, meta
280 317
281 318 class NoisePlot(Plot):
282 319 '''
283 320 Plot for noise
284 321 '''
285 322
286 323 CODE = 'noise'
287 324 plot_type = 'scatterbuffer'
288 325
289
290 326 def setup(self):
291 327 self.xaxis = 'time'
292 328 self.ncols = 1
293 329 self.nrows = 1
294 330 self.nplots = 1
295 331 self.ylabel = 'Intensity [dB]'
296 332 self.xlabel = 'Time'
297 333 self.titles = ['Noise']
298 334 self.colorbar = False
335 self.plots_adjust.update({'right': 0.85 })
336
337 def update(self, dataOut):
338
339 data = {}
340 meta = {}
341 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor).reshape(dataOut.nChannels, 1)
342 meta['yrange'] = numpy.array([])
343
344 return data, meta
299 345
300 346 def plot(self):
301 347
302 348 x = self.data.times
303 349 xmin = self.data.min_time
304 350 xmax = xmin + self.xrange * 60 * 60
305 Y = self.data[self.CODE]
351 Y = self.data['noise']
306 352
307 353 if self.axes[0].firsttime:
354 self.ymin = numpy.nanmin(Y) - 5
355 self.ymax = numpy.nanmax(Y) + 5
308 356 for ch in self.data.channels:
309 357 y = Y[ch]
310 358 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
311 plt.legend()
359 plt.legend(bbox_to_anchor=(1.18, 1.0))
312 360 else:
313 361 for ch in self.data.channels:
314 362 y = Y[ch]
315 363 self.axes[0].lines[ch].set_data(x, y)
316 364
317 self.ymin = numpy.nanmin(Y) - 5
318 self.ymax = numpy.nanmax(Y) + 5
319
320
365
321 366 class PowerProfilePlot(Plot):
322 367
323 CODE = 'spcprofile'
368 CODE = 'pow_profile'
324 369 plot_type = 'scatter'
325 buffering = False
326 370
327 371 def setup(self):
328 372
329 373 self.ncols = 1
330 374 self.nrows = 1
331 375 self.nplots = 1
332 376 self.height = 4
333 377 self.width = 3
334 378 self.ylabel = 'Range [km]'
335 379 self.xlabel = 'Intensity [dB]'
336 380 self.titles = ['Power Profile']
337 381 self.colorbar = False
338 382
383 def update(self, dataOut):
384
385 data = {}
386 meta = {}
387 data[self.CODE] = dataOut.getPower()
388
389 return data, meta
390
339 391 def plot(self):
340 392
341 y = self.data.heights
393 y = self.data.yrange
342 394 self.y = y
343 395
344 x = self.data['spcprofile']
396 x = self.data[-1][self.CODE]
345 397
346 398 if self.xmin is None: self.xmin = numpy.nanmin(x)*0.9
347 399 if self.xmax is None: self.xmax = numpy.nanmax(x)*1.1
348 400
349 401 if self.axes[0].firsttime:
350 402 for ch in self.data.channels:
351 403 self.axes[0].plot(x[ch], y, lw=1, label='Ch{}'.format(ch))
352 404 plt.legend()
353 405 else:
354 406 for ch in self.data.channels:
355 407 self.axes[0].lines[ch].set_data(x[ch], y)
356 408
357 409
358 410 class SpectraCutPlot(Plot):
359 411
360 412 CODE = 'spc_cut'
361 413 plot_type = 'scatter'
362 414 buffering = False
363 415
364 416 def setup(self):
365 417
366 418 self.nplots = len(self.data.channels)
367 419 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
368 420 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
369 421 self.width = 3.4 * self.ncols + 1.5
370 422 self.height = 3 * self.nrows
371 423 self.ylabel = 'Power [dB]'
372 424 self.colorbar = False
373 425 self.plots_adjust.update({'left':0.1, 'hspace':0.3, 'right': 0.75, 'bottom':0.08})
374 426
427 def update(self, dataOut):
428
429 data = {}
430 meta = {}
431 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
432 data['spc'] = spc
433 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
434
435 return data, meta
436
375 437 def plot(self):
376 438 if self.xaxis == "frequency":
377 439 x = self.data.xrange[0][1:]
378 440 self.xlabel = "Frequency (kHz)"
379 441 elif self.xaxis == "time":
380 442 x = self.data.xrange[1]
381 443 self.xlabel = "Time (ms)"
382 444 else:
383 445 x = self.data.xrange[2]
384 446 self.xlabel = "Velocity (m/s)"
385 447
386 448 self.titles = []
387 449
388 y = self.data.heights
389 #self.y = y
390 z = self.data['spc_cut']
450 y = self.data.yrange
451 z = self.data[-1]['spc']
391 452
392 453 if self.height_index:
393 454 index = numpy.array(self.height_index)
394 455 else:
395 456 index = numpy.arange(0, len(y), int((len(y))/9))
396 457
397 458 for n, ax in enumerate(self.axes):
398 459 if ax.firsttime:
399 460 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
400 461 self.xmin = self.xmin if self.xmin else -self.xmax
401 462 self.ymin = self.ymin if self.ymin else numpy.nanmin(z)
402 463 self.ymax = self.ymax if self.ymax else numpy.nanmax(z)
403 464 ax.plt = ax.plot(x, z[n, :, index].T)
404 465 labels = ['Range = {:2.1f}km'.format(y[i]) for i in index]
405 466 self.figures[0].legend(ax.plt, labels, loc='center right')
406 467 else:
407 468 for i, line in enumerate(ax.plt):
408 469 line.set_data(x, z[n, :, i])
409 470 self.titles.append('CH {}'.format(n))
410 471
411 472
412 473 class BeaconPhase(Plot):
413 474
414 475 __isConfig = None
415 476 __nsubplots = None
416 477
417 478 PREFIX = 'beacon_phase'
418 479
419 480 def __init__(self):
420 481 Plot.__init__(self)
421 482 self.timerange = 24*60*60
422 483 self.isConfig = False
423 484 self.__nsubplots = 1
424 485 self.counter_imagwr = 0
425 486 self.WIDTH = 800
426 487 self.HEIGHT = 400
427 488 self.WIDTHPROF = 120
428 489 self.HEIGHTPROF = 0
429 490 self.xdata = None
430 491 self.ydata = None
431 492
432 493 self.PLOT_CODE = BEACON_CODE
433 494
434 495 self.FTP_WEI = None
435 496 self.EXP_CODE = None
436 497 self.SUB_EXP_CODE = None
437 498 self.PLOT_POS = None
438 499
439 500 self.filename_phase = None
440 501
441 502 self.figfile = None
442 503
443 504 self.xmin = None
444 505 self.xmax = None
445 506
446 507 def getSubplots(self):
447 508
448 509 ncol = 1
449 510 nrow = 1
450 511
451 512 return nrow, ncol
452 513
453 514 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
454 515
455 516 self.__showprofile = showprofile
456 517 self.nplots = nplots
457 518
458 519 ncolspan = 7
459 520 colspan = 6
460 521 self.__nsubplots = 2
461 522
462 523 self.createFigure(id = id,
463 524 wintitle = wintitle,
464 525 widthplot = self.WIDTH+self.WIDTHPROF,
465 526 heightplot = self.HEIGHT+self.HEIGHTPROF,
466 527 show=show)
467 528
468 529 nrow, ncol = self.getSubplots()
469 530
470 531 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
471 532
472 533 def save_phase(self, filename_phase):
473 534 f = open(filename_phase,'w+')
474 535 f.write('\n\n')
475 536 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
476 537 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
477 538 f.close()
478 539
479 540 def save_data(self, filename_phase, data, data_datetime):
480 541 f=open(filename_phase,'a')
481 542 timetuple_data = data_datetime.timetuple()
482 543 day = str(timetuple_data.tm_mday)
483 544 month = str(timetuple_data.tm_mon)
484 545 year = str(timetuple_data.tm_year)
485 546 hour = str(timetuple_data.tm_hour)
486 547 minute = str(timetuple_data.tm_min)
487 548 second = str(timetuple_data.tm_sec)
488 549 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
489 550 f.close()
490 551
491 552 def plot(self):
492 553 log.warning('TODO: Not yet implemented...')
493 554
494 555 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
495 556 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
496 557 timerange=None,
497 558 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
498 559 server=None, folder=None, username=None, password=None,
499 560 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
500 561
501 562 if dataOut.flagNoData:
502 563 return dataOut
503 564
504 565 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
505 566 return
506 567
507 568 if pairsList == None:
508 569 pairsIndexList = dataOut.pairsIndexList[:10]
509 570 else:
510 571 pairsIndexList = []
511 572 for pair in pairsList:
512 573 if pair not in dataOut.pairsList:
513 574 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
514 575 pairsIndexList.append(dataOut.pairsList.index(pair))
515 576
516 577 if pairsIndexList == []:
517 578 return
518 579
519 580 # if len(pairsIndexList) > 4:
520 581 # pairsIndexList = pairsIndexList[0:4]
521 582
522 583 hmin_index = None
523 584 hmax_index = None
524 585
525 586 if hmin != None and hmax != None:
526 587 indexes = numpy.arange(dataOut.nHeights)
527 588 hmin_list = indexes[dataOut.heightList >= hmin]
528 589 hmax_list = indexes[dataOut.heightList <= hmax]
529 590
530 591 if hmin_list.any():
531 592 hmin_index = hmin_list[0]
532 593
533 594 if hmax_list.any():
534 595 hmax_index = hmax_list[-1]+1
535 596
536 597 x = dataOut.getTimeRange()
537 598
538 599 thisDatetime = dataOut.datatime
539 600
540 601 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
541 602 xlabel = "Local Time"
542 603 ylabel = "Phase (degrees)"
543 604
544 605 update_figfile = False
545 606
546 607 nplots = len(pairsIndexList)
547 608 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
548 609 phase_beacon = numpy.zeros(len(pairsIndexList))
549 610 for i in range(nplots):
550 611 pair = dataOut.pairsList[pairsIndexList[i]]
551 612 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
552 613 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
553 614 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
554 615 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
555 616 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
556 617
557 618 if dataOut.beacon_heiIndexList:
558 619 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
559 620 else:
560 621 phase_beacon[i] = numpy.average(phase)
561 622
562 623 if not self.isConfig:
563 624
564 625 nplots = len(pairsIndexList)
565 626
566 627 self.setup(id=id,
567 628 nplots=nplots,
568 629 wintitle=wintitle,
569 630 showprofile=showprofile,
570 631 show=show)
571 632
572 633 if timerange != None:
573 634 self.timerange = timerange
574 635
575 636 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
576 637
577 638 if ymin == None: ymin = 0
578 639 if ymax == None: ymax = 360
579 640
580 641 self.FTP_WEI = ftp_wei
581 642 self.EXP_CODE = exp_code
582 643 self.SUB_EXP_CODE = sub_exp_code
583 644 self.PLOT_POS = plot_pos
584 645
585 646 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
586 647 self.isConfig = True
587 648 self.figfile = figfile
588 649 self.xdata = numpy.array([])
589 650 self.ydata = numpy.array([])
590 651
591 652 update_figfile = True
592 653
593 654 #open file beacon phase
594 655 path = '%s%03d' %(self.PREFIX, self.id)
595 656 beacon_file = os.path.join(path,'%s.txt'%self.name)
596 657 self.filename_phase = os.path.join(figpath,beacon_file)
597 658 #self.save_phase(self.filename_phase)
598 659
599 660
600 661 #store data beacon phase
601 662 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
602 663
603 664 self.setWinTitle(title)
604 665
605 666
606 667 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
607 668
608 669 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
609 670
610 671 axes = self.axesList[0]
611 672
612 673 self.xdata = numpy.hstack((self.xdata, x[0:1]))
613 674
614 675 if len(self.ydata)==0:
615 676 self.ydata = phase_beacon.reshape(-1,1)
616 677 else:
617 678 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
618 679
619 680
620 681 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
621 682 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
622 683 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
623 684 XAxisAsTime=True, grid='both'
624 685 )
625 686
626 687 self.draw()
627 688
628 689 if dataOut.ltctime >= self.xmax:
629 690 self.counter_imagwr = wr_period
630 691 self.isConfig = False
631 692 update_figfile = True
632 693
633 694 self.save(figpath=figpath,
634 695 figfile=figfile,
635 696 save=save,
636 697 ftp=ftp,
637 698 wr_period=wr_period,
638 699 thisDatetime=thisDatetime,
639 700 update_figfile=update_figfile)
640 701
641 702 return dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,297 +1,302
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9
10 10 from schainpy.model.graphics.jroplot_base import Plot, plt
11 11
12 12
13 13 class ScopePlot(Plot):
14 14
15 15 '''
16 16 Plot for Scope
17 17 '''
18 18
19 19 CODE = 'scope'
20 20 plot_type = 'scatter'
21 21
22 22 def setup(self):
23 23
24 24 self.xaxis = 'Range (Km)'
25 25 self.ncols = 1
26 26 self.nrows = 1
27 27 self.nplots = 1
28 28 self.ylabel = 'Intensity [dB]'
29 29 self.titles = ['Scope']
30 30 self.colorbar = False
31 31 self.width = 6
32 32 self.height = 4
33 33
34 def update(self, dataOut):
35
36 data = {}
37 meta = {
38 'nProfiles': dataOut.nProfiles,
39 'flagDataAsBlock': dataOut.flagDataAsBlock,
40 'profileIndex': dataOut.profileIndex,
41 }
42 if self.CODE == 'scope':
43 data[self.CODE] = dataOut.data
44 elif self.CODE == 'pp_power':
45 data[self.CODE] = dataOut.dataPP_POWER
46 elif self.CODE == 'pp_signal':
47 data[self.CODE] = dataOut.dataPP_POW
48 elif self.CODE == 'pp_velocity':
49 data[self.CODE] = dataOut.dataPP_DOP
50 elif self.CODE == 'pp_specwidth':
51 data[self.CODE] = dataOut.dataPP_WIDTH
52
53 return data, meta
54
34 55 def plot_iq(self, x, y, channelIndexList, thisDatetime, wintitle):
35 56
36 57 yreal = y[channelIndexList,:].real
37 58 yimag = y[channelIndexList,:].imag
38 59 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y"))
39 60 self.xlabel = "Range (Km)"
40 61 self.ylabel = "Intensity - IQ"
41 62
42 63 self.y = yreal
43 64 self.x = x
44 self.xmin = min(x)
45 self.xmax = max(x)
46
47 65
48 66 self.titles[0] = title
49 67
50 68 for i,ax in enumerate(self.axes):
51 69 title = "Channel %d" %(i)
52 70 if ax.firsttime:
71 self.xmin = min(x)
72 self.xmax = max(x)
53 73 ax.plt_r = ax.plot(x, yreal[i,:], color='b')[0]
54 74 ax.plt_i = ax.plot(x, yimag[i,:], color='r')[0]
55 75 else:
56 76 ax.plt_r.set_data(x, yreal[i,:])
57 77 ax.plt_i.set_data(x, yimag[i,:])
58 78
59 79 def plot_power(self, x, y, channelIndexList, thisDatetime, wintitle):
60 80 y = y[channelIndexList,:] * numpy.conjugate(y[channelIndexList,:])
61 81 yreal = y.real
62 82 yreal = 10*numpy.log10(yreal)
63 83 self.y = yreal
64 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y"))
84 title = wintitle + " Power: %s" %(thisDatetime.strftime("%d-%b-%Y"))
65 85 self.xlabel = "Range (Km)"
66 self.ylabel = "Intensity"
67 self.xmin = min(x)
68 self.xmax = max(x)
86 self.ylabel = "Intensity [dB]"
69 87
70 88
71 89 self.titles[0] = title
72 90
73 91 for i,ax in enumerate(self.axes):
74 92 title = "Channel %d" %(i)
75
76 93 ychannel = yreal[i,:]
77 94
78 95 if ax.firsttime:
96 self.xmin = min(x)
97 self.xmax = max(x)
79 98 ax.plt_r = ax.plot(x, ychannel)[0]
80 99 else:
81 #pass
82 100 ax.plt_r.set_data(x, ychannel)
83 101
84 102 def plot_weatherpower(self, x, y, channelIndexList, thisDatetime, wintitle):
85 103
86 104
87 105 y = y[channelIndexList,:]
88 106 yreal = y.real
89 107 yreal = 10*numpy.log10(yreal)
90 108 self.y = yreal
91 109 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
92 110 self.xlabel = "Range (Km)"
93 111 self.ylabel = "Intensity"
94 112 self.xmin = min(x)
95 113 self.xmax = max(x)
96 114
97 115 self.titles[0] =title
98 116 for i,ax in enumerate(self.axes):
99 117 title = "Channel %d" %(i)
100 118
101 119 ychannel = yreal[i,:]
102 120
103 121 if ax.firsttime:
104 122 ax.plt_r = ax.plot(x, ychannel)[0]
105 123 else:
106 124 #pass
107 125 ax.plt_r.set_data(x, ychannel)
108 126
109 127 def plot_weathervelocity(self, x, y, channelIndexList, thisDatetime, wintitle):
110 128
111 129 x = x[channelIndexList,:]
112 130 yreal = y
113 131 self.y = yreal
114 132 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
115 133 self.xlabel = "Velocity (m/s)"
116 134 self.ylabel = "Range (Km)"
117 135 self.xmin = numpy.min(x)
118 136 self.xmax = numpy.max(x)
119 137 self.titles[0] =title
120 138 for i,ax in enumerate(self.axes):
121 139 title = "Channel %d" %(i)
122 140 xchannel = x[i,:]
123 141 if ax.firsttime:
124 142 ax.plt_r = ax.plot(xchannel, yreal)[0]
125 143 else:
126 144 #pass
127 145 ax.plt_r.set_data(xchannel, yreal)
128 146
129 147 def plot_weatherspecwidth(self, x, y, channelIndexList, thisDatetime, wintitle):
130 148
131 149 x = x[channelIndexList,:]
132 150 yreal = y
133 151 self.y = yreal
134 152 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
135 153 self.xlabel = "width "
136 154 self.ylabel = "Range (Km)"
137 155 self.xmin = numpy.min(x)
138 156 self.xmax = numpy.max(x)
139 157 self.titles[0] =title
140 158 for i,ax in enumerate(self.axes):
141 159 title = "Channel %d" %(i)
142 160 xchannel = x[i,:]
143 161 if ax.firsttime:
144 162 ax.plt_r = ax.plot(xchannel, yreal)[0]
145 163 else:
146 164 #pass
147 165 ax.plt_r.set_data(xchannel, yreal)
148 166
149 167 def plot(self):
150 168 if self.channels:
151 169 channels = self.channels
152 170 else:
153 171 channels = self.data.channels
154 172
155 173 thisDatetime = datetime.datetime.utcfromtimestamp(self.data.times[-1])
156 if self.CODE == "pp_power":
157 scope = self.data['pp_power']
158 elif self.CODE == "pp_signal":
159 scope = self.data["pp_signal"]
160 elif self.CODE == "pp_velocity":
161 scope = self.data["pp_velocity"]
162 elif self.CODE == "pp_specwidth":
163 scope = self.data["pp_specwidth"]
164 else:
165 scope =self.data["scope"]
174
175 scope = self.data[-1][self.CODE]
166 176
167 177 if self.data.flagDataAsBlock:
168 178
169 179 for i in range(self.data.nProfiles):
170 180
171 181 wintitle1 = " [Profile = %d] " %i
172 182 if self.CODE =="scope":
173 183 if self.type == "power":
174 self.plot_power(self.data.heights,
184 self.plot_power(self.data.yrange,
175 185 scope[:,i,:],
176 186 channels,
177 187 thisDatetime,
178 188 wintitle1
179 189 )
180 190
181 191 if self.type == "iq":
182 self.plot_iq(self.data.heights,
192 self.plot_iq(self.data.yrange,
183 193 scope[:,i,:],
184 194 channels,
185 195 thisDatetime,
186 196 wintitle1
187 197 )
188 198 if self.CODE=="pp_power":
189 self.plot_weatherpower(self.data.heights,
199 self.plot_weatherpower(self.data.yrange,
190 200 scope[:,i,:],
191 201 channels,
192 202 thisDatetime,
193 203 wintitle
194 204 )
195 205 if self.CODE=="pp_signal":
196 self.plot_weatherpower(self.data.heights,
206 self.plot_weatherpower(self.data.yrange,
197 207 scope[:,i,:],
198 208 channels,
199 209 thisDatetime,
200 210 wintitle
201 211 )
202 212 if self.CODE=="pp_velocity":
203 213 self.plot_weathervelocity(scope[:,i,:],
204 self.data.heights,
214 self.data.yrange,
205 215 channels,
206 216 thisDatetime,
207 217 wintitle
208 218 )
209 219 if self.CODE=="pp_spcwidth":
210 220 self.plot_weatherspecwidth(scope[:,i,:],
211 self.data.heights,
221 self.data.yrange,
212 222 channels,
213 223 thisDatetime,
214 224 wintitle
215 225 )
216 226 else:
217 227 wintitle = " [Profile = %d] " %self.data.profileIndex
218 228 if self.CODE== "scope":
219 229 if self.type == "power":
220 self.plot_power(self.data.heights,
230 self.plot_power(self.data.yrange,
221 231 scope,
222 232 channels,
223 233 thisDatetime,
224 234 wintitle
225 235 )
226 236
227 237 if self.type == "iq":
228 self.plot_iq(self.data.heights,
238 self.plot_iq(self.data.yrange,
229 239 scope,
230 240 channels,
231 241 thisDatetime,
232 242 wintitle
233 243 )
234 244 if self.CODE=="pp_power":
235 self.plot_weatherpower(self.data.heights,
245 self.plot_weatherpower(self.data.yrange,
236 246 scope,
237 247 channels,
238 248 thisDatetime,
239 249 wintitle
240 250 )
241 251 if self.CODE=="pp_signal":
242 self.plot_weatherpower(self.data.heights,
252 self.plot_weatherpower(self.data.yrange,
243 253 scope,
244 254 channels,
245 255 thisDatetime,
246 256 wintitle
247 257 )
248 258 if self.CODE=="pp_velocity":
249 259 self.plot_weathervelocity(scope,
250 self.data.heights,
260 self.data.yrange,
251 261 channels,
252 262 thisDatetime,
253 263 wintitle
254 264 )
255 265 if self.CODE=="pp_specwidth":
256 266 self.plot_weatherspecwidth(scope,
257 self.data.heights,
267 self.data.yrange,
258 268 channels,
259 269 thisDatetime,
260 270 wintitle
261 271 )
262 272
263 273
264
265 274 class PulsepairPowerPlot(ScopePlot):
266 275 '''
267 276 Plot for P= S+N
268 277 '''
269 278
270 279 CODE = 'pp_power'
271 280 plot_type = 'scatter'
272 buffering = False
273 281
274 282 class PulsepairVelocityPlot(ScopePlot):
275 283 '''
276 284 Plot for VELOCITY
277 285 '''
278 286 CODE = 'pp_velocity'
279 287 plot_type = 'scatter'
280 buffering = False
281 288
282 289 class PulsepairSpecwidthPlot(ScopePlot):
283 290 '''
284 291 Plot for WIDTH
285 292 '''
286 293 CODE = 'pp_specwidth'
287 294 plot_type = 'scatter'
288 buffering = False
289 295
290 296 class PulsepairSignalPlot(ScopePlot):
291 297 '''
292 298 Plot for S
293 299 '''
294 300
295 301 CODE = 'pp_signal'
296 302 plot_type = 'scatter'
297 buffering = False
Show file before | Show file after | Hide comments
@@ -1,355 +1,355
1 1 '''
2 2 Created on Nov 9, 2016
3 3
4 4 @author: roj- LouVD
5 5 '''
6 6
7 7
8 8 import os
9 9 import sys
10 10 import time
11 11 import glob
12 12 import datetime
13 13
14 14 import numpy
15 15
16 16 import schainpy.admin
17 17 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator
18 18 from schainpy.model.data.jrodata import Parameters
19 19 from schainpy.model.io.jroIO_base import Reader
20 20 from schainpy.utils import log
21 21
22 22 FILE_HEADER_STRUCTURE = numpy.dtype([
23 23 ('FMN', '<u4'),
24 24 ('nrec', '<u4'),
25 25 ('fr_offset', '<u4'),
26 26 ('id', '<u4'),
27 27 ('site', 'u1', (32,))
28 28 ])
29 29
30 30 REC_HEADER_STRUCTURE = numpy.dtype([
31 31 ('rmn', '<u4'),
32 32 ('rcounter', '<u4'),
33 33 ('nr_offset', '<u4'),
34 34 ('tr_offset', '<u4'),
35 35 ('time', '<u4'),
36 36 ('time_msec', '<u4'),
37 37 ('tag', 'u1', (32,)),
38 38 ('comments', 'u1', (32,)),
39 39 ('lat', '<f4'),
40 40 ('lon', '<f4'),
41 41 ('gps_status', '<u4'),
42 42 ('freq', '<u4'),
43 43 ('freq0', '<u4'),
44 44 ('nchan', '<u4'),
45 45 ('delta_r', '<u4'),
46 46 ('nranges', '<u4'),
47 47 ('r0', '<u4'),
48 48 ('prf', '<u4'),
49 49 ('ncoh', '<u4'),
50 50 ('npoints', '<u4'),
51 51 ('polarization', '<i4'),
52 52 ('rx_filter', '<u4'),
53 53 ('nmodes', '<u4'),
54 54 ('dmode_index', '<u4'),
55 55 ('dmode_rngcorr', '<u4'),
56 56 ('nrxs', '<u4'),
57 57 ('acf_length', '<u4'),
58 58 ('acf_lags', '<u4'),
59 59 ('sea_to_atmos', '<f4'),
60 60 ('sea_notch', '<u4'),
61 61 ('lh_sea', '<u4'),
62 62 ('hh_sea', '<u4'),
63 63 ('nbins_sea', '<u4'),
64 64 ('min_snr', '<f4'),
65 65 ('min_cc', '<f4'),
66 66 ('max_time_diff', '<f4')
67 67 ])
68 68
69 69 DATA_STRUCTURE = numpy.dtype([
70 70 ('range', '<u4'),
71 71 ('status', '<u4'),
72 72 ('zonal', '<f4'),
73 73 ('meridional', '<f4'),
74 74 ('vertical', '<f4'),
75 75 ('zonal_a', '<f4'),
76 76 ('meridional_a', '<f4'),
77 77 ('corrected_fading', '<f4'), # seconds
78 78 ('uncorrected_fading', '<f4'), # seconds
79 79 ('time_diff', '<f4'),
80 80 ('major_axis', '<f4'),
81 81 ('axial_ratio', '<f4'),
82 82 ('orientation', '<f4'),
83 83 ('sea_power', '<u4'),
84 84 ('sea_algorithm', '<u4')
85 85 ])
86 86
87 87
88 88 class BLTRParamReader(Reader, ProcessingUnit):
89 89 '''
90 90 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR
91 91 from *.sswma files
92 92 '''
93 93
94 94 ext = '.sswma'
95 95
96 96 def __init__(self):
97 97
98 98 ProcessingUnit.__init__(self)
99 99
100 100 self.dataOut = Parameters()
101 101 self.dataOut.timezone = 300
102 102 self.counter_records = 0
103 103 self.flagNoMoreFiles = 0
104 104 self.isConfig = False
105 105 self.filename = None
106 106 self.status_value = 0
107 107 self.datatime = datetime.datetime(1900,1,1)
108 108 self.filefmt = "*********%Y%m%d******"
109 109
110 110 def setup(self, **kwargs):
111 111
112 112 self.set_kwargs(**kwargs)
113 113
114 114 if self.path is None:
115 115 raise ValueError("The path is not valid")
116 116
117 117 if self.online:
118 118 log.log("Searching files in online mode...", self.name)
119 119
120 120 for nTries in range(self.nTries):
121 121 fullpath = self.searchFilesOnLine(self.path, self.startDate,
122 122 self.endDate, self.expLabel, self.ext, self.walk,
123 123 self.filefmt, self.folderfmt)
124 124 try:
125 125 fullpath = next(fullpath)
126 126 except:
127 127 fullpath = None
128 128
129 129 if fullpath:
130 130 self.fileSize = os.path.getsize(fullpath)
131 131 self.filename = fullpath
132 132 self.flagIsNewFile = 1
133 133 if self.fp != None:
134 134 self.fp.close()
135 135 self.fp = self.open_file(fullpath, self.open_mode)
136 136 self.flagNoMoreFiles = 0
137 137 break
138 138
139 139 log.warning(
140 140 'Waiting {} sec for a valid file in {}: try {} ...'.format(
141 141 self.delay, self.path, nTries + 1),
142 142 self.name)
143 143 time.sleep(self.delay)
144 144
145 145 if not(fullpath):
146 146 raise schainpy.admin.SchainError(
147 147 'There isn\'t any valid file in {}'.format(self.path))
148 148 self.readFirstHeader()
149 149 else:
150 150 log.log("Searching files in {}".format(self.path), self.name)
151 151 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
152 152 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
153 153 self.setNextFile()
154 154
155 155 def checkForRealPath(self, nextFile, nextDay):
156 156 '''
157 157 '''
158 158
159 159 dt = self.datatime + datetime.timedelta(1)
160 160 filename = '{}.{}{}'.format(self.siteFile, dt.strftime('%Y%m%d'), self.ext)
161 161 fullfilename = os.path.join(self.path, filename)
162 162 if os.path.exists(fullfilename):
163 163 return fullfilename, filename
164 164 return None, filename
165 165
166 166
167 167 def readFirstHeader(self):
168 168 '''
169 169 '''
170 170
171 171 # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
172 172 self.siteFile = self.filename.split('/')[-1].split('.')[0]
173 173 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
174 174 self.nrecords = self.header_file['nrec'][0]
175 175 self.counter_records = 0
176 176 self.flagIsNewFile = 0
177 177 self.fileIndex += 1
178 178
179 179 def readNextBlock(self):
180 180
181 181 while True:
182 182 if not self.online and self.counter_records == self.nrecords:
183 183 self.flagIsNewFile = 1
184 184 if not self.setNextFile():
185 185 return 0
186 186 try:
187 187 pointer = self.fp.tell()
188 188 self.readBlock()
189 189 except:
190 190 if self.online and self.waitDataBlock(pointer, 38512) == 1:
191 191 continue
192 192 else:
193 193 if not self.setNextFile():
194 194 return 0
195 195
196 196 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
197 197 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
198 198 log.warning(
199 199 'Reading Record No. {}/{} -> {} [Skipping]'.format(
200 200 self.counter_records,
201 201 self.nrecords,
202 202 self.datatime.ctime()),
203 203 'BLTRParamReader')
204 204 continue
205 205 break
206 206
207 207 log.log('Reading Record No. {} -> {}'.format(
208 208 self.counter_records,
209 209 self.datatime.ctime()), 'BLTRParamReader')
210 210
211 211 return 1
212 212
213 213 def readBlock(self):
214 214
215 215 pointer = self.fp.tell()
216 216 header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
217 217 self.nchannels = int(header_rec['nchan'][0] / 2)
218 218 self.kchan = header_rec['nrxs'][0]
219 219 self.nmodes = header_rec['nmodes'][0]
220 220 self.nranges = header_rec['nranges'][0]
221 221 self.fp.seek(pointer)
222 222 self.height = numpy.empty((self.nmodes, self.nranges))
223 223 self.snr = numpy.empty((self.nmodes, int(self.nchannels), self.nranges))
224 224 self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
225 225 self.flagDiscontinuousBlock = 0
226 226
227 227 for mode in range(self.nmodes):
228 228 self.readHeader()
229 229 data = self.readData()
230 230 self.height[mode] = (data[0] - self.correction) / 1000.
231 231 self.buffer[mode] = data[1]
232 232 self.snr[mode] = data[2]
233 233
234 234 self.counter_records = self.counter_records + self.nmodes
235 235
236 236 return
237 237
238 238 def readHeader(self):
239 239 '''
240 240 RecordHeader of BLTR rawdata file
241 241 '''
242 242
243 243 header_structure = numpy.dtype(
244 244 REC_HEADER_STRUCTURE.descr + [
245 245 ('antenna_coord', 'f4', (2, int(self.nchannels))),
246 246 ('rx_gains', 'u4', (int(self.nchannels),)),
247 247 ('rx_analysis', 'u4', (int(self.nchannels),))
248 248 ]
249 249 )
250 250
251 251 self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
252 252 self.lat = self.header_rec['lat'][0]
253 253 self.lon = self.header_rec['lon'][0]
254 254 self.delta = self.header_rec['delta_r'][0]
255 255 self.correction = self.header_rec['dmode_rngcorr'][0]
256 256 self.imode = self.header_rec['dmode_index'][0]
257 257 self.antenna = self.header_rec['antenna_coord']
258 258 self.rx_gains = self.header_rec['rx_gains']
259 259 self.time = self.header_rec['time'][0]
260 260 dt = datetime.datetime.utcfromtimestamp(self.time)
261 261 if dt.date()>self.datatime.date():
262 262 self.flagDiscontinuousBlock = 1
263 263 self.datatime = dt
264 264
265 265 def readData(self):
266 266 '''
267 267 Reading and filtering data block record of BLTR rawdata file,
268 268 filtering is according to status_value.
269 269
270 270 Input:
271 271 status_value - Array data is set to NAN for values that are not
272 272 equal to status_value
273 273
274 274 '''
275 275 self.nchannels = int(self.nchannels)
276 276
277 277 data_structure = numpy.dtype(
278 278 DATA_STRUCTURE.descr + [
279 279 ('rx_saturation', 'u4', (self.nchannels,)),
280 280 ('chan_offset', 'u4', (2 * self.nchannels,)),
281 281 ('rx_amp', 'u4', (self.nchannels,)),
282 282 ('rx_snr', 'f4', (self.nchannels,)),
283 283 ('cross_snr', 'f4', (self.kchan,)),
284 284 ('sea_power_relative', 'f4', (self.kchan,))]
285 285 )
286 286
287 287 data = numpy.fromfile(self.fp, data_structure, self.nranges)
288 288
289 289 height = data['range']
290 290 winds = numpy.array(
291 291 (data['zonal'], data['meridional'], data['vertical']))
292 292 snr = data['rx_snr'].T
293 293
294 294 winds[numpy.where(winds == -9999.)] = numpy.nan
295 295 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
296 296 snr[numpy.where(snr == -9999.)] = numpy.nan
297 297 snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
298 298 snr = numpy.power(10, snr / 10)
299 299
300 300 return height, winds, snr
301 301
302 302 def set_output(self):
303 303 '''
304 304 Storing data from databuffer to dataOut object
305 305 '''
306 306
307 self.dataOut.data_SNR = self.snr
307 self.dataOut.data_snr = self.snr
308 308 self.dataOut.height = self.height
309 309 self.dataOut.data = self.buffer
310 310 self.dataOut.utctimeInit = self.time
311 311 self.dataOut.utctime = self.dataOut.utctimeInit
312 312 self.dataOut.useLocalTime = False
313 313 self.dataOut.paramInterval = 157
314 314 self.dataOut.site = self.siteFile
315 315 self.dataOut.nrecords = self.nrecords / self.nmodes
316 316 self.dataOut.lat = self.lat
317 317 self.dataOut.lon = self.lon
318 318 self.dataOut.channelList = list(range(self.nchannels))
319 319 self.dataOut.kchan = self.kchan
320 320 self.dataOut.delta = self.delta
321 321 self.dataOut.correction = self.correction
322 322 self.dataOut.nmodes = self.nmodes
323 323 self.dataOut.imode = self.imode
324 324 self.dataOut.antenna = self.antenna
325 325 self.dataOut.rx_gains = self.rx_gains
326 326 self.dataOut.flagNoData = False
327 327 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
328 328
329 329 def getData(self):
330 330 '''
331 331 Storing data from databuffer to dataOut object
332 332 '''
333 333 if self.flagNoMoreFiles:
334 334 self.dataOut.flagNoData = True
335 335 return 0
336 336
337 337 if not self.readNextBlock():
338 338 self.dataOut.flagNoData = True
339 339 return 0
340 340
341 341 self.set_output()
342 342
343 343 return 1
344 344
345 345 def run(self, **kwargs):
346 346 '''
347 347 '''
348 348
349 349 if not(self.isConfig):
350 350 self.setup(**kwargs)
351 351 self.isConfig = True
352 352
353 353 self.getData()
354 354
355 355 return No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,626 +1,627
1 1 import os
2 2 import time
3 3 import datetime
4 4
5 5 import numpy
6 6 import h5py
7 7
8 8 import schainpy.admin
9 9 from schainpy.model.data.jrodata import *
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 11 from schainpy.model.io.jroIO_base import *
12 12 from schainpy.utils import log
13 13
14 14
15 15 class HDFReader(Reader, ProcessingUnit):
16 16 """Processing unit to read HDF5 format files
17 17
18 18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 20 attributes.
21 21 It is possible to read any HDF5 file by given the structure in the `description`
22 22 parameter, also you can add extra values to metadata with the parameter `extras`.
23 23
24 24 Parameters:
25 25 -----------
26 26 path : str
27 27 Path where files are located.
28 28 startDate : date
29 29 Start date of the files
30 30 endDate : list
31 31 End date of the files
32 32 startTime : time
33 33 Start time of the files
34 34 endTime : time
35 35 End time of the files
36 36 description : dict, optional
37 37 Dictionary with the description of the HDF5 file
38 38 extras : dict, optional
39 39 Dictionary with extra metadata to be be added to `dataOut`
40 40
41 41 Examples
42 42 --------
43 43
44 44 desc = {
45 45 'Data': {
46 46 'data_output': ['u', 'v', 'w'],
47 47 'utctime': 'timestamps',
48 48 } ,
49 49 'Metadata': {
50 50 'heightList': 'heights'
51 51 }
52 52 }
53 53
54 54 desc = {
55 55 'Data': {
56 56 'data_output': 'winds',
57 57 'utctime': 'timestamps'
58 58 },
59 59 'Metadata': {
60 60 'heightList': 'heights'
61 61 }
62 62 }
63 63
64 64 extras = {
65 65 'timeZone': 300
66 66 }
67 67
68 68 reader = project.addReadUnit(
69 69 name='HDFReader',
70 70 path='/path/to/files',
71 71 startDate='2019/01/01',
72 72 endDate='2019/01/31',
73 73 startTime='00:00:00',
74 74 endTime='23:59:59',
75 75 # description=json.dumps(desc),
76 76 # extras=json.dumps(extras),
77 77 )
78 78
79 79 """
80 80
81 81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82 82
83 83 def __init__(self):
84 84 ProcessingUnit.__init__(self)
85 85 self.dataOut = Parameters()
86 86 self.ext = ".hdf5"
87 87 self.optchar = "D"
88 88 self.meta = {}
89 89 self.data = {}
90 90 self.open_file = h5py.File
91 91 self.open_mode = 'r'
92 92 self.description = {}
93 93 self.extras = {}
94 94 self.filefmt = "*%Y%j***"
95 95 self.folderfmt = "*%Y%j"
96 96
97 97 def setup(self, **kwargs):
98 98
99 99 self.set_kwargs(**kwargs)
100 100 if not self.ext.startswith('.'):
101 101 self.ext = '.{}'.format(self.ext)
102 102
103 103 if self.online:
104 104 log.log("Searching files in online mode...", self.name)
105 105
106 106 for nTries in range(self.nTries):
107 107 fullpath = self.searchFilesOnLine(self.path, self.startDate,
108 108 self.endDate, self.expLabel, self.ext, self.walk,
109 109 self.filefmt, self.folderfmt)
110 110 try:
111 111 fullpath = next(fullpath)
112 112 except:
113 113 fullpath = None
114 114
115 115 if fullpath:
116 116 break
117 117
118 118 log.warning(
119 119 'Waiting {} sec for a valid file in {}: try {} ...'.format(
120 120 self.delay, self.path, nTries + 1),
121 121 self.name)
122 122 time.sleep(self.delay)
123 123
124 124 if not(fullpath):
125 125 raise schainpy.admin.SchainError(
126 126 'There isn\'t any valid file in {}'.format(self.path))
127 127
128 128 pathname, filename = os.path.split(fullpath)
129 129 self.year = int(filename[1:5])
130 130 self.doy = int(filename[5:8])
131 131 self.set = int(filename[8:11]) - 1
132 132 else:
133 133 log.log("Searching files in {}".format(self.path), self.name)
134 134 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
135 135 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
136 136
137 137 self.setNextFile()
138 138
139 139 return
140 140
141 141 def readFirstHeader(self):
142 142 '''Read metadata and data'''
143 143
144 144 self.__readMetadata()
145 145 self.__readData()
146 146 self.__setBlockList()
147 147
148 148 if 'type' in self.meta:
149 149 self.dataOut = eval(self.meta['type'])()
150 150
151 151 for attr in self.meta:
152 152 setattr(self.dataOut, attr, self.meta[attr])
153 153
154 154 self.blockIndex = 0
155 155
156 156 return
157 157
158 158 def __setBlockList(self):
159 159 '''
160 160 Selects the data within the times defined
161 161
162 162 self.fp
163 163 self.startTime
164 164 self.endTime
165 165 self.blockList
166 166 self.blocksPerFile
167 167
168 168 '''
169 169
170 170 startTime = self.startTime
171 171 endTime = self.endTime
172 172
173 173 thisUtcTime = self.data['utctime']
174 174 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
175 175
176 176 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
177 177
178 178 thisDate = thisDatetime.date()
179 179 thisTime = thisDatetime.time()
180 180
181 181 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
182 182 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
183 183
184 184 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
185 185
186 186 self.blockList = ind
187 187 self.blocksPerFile = len(ind)
188 188 return
189 189
190 190 def __readMetadata(self):
191 191 '''
192 192 Reads Metadata
193 193 '''
194 194
195 195 meta = {}
196 196
197 197 if self.description:
198 198 for key, value in self.description['Metadata'].items():
199 199 meta[key] = self.fp[value].value
200 200 else:
201 201 grp = self.fp['Metadata']
202 202 for name in grp:
203 203 meta[name] = grp[name].value
204 204
205 205 if self.extras:
206 206 for key, value in self.extras.items():
207 207 meta[key] = value
208 208 self.meta = meta
209 209
210 210 return
211 211
212 212 def __readData(self):
213 213
214 214 data = {}
215 215
216 216 if self.description:
217 217 for key, value in self.description['Data'].items():
218 218 if isinstance(value, str):
219 219 if isinstance(self.fp[value], h5py.Dataset):
220 220 data[key] = self.fp[value].value
221 221 elif isinstance(self.fp[value], h5py.Group):
222 222 array = []
223 223 for ch in self.fp[value]:
224 224 array.append(self.fp[value][ch].value)
225 225 data[key] = numpy.array(array)
226 226 elif isinstance(value, list):
227 227 array = []
228 228 for ch in value:
229 229 array.append(self.fp[ch].value)
230 230 data[key] = numpy.array(array)
231 231 else:
232 232 grp = self.fp['Data']
233 233 for name in grp:
234 234 if isinstance(grp[name], h5py.Dataset):
235 235 array = grp[name].value
236 236 elif isinstance(grp[name], h5py.Group):
237 237 array = []
238 238 for ch in grp[name]:
239 239 array.append(grp[name][ch].value)
240 240 array = numpy.array(array)
241 241 else:
242 242 log.warning('Unknown type: {}'.format(name))
243 243
244 244 if name in self.description:
245 245 key = self.description[name]
246 246 else:
247 247 key = name
248 248 data[key] = array
249 249
250 250 self.data = data
251 251 return
252 252
253 253 def getData(self):
254 254
255 255 for attr in self.data:
256 256 if self.data[attr].ndim == 1:
257 257 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
258 258 else:
259 259 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
260 260
261 261 self.dataOut.flagNoData = False
262 262 self.blockIndex += 1
263 263
264 264 log.log("Block No. {}/{} -> {}".format(
265 265 self.blockIndex,
266 266 self.blocksPerFile,
267 267 self.dataOut.datatime.ctime()), self.name)
268 268
269 269 return
270 270
271 271 def run(self, **kwargs):
272 272
273 273 if not(self.isConfig):
274 274 self.setup(**kwargs)
275 275 self.isConfig = True
276 276
277 277 if self.blockIndex == self.blocksPerFile:
278 278 self.setNextFile()
279 279
280 280 self.getData()
281 281
282 282 return
283 283
284 284 @MPDecorator
285 285 class HDFWriter(Operation):
286 286 """Operation to write HDF5 files.
287 287
288 288 The HDF5 file contains by default two groups Data and Metadata where
289 289 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
290 290 parameters, data attributes are normaly time dependent where the metadata
291 291 are not.
292 292 It is possible to customize the structure of the HDF5 file with the
293 293 optional description parameter see the examples.
294 294
295 295 Parameters:
296 296 -----------
297 297 path : str
298 298 Path where files will be saved.
299 299 blocksPerFile : int
300 300 Number of blocks per file
301 301 metadataList : list
302 302 List of the dataOut attributes that will be saved as metadata
303 303 dataList : int
304 304 List of the dataOut attributes that will be saved as data
305 305 setType : bool
306 306 If True the name of the files corresponds to the timestamp of the data
307 307 description : dict, optional
308 308 Dictionary with the desired description of the HDF5 file
309 309
310 310 Examples
311 311 --------
312 312
313 313 desc = {
314 314 'data_output': {'winds': ['z', 'w', 'v']},
315 315 'utctime': 'timestamps',
316 316 'heightList': 'heights'
317 317 }
318 318 desc = {
319 319 'data_output': ['z', 'w', 'v'],
320 320 'utctime': 'timestamps',
321 321 'heightList': 'heights'
322 322 }
323 323 desc = {
324 324 'Data': {
325 325 'data_output': 'winds',
326 326 'utctime': 'timestamps'
327 327 },
328 328 'Metadata': {
329 329 'heightList': 'heights'
330 330 }
331 331 }
332 332
333 333 writer = proc_unit.addOperation(name='HDFWriter')
334 334 writer.addParameter(name='path', value='/path/to/file')
335 335 writer.addParameter(name='blocksPerFile', value='32')
336 336 writer.addParameter(name='metadataList', value='heightList,timeZone')
337 337 writer.addParameter(name='dataList',value='data_output,utctime')
338 338 # writer.addParameter(name='description',value=json.dumps(desc))
339 339
340 340 """
341 341
342 342 ext = ".hdf5"
343 343 optchar = "D"
344 344 filename = None
345 345 path = None
346 346 setFile = None
347 347 fp = None
348 348 firsttime = True
349 349 #Configurations
350 350 blocksPerFile = None
351 351 blockIndex = None
352 352 dataOut = None
353 353 #Data Arrays
354 354 dataList = None
355 355 metadataList = None
356 356 currentDay = None
357 357 lastTime = None
358 358
359 359 def __init__(self):
360 360
361 361 Operation.__init__(self)
362 362 return
363 363
364 364 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
365 365 self.path = path
366 366 self.blocksPerFile = blocksPerFile
367 367 self.metadataList = metadataList
368 368 self.dataList = [s.strip() for s in dataList]
369 369 self.setType = setType
370 370 self.description = description
371 371
372 372 if self.metadataList is None:
373 373 self.metadataList = self.dataOut.metadata_list
374 374
375 375 tableList = []
376 376 dsList = []
377 377
378 378 for i in range(len(self.dataList)):
379 379 dsDict = {}
380 380 if hasattr(self.dataOut, self.dataList[i]):
381 381 dataAux = getattr(self.dataOut, self.dataList[i])
382 382 dsDict['variable'] = self.dataList[i]
383 383 else:
384 384 log.warning('Attribute {} not found in dataOut', self.name)
385 385 continue
386 386
387 387 if dataAux is None:
388 388 continue
389 389 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
390 390 dsDict['nDim'] = 0
391 391 else:
392 392 dsDict['nDim'] = len(dataAux.shape)
393 393 dsDict['shape'] = dataAux.shape
394 394 dsDict['dsNumber'] = dataAux.shape[0]
395 395 dsDict['dtype'] = dataAux.dtype
396 396
397 397 dsList.append(dsDict)
398 398
399 399 self.dsList = dsList
400 400 self.currentDay = self.dataOut.datatime.date()
401 401
402 402 def timeFlag(self):
403 403 currentTime = self.dataOut.utctime
404 404 timeTuple = time.localtime(currentTime)
405 405 dataDay = timeTuple.tm_yday
406 406
407 407 if self.lastTime is None:
408 408 self.lastTime = currentTime
409 409 self.currentDay = dataDay
410 410 return False
411 411
412 412 timeDiff = currentTime - self.lastTime
413 413
414 414 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
415 415 if dataDay != self.currentDay:
416 416 self.currentDay = dataDay
417 417 return True
418 418 elif timeDiff > 3*60*60:
419 419 self.lastTime = currentTime
420 420 return True
421 421 else:
422 422 self.lastTime = currentTime
423 423 return False
424 424
425 425 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
426 426 dataList=[], setType=None, description={}):
427 427
428 428 self.dataOut = dataOut
429 429 if not(self.isConfig):
430 430 self.setup(path=path, blocksPerFile=blocksPerFile,
431 431 metadataList=metadataList, dataList=dataList,
432 432 setType=setType, description=description)
433 433
434 434 self.isConfig = True
435 435 self.setNextFile()
436 436
437 437 self.putData()
438 438 return
439 439
440 440 def setNextFile(self):
441 441
442 442 ext = self.ext
443 443 path = self.path
444 444 setFile = self.setFile
445 445
446 446 timeTuple = time.localtime(self.dataOut.utctime)
447 447 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
448 448 fullpath = os.path.join(path, subfolder)
449 449
450 450 if os.path.exists(fullpath):
451 451 filesList = os.listdir(fullpath)
452 452 filesList = [k for k in filesList if k.startswith(self.optchar)]
453 453 if len( filesList ) > 0:
454 454 filesList = sorted(filesList, key=str.lower)
455 455 filen = filesList[-1]
456 456 # el filename debera tener el siguiente formato
457 457 # 0 1234 567 89A BCDE (hex)
458 458 # x YYYY DDD SSS .ext
459 459 if isNumber(filen[8:11]):
460 460 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
461 461 else:
462 462 setFile = -1
463 463 else:
464 464 setFile = -1 #inicializo mi contador de seteo
465 465 else:
466 466 os.makedirs(fullpath)
467 467 setFile = -1 #inicializo mi contador de seteo
468 468
469 469 if self.setType is None:
470 470 setFile += 1
471 471 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
472 472 timeTuple.tm_year,
473 473 timeTuple.tm_yday,
474 474 setFile,
475 475 ext )
476 476 else:
477 477 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
478 478 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
479 479 timeTuple.tm_year,
480 480 timeTuple.tm_yday,
481 481 setFile,
482 482 ext )
483 483
484 484 self.filename = os.path.join( path, subfolder, file )
485 485
486 486 #Setting HDF5 File
487 487 self.fp = h5py.File(self.filename, 'w')
488 488 #write metadata
489 489 self.writeMetadata(self.fp)
490 490 #Write data
491 491 self.writeData(self.fp)
492 492
493 493 def getLabel(self, name, x=None):
494 494
495 495 if x is None:
496 496 if 'Data' in self.description:
497 497 data = self.description['Data']
498 498 if 'Metadata' in self.description:
499 499 data.update(self.description['Metadata'])
500 500 else:
501 501 data = self.description
502 502 if name in data:
503 503 if isinstance(data[name], str):
504 504 return data[name]
505 505 elif isinstance(data[name], list):
506 506 return None
507 507 elif isinstance(data[name], dict):
508 508 for key, value in data[name].items():
509 509 return key
510 510 return name
511 511 else:
512 512 if 'Metadata' in self.description:
513 513 meta = self.description['Metadata']
514 514 else:
515 515 meta = self.description
516 516 if name in meta:
517 517 if isinstance(meta[name], list):
518 518 return meta[name][x]
519 519 elif isinstance(meta[name], dict):
520 520 for key, value in meta[name].items():
521 521 return value[x]
522 522 if 'cspc' in name:
523 523 return 'pair{:02d}'.format(x)
524 524 else:
525 525 return 'channel{:02d}'.format(x)
526 526
527 527 def writeMetadata(self, fp):
528 528
529 529 if self.description:
530 530 if 'Metadata' in self.description:
531 531 grp = fp.create_group('Metadata')
532 532 else:
533 533 grp = fp
534 534 else:
535 535 grp = fp.create_group('Metadata')
536 536
537 537 for i in range(len(self.metadataList)):
538 538 if not hasattr(self.dataOut, self.metadataList[i]):
539 539 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
540 540 continue
541 541 value = getattr(self.dataOut, self.metadataList[i])
542 542 if isinstance(value, bool):
543 543 if value is True:
544 544 value = 1
545 545 else:
546 546 value = 0
547 547 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
548 548 return
549 549
550 550 def writeData(self, fp):
551 551
552 552 if self.description:
553 553 if 'Data' in self.description:
554 554 grp = fp.create_group('Data')
555 555 else:
556 556 grp = fp
557 557 else:
558 558 grp = fp.create_group('Data')
559 559
560 560 dtsets = []
561 561 data = []
562 562
563 563 for dsInfo in self.dsList:
564 564 if dsInfo['nDim'] == 0:
565 565 ds = grp.create_dataset(
566 566 self.getLabel(dsInfo['variable']),
567 567 (self.blocksPerFile, ),
568 568 chunks=True,
569 569 dtype=numpy.float64)
570 570 dtsets.append(ds)
571 571 data.append((dsInfo['variable'], -1))
572 572 else:
573 573 label = self.getLabel(dsInfo['variable'])
574 574 if label is not None:
575 575 sgrp = grp.create_group(label)
576 576 else:
577 577 sgrp = grp
578 578 for i in range(dsInfo['dsNumber']):
579 579 ds = sgrp.create_dataset(
580 580 self.getLabel(dsInfo['variable'], i),
581 581 (self.blocksPerFile, ) + dsInfo['shape'][1:],
582 582 chunks=True,
583 583 dtype=dsInfo['dtype'])
584 584 dtsets.append(ds)
585 585 data.append((dsInfo['variable'], i))
586 586 fp.flush()
587 587
588 588 log.log('Creating file: {}'.format(fp.filename), self.name)
589 589
590 590 self.ds = dtsets
591 591 self.data = data
592 592 self.firsttime = True
593 593 self.blockIndex = 0
594 594 return
595 595
596 596 def putData(self):
597 597
598 598 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
599 599 self.closeFile()
600 600 self.setNextFile()
601 601
602 602 for i, ds in enumerate(self.ds):
603 603 attr, ch = self.data[i]
604 604 if ch == -1:
605 605 ds[self.blockIndex] = getattr(self.dataOut, attr)
606 606 else:
607 607 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
608 608
609 609 self.fp.flush()
610 610 self.blockIndex += 1
611 611 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
612 612
613 613 return
614 614
615 615 def closeFile(self):
616 616
617 617 if self.blockIndex != self.blocksPerFile:
618 618 for ds in self.ds:
619 619 ds.resize(self.blockIndex, axis=0)
620 620
621 self.fp.flush()
622 self.fp.close()
621 if self.fp:
622 self.fp.flush()
623 self.fp.close()
623 624
624 625 def close(self):
625 626
626 627 self.closeFile()
Show file before | Show file after | Hide comments
@@ -1,343 +1,343
1 1 '''
2 2 Created on Set 10, 2017
3 3
4 4 @author: Juan C. Espinoza
5 5 '''
6 6
7 7
8 8 import os
9 9 import sys
10 10 import time
11 11 import glob
12 12 import datetime
13 13
14 14 import numpy
15 15
16 16 from schainpy.model.proc.jroproc_base import ProcessingUnit
17 17 from schainpy.model.data.jrodata import Parameters
18 18 from schainpy.model.io.jroIO_base import JRODataReader, isNumber
19 19 from schainpy.utils import log
20 20
21 21 FILE_HEADER_STRUCTURE = numpy.dtype([
22 22 ('year', 'f'),
23 23 ('doy', 'f'),
24 24 ('nint', 'f'),
25 25 ('navg', 'f'),
26 26 ('fh', 'f'),
27 27 ('dh', 'f'),
28 28 ('nheights', 'f'),
29 29 ('ipp', 'f')
30 30 ])
31 31
32 32 REC_HEADER_STRUCTURE = numpy.dtype([
33 33 ('magic', 'f'),
34 34 ('hours', 'f'),
35 35 ('interval', 'f'),
36 36 ('h0', 'f'),
37 37 ('nheights', 'f'),
38 38 ('snr1', 'f'),
39 39 ('snr2', 'f'),
40 40 ('snr', 'f'),
41 41 ])
42 42
43 43 DATA_STRUCTURE = numpy.dtype([
44 44 ('range', '<u4'),
45 45 ('status', '<u4'),
46 46 ('zonal', '<f4'),
47 47 ('meridional', '<f4'),
48 48 ('vertical', '<f4'),
49 49 ('zonal_a', '<f4'),
50 50 ('meridional_a', '<f4'),
51 51 ('corrected_fading', '<f4'), # seconds
52 52 ('uncorrected_fading', '<f4'), # seconds
53 53 ('time_diff', '<f4'),
54 54 ('major_axis', '<f4'),
55 55 ('axial_ratio', '<f4'),
56 56 ('orientation', '<f4'),
57 57 ('sea_power', '<u4'),
58 58 ('sea_algorithm', '<u4')
59 59 ])
60 60
61 61
62 62 class JULIAParamReader(JRODataReader, ProcessingUnit):
63 63 '''
64 64 Julia data (eej, spf, 150km) *.dat files
65 65 '''
66 66
67 67 ext = '.dat'
68 68
69 69 def __init__(self, **kwargs):
70 70
71 71 ProcessingUnit.__init__(self, **kwargs)
72 72
73 73 self.dataOut = Parameters()
74 74 self.counter_records = 0
75 75 self.flagNoMoreFiles = 0
76 76 self.isConfig = False
77 77 self.filename = None
78 78 self.clockpulse = 0.15
79 79 self.kd = 213.6
80 80
81 81 def setup(self,
82 82 path=None,
83 83 startDate=None,
84 84 endDate=None,
85 85 ext=None,
86 86 startTime=datetime.time(0, 0, 0),
87 87 endTime=datetime.time(23, 59, 59),
88 88 timezone=0,
89 89 format=None,
90 90 **kwargs):
91 91
92 92 self.path = path
93 93 self.startDate = startDate
94 94 self.endDate = endDate
95 95 self.startTime = startTime
96 96 self.endTime = endTime
97 97 self.datatime = datetime.datetime(1900, 1, 1)
98 98 self.format = format
99 99
100 100 if self.path is None:
101 101 raise ValueError("The path is not valid")
102 102
103 103 if ext is None:
104 104 ext = self.ext
105 105
106 106 self.search_files(self.path, startDate, endDate, ext)
107 107 self.timezone = timezone
108 108 self.fileIndex = 0
109 109
110 110 if not self.fileList:
111 111 log.warning('There is no files matching these date in the folder: {}'.format(
112 112 path), self.name)
113 113
114 114 self.setNextFile()
115 115
116 116 def search_files(self, path, startDate, endDate, ext):
117 117 '''
118 118 Searching for BLTR rawdata file in path
119 119 Creating a list of file to proces included in [startDate,endDate]
120 120
121 121 Input:
122 122 path - Path to find BLTR rawdata files
123 123 startDate - Select file from this date
124 124 enDate - Select file until this date
125 125 ext - Extension of the file to read
126 126 '''
127 127
128 128 log.success('Searching files in {} '.format(path), self.name)
129 129 fileList0 = glob.glob1(path, '{}*{}'.format(self.format.upper(), ext))
130 130 fileList0.sort()
131 131
132 132 self.fileList = []
133 133 self.dateFileList = []
134 134
135 135 for thisFile in fileList0:
136 136 year = thisFile[2:4]
137 137 if not isNumber(year):
138 138 continue
139 139
140 140 month = thisFile[4:6]
141 141 if not isNumber(month):
142 142 continue
143 143
144 144 day = thisFile[6:8]
145 145 if not isNumber(day):
146 146 continue
147 147
148 148 year, month, day = int(year), int(month), int(day)
149 149 dateFile = datetime.date(year+2000, month, day)
150 150
151 151 if (startDate > dateFile) or (endDate < dateFile):
152 152 continue
153 153
154 154 self.fileList.append(thisFile)
155 155 self.dateFileList.append(dateFile)
156 156
157 157 return
158 158
159 159 def setNextFile(self):
160 160
161 161 file_id = self.fileIndex
162 162
163 163 if file_id == len(self.fileList):
164 164 log.success('No more files in the folder', self.name)
165 165 self.flagNoMoreFiles = 1
166 166 return 0
167 167
168 168 log.success('Opening {}'.format(self.fileList[file_id]), self.name)
169 169 filename = os.path.join(self.path, self.fileList[file_id])
170 170
171 171 dirname, name = os.path.split(filename)
172 172 self.siteFile = name.split('.')[0]
173 173 if self.filename is not None:
174 174 self.fp.close()
175 175 self.filename = filename
176 176 self.fp = open(self.filename, 'rb')
177 177
178 178 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
179 179 yy = self.header_file['year'] - 1900 * (self.header_file['year'] > 3000)
180 180 self.year = int(yy + 1900 * (yy < 1000))
181 181 self.doy = int(self.header_file['doy'])
182 182 self.dH = round(self.header_file['dh'], 2)
183 183 self.ipp = round(self.header_file['ipp'], 2)
184 184 self.sizeOfFile = os.path.getsize(self.filename)
185 185 self.counter_records = 0
186 186 self.flagIsNewFile = 0
187 187 self.fileIndex += 1
188 188
189 189 return 1
190 190
191 191 def readNextBlock(self):
192 192
193 193 while True:
194 194 if not self.readBlock():
195 195 self.flagIsNewFile = 1
196 196 if not self.setNextFile():
197 197 return 0
198 198
199 199 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
200 200 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
201 201 log.warning(
202 202 'Reading Record No. {} -> {} [Skipping]'.format(
203 203 self.counter_records,
204 204 self.datatime.ctime()),
205 205 self.name)
206 206 continue
207 207 break
208 208
209 209 log.log('Reading Record No. {} -> {}'.format(
210 210 self.counter_records,
211 211 self.datatime.ctime()), self.name)
212 212
213 213 return 1
214 214
215 215 def readBlock(self):
216 216
217 217 pointer = self.fp.tell()
218 218 heights, dt = self.readHeader()
219 219 self.fp.seek(pointer)
220 220 buffer_h = []
221 221 buffer_d = []
222 222 while True:
223 223 pointer = self.fp.tell()
224 224 if pointer == self.sizeOfFile:
225 225 return 0
226 226 heights, datatime = self.readHeader()
227 227 if dt == datatime:
228 228 buffer_h.append(heights)
229 229 buffer_d.append(self.readData(len(heights)))
230 230 continue
231 231 self.fp.seek(pointer)
232 232 break
233 233
234 234 if dt.date() > self.datatime.date():
235 235 self.flagDiscontinuousBlock = 1
236 236 self.datatime = dt
237 237 self.time = (dt - datetime.datetime(1970, 1, 1)).total_seconds() + time.timezone
238 238 self.heights = numpy.concatenate(buffer_h)
239 239 self.buffer = numpy.zeros((5, len(self.heights))) + numpy.nan
240 240 self.buffer[0, :] = numpy.concatenate([buf[0] for buf in buffer_d])
241 241 self.buffer[1, :] = numpy.concatenate([buf[1] for buf in buffer_d])
242 242 self.buffer[2, :] = numpy.concatenate([buf[2] for buf in buffer_d])
243 243 self.buffer[3, :] = numpy.concatenate([buf[3] for buf in buffer_d])
244 244 self.buffer[4, :] = numpy.concatenate([buf[4] for buf in buffer_d])
245 245
246 246 self.counter_records += 1
247 247
248 248 return 1
249 249
250 250 def readHeader(self):
251 251 '''
252 252 Parse recordHeader
253 253 '''
254 254
255 255 self.header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
256 256 self.interval = self.header_rec['interval']
257 257 if self.header_rec['magic'] == 888.:
258 258 self.header_rec['h0'] = round(self.header_rec['h0'], 2)
259 259 nheights = int(self.header_rec['nheights'])
260 260 hours = float(self.header_rec['hours'][0])
261 261 heights = numpy.arange(nheights) * self.dH + self.header_rec['h0']
262 262 datatime = datetime.datetime(self.year, 1, 1) + datetime.timedelta(days=self.doy-1, hours=hours)
263 263 return heights, datatime
264 264 else:
265 265 return False
266 266
267 267 def readData(self, N):
268 268 '''
269 269 Parse data
270 270 '''
271 271
272 272 buffer = numpy.fromfile(self.fp, 'f', 8*N).reshape(N, 8)
273 273
274 274 pow0 = buffer[:, 0]
275 275 pow1 = buffer[:, 1]
276 276 acf0 = (buffer[:,2] + buffer[:,3]*1j) / pow0
277 277 acf1 = (buffer[:,4] + buffer[:,5]*1j) / pow1
278 278 dccf = (buffer[:,6] + buffer[:,7]*1j) / (pow0*pow1)
279 279
280 280 ### SNR
281 281 sno = (pow0 + pow1 - self.header_rec['snr']) / self.header_rec['snr']
282 282 sno10 = numpy.log10(sno)
283 283 # dsno = 1.0 / numpy.sqrt(self.header_file['nint'] * self.header_file['navg']) * (1 + (1 / sno))
284 284
285 285 ### Vertical Drift
286 286 sp = numpy.sqrt(numpy.abs(acf0)*numpy.abs(acf1))
287 287 sp[numpy.where(numpy.abs(sp) >= 1.0)] = numpy.sqrt(0.9999)
288 288
289 289 vzo = -numpy.arctan2(acf0.imag + acf1.imag,acf0.real + acf1.real)*1.5E5*1.5/(self.ipp*numpy.pi)
290 290 dvzo = numpy.sqrt(1.0 - sp*sp)*0.338*1.5E5/(numpy.sqrt(self.header_file['nint']*self.header_file['navg'])*sp*self.ipp)
291 291 err = numpy.where(dvzo <= 0.1)
292 292 dvzo[err] = 0.1
293 293
294 294 #Zonal Drifts
295 295 dt = self.header_file['nint']*self.ipp / 1.5E5
296 296 coh = numpy.sqrt(numpy.abs(dccf))
297 297 err = numpy.where(coh >= 1.0)
298 298 coh[err] = numpy.sqrt(0.99999)
299 299
300 300 err = numpy.where(coh <= 0.1)
301 301 coh[err] = numpy.sqrt(0.1)
302 302
303 303 vxo = numpy.arctan2(dccf.imag, dccf.real)*self.header_rec['h0']*1.0E3/(self.kd*dt)
304 304 dvxo = numpy.sqrt(1.0 - coh*coh)*self.header_rec['h0']*1.0E3/(numpy.sqrt(self.header_file['nint']*self.header_file['navg'])*coh*self.kd*dt)
305 305
306 306 err = numpy.where(dvxo <= 0.1)
307 307 dvxo[err] = 0.1
308 308
309 309 return vzo, dvzo, vxo, dvxo, sno10
310 310
311 311 def set_output(self):
312 312 '''
313 313 Storing data from databuffer to dataOut object
314 314 '''
315 315
316 self.dataOut.data_SNR = self.buffer[4].reshape(1, -1)
316 self.dataOut.data_snr = self.buffer[4].reshape(1, -1)
317 317 self.dataOut.heightList = self.heights
318 318 self.dataOut.data_param = self.buffer[0:4,]
319 319 self.dataOut.utctimeInit = self.time
320 320 self.dataOut.utctime = self.time
321 321 self.dataOut.useLocalTime = True
322 322 self.dataOut.paramInterval = self.interval
323 323 self.dataOut.timezone = self.timezone
324 324 self.dataOut.sizeOfFile = self.sizeOfFile
325 325 self.dataOut.flagNoData = False
326 326 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
327 327
328 328 def getData(self):
329 329 '''
330 330 Storing data from databuffer to dataOut object
331 331 '''
332 332 if self.flagNoMoreFiles:
333 333 self.dataOut.flagNoData = True
334 334 log.success('No file left to process', self.name)
335 335 return 0
336 336
337 337 if not self.readNextBlock():
338 338 self.dataOut.flagNoData = True
339 339 return 0
340 340
341 341 self.set_output()
342 342
343 343 return 1 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,402 +1,402
1 1 '''
2 2 Created on Oct 24, 2016
3 3
4 4 @author: roj- LouVD
5 5 '''
6 6
7 7 import numpy
8 8 import copy
9 9 import datetime
10 10 import time
11 11 from time import gmtime
12 12
13 13 from numpy import transpose
14 14
15 15 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
16 16 from schainpy.model.data.jrodata import Parameters
17 17
18 18
19 19 class BLTRParametersProc(ProcessingUnit):
20 20 '''
21 21 Processing unit for BLTR parameters data (winds)
22 22
23 23 Inputs:
24 24 self.dataOut.nmodes - Number of operation modes
25 25 self.dataOut.nchannels - Number of channels
26 26 self.dataOut.nranges - Number of ranges
27 27
28 self.dataOut.data_SNR - SNR array
28 self.dataOut.data_snr - SNR array
29 29 self.dataOut.data_output - Zonal, Vertical and Meridional velocity array
30 30 self.dataOut.height - Height array (km)
31 31 self.dataOut.time - Time array (seconds)
32 32
33 33 self.dataOut.fileIndex -Index of the file currently read
34 34 self.dataOut.lat - Latitude coordinate of BLTR location
35 35
36 36 self.dataOut.doy - Experiment doy (number of the day in the current year)
37 37 self.dataOut.month - Experiment month
38 38 self.dataOut.day - Experiment day
39 39 self.dataOut.year - Experiment year
40 40 '''
41 41
42 42 def __init__(self):
43 43 '''
44 44 Inputs: None
45 45 '''
46 46 ProcessingUnit.__init__(self)
47 47 self.dataOut = Parameters()
48 48
49 49 def setup(self, mode):
50 50 '''
51 51 '''
52 52 self.dataOut.mode = mode
53 53
54 54 def run(self, mode, snr_threshold=None):
55 55 '''
56 56 Inputs:
57 57 mode = High resolution (0) or Low resolution (1) data
58 58 snr_threshold = snr filter value
59 59 '''
60 60
61 61 if not self.isConfig:
62 62 self.setup(mode)
63 63 self.isConfig = True
64 64
65 65 if self.dataIn.type == 'Parameters':
66 66 self.dataOut.copy(self.dataIn)
67 67
68 68 self.dataOut.data_param = self.dataOut.data[mode]
69 69 self.dataOut.heightList = self.dataOut.height[0]
70 self.dataOut.data_SNR = self.dataOut.data_SNR[mode]
70 self.dataOut.data_snr = self.dataOut.data_snr[mode]
71 71
72 72 if snr_threshold is not None:
73 SNRavg = numpy.average(self.dataOut.data_SNR, axis=0)
73 SNRavg = numpy.average(self.dataOut.data_snr, axis=0)
74 74 SNRavgdB = 10*numpy.log10(SNRavg)
75 75 for i in range(3):
76 76 self.dataOut.data_param[i][SNRavgdB <= snr_threshold] = numpy.nan
77 77
78 78 # TODO
79 79
80 80 class OutliersFilter(Operation):
81 81
82 82 def __init__(self):
83 83 '''
84 84 '''
85 85 Operation.__init__(self)
86 86
87 87 def run(self, svalue2, method, factor, filter, npoints=9):
88 88 '''
89 89 Inputs:
90 90 svalue - string to select array velocity
91 91 svalue2 - string to choose axis filtering
92 92 method - 0 for SMOOTH or 1 for MEDIAN
93 93 factor - number used to set threshold
94 94 filter - 1 for data filtering using the standard deviation criteria else 0
95 95 npoints - number of points for mask filter
96 96 '''
97 97
98 98 print(' Outliers Filter {} {} / threshold = {}'.format(svalue, svalue, factor))
99 99
100 100
101 101 yaxis = self.dataOut.heightList
102 102 xaxis = numpy.array([[self.dataOut.utctime]])
103 103
104 104 # Zonal
105 105 value_temp = self.dataOut.data_output[0]
106 106
107 107 # Zonal
108 108 value_temp = self.dataOut.data_output[1]
109 109
110 110 # Vertical
111 111 value_temp = numpy.transpose(self.dataOut.data_output[2])
112 112
113 113 htemp = yaxis
114 114 std = value_temp
115 115 for h in range(len(htemp)):
116 116 nvalues_valid = len(numpy.where(numpy.isfinite(value_temp[h]))[0])
117 117 minvalid = npoints
118 118
119 119 #only if valid values greater than the minimum required (10%)
120 120 if nvalues_valid > minvalid:
121 121
122 122 if method == 0:
123 123 #SMOOTH
124 124 w = value_temp[h] - self.Smooth(input=value_temp[h], width=npoints, edge_truncate=1)
125 125
126 126
127 127 if method == 1:
128 128 #MEDIAN
129 129 w = value_temp[h] - self.Median(input=value_temp[h], width = npoints)
130 130
131 131 dw = numpy.std(w[numpy.where(numpy.isfinite(w))],ddof = 1)
132 132
133 133 threshold = dw*factor
134 134 value_temp[numpy.where(w > threshold),h] = numpy.nan
135 135 value_temp[numpy.where(w < -1*threshold),h] = numpy.nan
136 136
137 137
138 138 #At the end
139 139 if svalue2 == 'inHeight':
140 140 value_temp = numpy.transpose(value_temp)
141 141 output_array[:,m] = value_temp
142 142
143 143 if svalue == 'zonal':
144 144 self.dataOut.data_output[0] = output_array
145 145
146 146 elif svalue == 'meridional':
147 147 self.dataOut.data_output[1] = output_array
148 148
149 149 elif svalue == 'vertical':
150 150 self.dataOut.data_output[2] = output_array
151 151
152 152 return self.dataOut.data_output
153 153
154 154
155 155 def Median(self,input,width):
156 156 '''
157 157 Inputs:
158 158 input - Velocity array
159 159 width - Number of points for mask filter
160 160
161 161 '''
162 162
163 163 if numpy.mod(width,2) == 1:
164 164 pc = int((width - 1) / 2)
165 165 cont = 0
166 166 output = []
167 167
168 168 for i in range(len(input)):
169 169 if i >= pc and i < len(input) - pc:
170 170 new2 = input[i-pc:i+pc+1]
171 171 temp = numpy.where(numpy.isfinite(new2))
172 172 new = new2[temp]
173 173 value = numpy.median(new)
174 174 output.append(value)
175 175
176 176 output = numpy.array(output)
177 177 output = numpy.hstack((input[0:pc],output))
178 178 output = numpy.hstack((output,input[-pc:len(input)]))
179 179
180 180 return output
181 181
182 182 def Smooth(self,input,width,edge_truncate = None):
183 183 '''
184 184 Inputs:
185 185 input - Velocity array
186 186 width - Number of points for mask filter
187 187 edge_truncate - 1 for truncate the convolution product else
188 188
189 189 '''
190 190
191 191 if numpy.mod(width,2) == 0:
192 192 real_width = width + 1
193 193 nzeros = width / 2
194 194 else:
195 195 real_width = width
196 196 nzeros = (width - 1) / 2
197 197
198 198 half_width = int(real_width)/2
199 199 length = len(input)
200 200
201 201 gate = numpy.ones(real_width,dtype='float')
202 202 norm_of_gate = numpy.sum(gate)
203 203
204 204 nan_process = 0
205 205 nan_id = numpy.where(numpy.isnan(input))
206 206 if len(nan_id[0]) > 0:
207 207 nan_process = 1
208 208 pb = numpy.zeros(len(input))
209 209 pb[nan_id] = 1.
210 210 input[nan_id] = 0.
211 211
212 212 if edge_truncate == True:
213 213 output = numpy.convolve(input/norm_of_gate,gate,mode='same')
214 214 elif edge_truncate == False or edge_truncate == None:
215 215 output = numpy.convolve(input/norm_of_gate,gate,mode='valid')
216 216 output = numpy.hstack((input[0:half_width],output))
217 217 output = numpy.hstack((output,input[len(input)-half_width:len(input)]))
218 218
219 219 if nan_process:
220 220 pb = numpy.convolve(pb/norm_of_gate,gate,mode='valid')
221 221 pb = numpy.hstack((numpy.zeros(half_width),pb))
222 222 pb = numpy.hstack((pb,numpy.zeros(half_width)))
223 223 output[numpy.where(pb > 0.9999)] = numpy.nan
224 224 input[nan_id] = numpy.nan
225 225 return output
226 226
227 227 def Average(self,aver=0,nhaver=1):
228 228 '''
229 229 Inputs:
230 230 aver - Indicates the time period over which is averaged or consensus data
231 231 nhaver - Indicates the decimation factor in heights
232 232
233 233 '''
234 234 nhpoints = 48
235 235
236 236 lat_piura = -5.17
237 237 lat_huancayo = -12.04
238 238 lat_porcuya = -5.8
239 239
240 240 if '%2.2f'%self.dataOut.lat == '%2.2f'%lat_piura:
241 241 hcm = 3.
242 242 if self.dataOut.year == 2003 :
243 243 if self.dataOut.doy >= 25 and self.dataOut.doy < 64:
244 244 nhpoints = 12
245 245
246 246 elif '%2.2f'%self.dataOut.lat == '%2.2f'%lat_huancayo:
247 247 hcm = 3.
248 248 if self.dataOut.year == 2003 :
249 249 if self.dataOut.doy >= 25 and self.dataOut.doy < 64:
250 250 nhpoints = 12
251 251
252 252
253 253 elif '%2.2f'%self.dataOut.lat == '%2.2f'%lat_porcuya:
254 254 hcm = 5.#2
255 255
256 256 pdata = 0.2
257 257 taver = [1,2,3,4,6,8,12,24]
258 258 t0 = 0
259 259 tf = 24
260 260 ntime =(tf-t0)/taver[aver]
261 261 ti = numpy.arange(ntime)
262 262 tf = numpy.arange(ntime) + taver[aver]
263 263
264 264
265 265 old_height = self.dataOut.heightList
266 266
267 267 if nhaver > 1:
268 268 num_hei = len(self.dataOut.heightList)/nhaver/self.dataOut.nmodes
269 269 deltha = 0.05*nhaver
270 270 minhvalid = pdata*nhaver
271 271 for im in range(self.dataOut.nmodes):
272 272 new_height = numpy.arange(num_hei)*deltha + self.dataOut.height[im,0] + deltha/2.
273 273
274 274
275 275 data_fHeigths_List = []
276 276 data_fZonal_List = []
277 277 data_fMeridional_List = []
278 278 data_fVertical_List = []
279 279 startDTList = []
280 280
281 281
282 282 for i in range(ntime):
283 283 height = old_height
284 284
285 285 start = datetime.datetime(self.dataOut.year,self.dataOut.month,self.dataOut.day) + datetime.timedelta(hours = int(ti[i])) - datetime.timedelta(hours = 5)
286 286 stop = datetime.datetime(self.dataOut.year,self.dataOut.month,self.dataOut.day) + datetime.timedelta(hours = int(tf[i])) - datetime.timedelta(hours = 5)
287 287
288 288
289 289 limit_sec1 = time.mktime(start.timetuple())
290 290 limit_sec2 = time.mktime(stop.timetuple())
291 291
292 292 t1 = numpy.where(self.f_timesec >= limit_sec1)
293 293 t2 = numpy.where(self.f_timesec < limit_sec2)
294 294 time_select = []
295 295 for val_sec in t1[0]:
296 296 if val_sec in t2[0]:
297 297 time_select.append(val_sec)
298 298
299 299
300 300 time_select = numpy.array(time_select,dtype = 'int')
301 301 minvalid = numpy.ceil(pdata*nhpoints)
302 302
303 303 zon_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan
304 304 mer_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan
305 305 ver_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan
306 306
307 307 if nhaver > 1:
308 308 new_zon_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan
309 309 new_mer_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan
310 310 new_ver_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan
311 311
312 312 if len(time_select) > minvalid:
313 313 time_average = self.f_timesec[time_select]
314 314
315 315 for im in range(self.dataOut.nmodes):
316 316
317 317 for ih in range(self.dataOut.nranges):
318 318 if numpy.sum(numpy.isfinite(self.f_zon[time_select,ih,im])) >= minvalid:
319 319 zon_aver[ih,im] = numpy.nansum(self.f_zon[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_zon[time_select,ih,im]))
320 320
321 321 if numpy.sum(numpy.isfinite(self.f_mer[time_select,ih,im])) >= minvalid:
322 322 mer_aver[ih,im] = numpy.nansum(self.f_mer[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_mer[time_select,ih,im]))
323 323
324 324 if numpy.sum(numpy.isfinite(self.f_ver[time_select,ih,im])) >= minvalid:
325 325 ver_aver[ih,im] = numpy.nansum(self.f_ver[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_ver[time_select,ih,im]))
326 326
327 327 if nhaver > 1:
328 328 for ih in range(num_hei):
329 329 hvalid = numpy.arange(nhaver) + nhaver*ih
330 330
331 331 if numpy.sum(numpy.isfinite(zon_aver[hvalid,im])) >= minvalid:
332 332 new_zon_aver[ih,im] = numpy.nansum(zon_aver[hvalid,im]) / numpy.sum(numpy.isfinite(zon_aver[hvalid,im]))
333 333
334 334 if numpy.sum(numpy.isfinite(mer_aver[hvalid,im])) >= minvalid:
335 335 new_mer_aver[ih,im] = numpy.nansum(mer_aver[hvalid,im]) / numpy.sum(numpy.isfinite(mer_aver[hvalid,im]))
336 336
337 337 if numpy.sum(numpy.isfinite(ver_aver[hvalid,im])) >= minvalid:
338 338 new_ver_aver[ih,im] = numpy.nansum(ver_aver[hvalid,im]) / numpy.sum(numpy.isfinite(ver_aver[hvalid,im]))
339 339 if nhaver > 1:
340 340 zon_aver = new_zon_aver
341 341 mer_aver = new_mer_aver
342 342 ver_aver = new_ver_aver
343 343 height = new_height
344 344
345 345
346 346 tstart = time_average[0]
347 347 tend = time_average[-1]
348 348 startTime = time.gmtime(tstart)
349 349
350 350 year = startTime.tm_year
351 351 month = startTime.tm_mon
352 352 day = startTime.tm_mday
353 353 hour = startTime.tm_hour
354 354 minute = startTime.tm_min
355 355 second = startTime.tm_sec
356 356
357 357 startDTList.append(datetime.datetime(year,month,day,hour,minute,second))
358 358
359 359
360 360 o_height = numpy.array([])
361 361 o_zon_aver = numpy.array([])
362 362 o_mer_aver = numpy.array([])
363 363 o_ver_aver = numpy.array([])
364 364 if self.dataOut.nmodes > 1:
365 365 for im in range(self.dataOut.nmodes):
366 366
367 367 if im == 0:
368 368 h_select = numpy.where(numpy.bitwise_and(height[0,:] >=0,height[0,:] <= hcm,numpy.isfinite(height[0,:])))
369 369 else:
370 370 h_select = numpy.where(numpy.bitwise_and(height[1,:] > hcm,height[1,:] < 20,numpy.isfinite(height[1,:])))
371 371
372 372
373 373 ht = h_select[0]
374 374
375 375 o_height = numpy.hstack((o_height,height[im,ht]))
376 376 o_zon_aver = numpy.hstack((o_zon_aver,zon_aver[ht,im]))
377 377 o_mer_aver = numpy.hstack((o_mer_aver,mer_aver[ht,im]))
378 378 o_ver_aver = numpy.hstack((o_ver_aver,ver_aver[ht,im]))
379 379
380 380 data_fHeigths_List.append(o_height)
381 381 data_fZonal_List.append(o_zon_aver)
382 382 data_fMeridional_List.append(o_mer_aver)
383 383 data_fVertical_List.append(o_ver_aver)
384 384
385 385
386 386 else:
387 387 h_select = numpy.where(numpy.bitwise_and(height[0,:] <= hcm,numpy.isfinite(height[0,:])))
388 388 ht = h_select[0]
389 389 o_height = numpy.hstack((o_height,height[im,ht]))
390 390 o_zon_aver = numpy.hstack((o_zon_aver,zon_aver[ht,im]))
391 391 o_mer_aver = numpy.hstack((o_mer_aver,mer_aver[ht,im]))
392 392 o_ver_aver = numpy.hstack((o_ver_aver,ver_aver[ht,im]))
393 393
394 394 data_fHeigths_List.append(o_height)
395 395 data_fZonal_List.append(o_zon_aver)
396 396 data_fMeridional_List.append(o_mer_aver)
397 397 data_fVertical_List.append(o_ver_aver)
398 398
399 399
400 400 return startDTList, data_fHeigths_List, data_fZonal_List, data_fMeridional_List, data_fVertical_List
401 401
402 402
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,876 +1,898
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Spectra processing Unit and operations
6 6
7 7 Here you will find the processing unit `SpectraProc` and several operations
8 8 to work with Spectra data type
9 9 """
10 10
11 11 import time
12 12 import itertools
13 13
14 14 import numpy
15 15
16 16 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
17 17 from schainpy.model.data.jrodata import Spectra
18 18 from schainpy.model.data.jrodata import hildebrand_sekhon
19 19 from schainpy.utils import log
20 20
21 21
22 22 class SpectraProc(ProcessingUnit):
23 23
24 24 def __init__(self):
25 25
26 26 ProcessingUnit.__init__(self)
27 27
28 28 self.buffer = None
29 29 self.firstdatatime = None
30 30 self.profIndex = 0
31 31 self.dataOut = Spectra()
32 32 self.id_min = None
33 33 self.id_max = None
34 34 self.setupReq = False #Agregar a todas las unidades de proc
35 35
36 36 def __updateSpecFromVoltage(self):
37 37
38 38 self.dataOut.timeZone = self.dataIn.timeZone
39 39 self.dataOut.dstFlag = self.dataIn.dstFlag
40 40 self.dataOut.errorCount = self.dataIn.errorCount
41 41 self.dataOut.useLocalTime = self.dataIn.useLocalTime
42 42 try:
43 43 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
44 44 except:
45 45 pass
46 46 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
47 47 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
48 48 self.dataOut.channelList = self.dataIn.channelList
49 49 self.dataOut.heightList = self.dataIn.heightList
50 50 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
51 51 self.dataOut.nProfiles = self.dataOut.nFFTPoints
52 52 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
53 53 self.dataOut.utctime = self.firstdatatime
54 54 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
55 55 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
56 56 self.dataOut.flagShiftFFT = False
57 57 self.dataOut.nCohInt = self.dataIn.nCohInt
58 58 self.dataOut.nIncohInt = 1
59 59 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
60 60 self.dataOut.frequency = self.dataIn.frequency
61 61 self.dataOut.realtime = self.dataIn.realtime
62 62 self.dataOut.azimuth = self.dataIn.azimuth
63 63 self.dataOut.zenith = self.dataIn.zenith
64 64 self.dataOut.beam.codeList = self.dataIn.beam.codeList
65 65 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
66 66 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
67 67
68 68 def __getFft(self):
69 69 """
70 70 Convierte valores de Voltaje a Spectra
71 71
72 72 Affected:
73 73 self.dataOut.data_spc
74 74 self.dataOut.data_cspc
75 75 self.dataOut.data_dc
76 76 self.dataOut.heightList
77 77 self.profIndex
78 78 self.buffer
79 79 self.dataOut.flagNoData
80 80 """
81 81 fft_volt = numpy.fft.fft(
82 82 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
83 83 fft_volt = fft_volt.astype(numpy.dtype('complex'))
84 84 dc = fft_volt[:, 0, :]
85 85
86 86 # calculo de self-spectra
87 87 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
88 88 spc = fft_volt * numpy.conjugate(fft_volt)
89 89 spc = spc.real
90 90
91 91 blocksize = 0
92 92 blocksize += dc.size
93 93 blocksize += spc.size
94 94
95 95 cspc = None
96 96 pairIndex = 0
97 97 if self.dataOut.pairsList != None:
98 98 # calculo de cross-spectra
99 99 cspc = numpy.zeros(
100 100 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
101 101 for pair in self.dataOut.pairsList:
102 102 if pair[0] not in self.dataOut.channelList:
103 103 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
104 104 str(pair), str(self.dataOut.channelList)))
105 105 if pair[1] not in self.dataOut.channelList:
106 106 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
107 107 str(pair), str(self.dataOut.channelList)))
108 108
109 109 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
110 110 numpy.conjugate(fft_volt[pair[1], :, :])
111 111 pairIndex += 1
112 112 blocksize += cspc.size
113 113
114 114 self.dataOut.data_spc = spc
115 115 self.dataOut.data_cspc = cspc
116 116 self.dataOut.data_dc = dc
117 117 self.dataOut.blockSize = blocksize
118 118 self.dataOut.flagShiftFFT = False
119 119
120 120 def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False):
121 121
122 122 if self.dataIn.type == "Spectra":
123 123 self.dataOut.copy(self.dataIn)
124 124 if shift_fft:
125 125 #desplaza a la derecha en el eje 2 determinadas posiciones
126 126 shift = int(self.dataOut.nFFTPoints/2)
127 127 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
128 128
129 129 if self.dataOut.data_cspc is not None:
130 130 #desplaza a la derecha en el eje 2 determinadas posiciones
131 131 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
132 132 if pairsList:
133 133 self.__selectPairs(pairsList)
134 134
135 135 elif self.dataIn.type == "Voltage":
136 136
137 137 self.dataOut.flagNoData = True
138 138
139 139 if nFFTPoints == None:
140 140 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
141 141
142 142 if nProfiles == None:
143 143 nProfiles = nFFTPoints
144 144
145 145 if ippFactor == None:
146 146 self.dataOut.ippFactor = 1
147 147
148 148 self.dataOut.nFFTPoints = nFFTPoints
149 149
150 150 if self.buffer is None:
151 151 self.buffer = numpy.zeros((self.dataIn.nChannels,
152 152 nProfiles,
153 153 self.dataIn.nHeights),
154 154 dtype='complex')
155 155
156 156 if self.dataIn.flagDataAsBlock:
157 157 nVoltProfiles = self.dataIn.data.shape[1]
158 158
159 159 if nVoltProfiles == nProfiles:
160 160 self.buffer = self.dataIn.data.copy()
161 161 self.profIndex = nVoltProfiles
162 162
163 163 elif nVoltProfiles < nProfiles:
164 164
165 165 if self.profIndex == 0:
166 166 self.id_min = 0
167 167 self.id_max = nVoltProfiles
168 168
169 169 self.buffer[:, self.id_min:self.id_max,
170 170 :] = self.dataIn.data
171 171 self.profIndex += nVoltProfiles
172 172 self.id_min += nVoltProfiles
173 173 self.id_max += nVoltProfiles
174 174 else:
175 175 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
176 176 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
177 177 self.dataOut.flagNoData = True
178 178 else:
179 179 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
180 180 self.profIndex += 1
181 181
182 182 if self.firstdatatime == None:
183 183 self.firstdatatime = self.dataIn.utctime
184 184
185 185 if self.profIndex == nProfiles:
186 186 self.__updateSpecFromVoltage()
187 187 if pairsList == None:
188 188 self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)]
189 189 else:
190 190 self.dataOut.pairsList = pairsList
191 191 self.__getFft()
192 192 self.dataOut.flagNoData = False
193 193 self.firstdatatime = None
194 194 self.profIndex = 0
195 195 else:
196 196 raise ValueError("The type of input object '%s' is not valid".format(
197 197 self.dataIn.type))
198 198
199 199 def __selectPairs(self, pairsList):
200 200
201 201 if not pairsList:
202 202 return
203 203
204 204 pairs = []
205 205 pairsIndex = []
206 206
207 207 for pair in pairsList:
208 208 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
209 209 continue
210 210 pairs.append(pair)
211 211 pairsIndex.append(pairs.index(pair))
212 212
213 213 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
214 214 self.dataOut.pairsList = pairs
215 215
216 216 return
217 217
218 218 def selectFFTs(self, minFFT, maxFFT ):
219 219 """
220 220 Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
221 221 minFFT<= FFT <= maxFFT
222 222 """
223 223
224 224 if (minFFT > maxFFT):
225 225 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
226 226
227 227 if (minFFT < self.dataOut.getFreqRange()[0]):
228 228 minFFT = self.dataOut.getFreqRange()[0]
229 229
230 230 if (maxFFT > self.dataOut.getFreqRange()[-1]):
231 231 maxFFT = self.dataOut.getFreqRange()[-1]
232 232
233 233 minIndex = 0
234 234 maxIndex = 0
235 235 FFTs = self.dataOut.getFreqRange()
236 236
237 237 inda = numpy.where(FFTs >= minFFT)
238 238 indb = numpy.where(FFTs <= maxFFT)
239 239
240 240 try:
241 241 minIndex = inda[0][0]
242 242 except:
243 243 minIndex = 0
244 244
245 245 try:
246 246 maxIndex = indb[0][-1]
247 247 except:
248 248 maxIndex = len(FFTs)
249 249
250 250 self.selectFFTsByIndex(minIndex, maxIndex)
251 251
252 252 return 1
253 253
254 254 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
255 255 newheis = numpy.where(
256 256 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
257 257
258 258 if hei_ref != None:
259 259 newheis = numpy.where(self.dataOut.heightList > hei_ref)
260 260
261 261 minIndex = min(newheis[0])
262 262 maxIndex = max(newheis[0])
263 263 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
264 264 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
265 265
266 266 # determina indices
267 267 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
268 268 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
269 269 avg_dB = 10 * \
270 270 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
271 271 beacon_dB = numpy.sort(avg_dB)[-nheis:]
272 272 beacon_heiIndexList = []
273 273 for val in avg_dB.tolist():
274 274 if val >= beacon_dB[0]:
275 275 beacon_heiIndexList.append(avg_dB.tolist().index(val))
276 276
277 277 #data_spc = data_spc[:,:,beacon_heiIndexList]
278 278 data_cspc = None
279 279 if self.dataOut.data_cspc is not None:
280 280 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
281 281 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
282 282
283 283 data_dc = None
284 284 if self.dataOut.data_dc is not None:
285 285 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
286 286 #data_dc = data_dc[:,beacon_heiIndexList]
287 287
288 288 self.dataOut.data_spc = data_spc
289 289 self.dataOut.data_cspc = data_cspc
290 290 self.dataOut.data_dc = data_dc
291 291 self.dataOut.heightList = heightList
292 292 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
293 293
294 294 return 1
295 295
296 296 def selectFFTsByIndex(self, minIndex, maxIndex):
297 297 """
298 298
299 299 """
300 300
301 301 if (minIndex < 0) or (minIndex > maxIndex):
302 302 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
303 303
304 304 if (maxIndex >= self.dataOut.nProfiles):
305 305 maxIndex = self.dataOut.nProfiles-1
306 306
307 307 #Spectra
308 308 data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
309 309
310 310 data_cspc = None
311 311 if self.dataOut.data_cspc is not None:
312 312 data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
313 313
314 314 data_dc = None
315 315 if self.dataOut.data_dc is not None:
316 316 data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
317 317
318 318 self.dataOut.data_spc = data_spc
319 319 self.dataOut.data_cspc = data_cspc
320 320 self.dataOut.data_dc = data_dc
321 321
322 322 self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
323 323 self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
324 324 self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
325 325
326 326 return 1
327 327
328 328 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
329 329 # validacion de rango
330 330 if minHei == None:
331 331 minHei = self.dataOut.heightList[0]
332 332
333 333 if maxHei == None:
334 334 maxHei = self.dataOut.heightList[-1]
335 335
336 336 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
337 337 print('minHei: %.2f is out of the heights range' % (minHei))
338 338 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
339 339 minHei = self.dataOut.heightList[0]
340 340
341 341 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
342 342 print('maxHei: %.2f is out of the heights range' % (maxHei))
343 343 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
344 344 maxHei = self.dataOut.heightList[-1]
345 345
346 346 # validacion de velocidades
347 347 velrange = self.dataOut.getVelRange(1)
348 348
349 349 if minVel == None:
350 350 minVel = velrange[0]
351 351
352 352 if maxVel == None:
353 353 maxVel = velrange[-1]
354 354
355 355 if (minVel < velrange[0]) or (minVel > maxVel):
356 356 print('minVel: %.2f is out of the velocity range' % (minVel))
357 357 print('minVel is setting to %.2f' % (velrange[0]))
358 358 minVel = velrange[0]
359 359
360 360 if (maxVel > velrange[-1]) or (maxVel < minVel):
361 361 print('maxVel: %.2f is out of the velocity range' % (maxVel))
362 362 print('maxVel is setting to %.2f' % (velrange[-1]))
363 363 maxVel = velrange[-1]
364 364
365 365 # seleccion de indices para rango
366 366 minIndex = 0
367 367 maxIndex = 0
368 368 heights = self.dataOut.heightList
369 369
370 370 inda = numpy.where(heights >= minHei)
371 371 indb = numpy.where(heights <= maxHei)
372 372
373 373 try:
374 374 minIndex = inda[0][0]
375 375 except:
376 376 minIndex = 0
377 377
378 378 try:
379 379 maxIndex = indb[0][-1]
380 380 except:
381 381 maxIndex = len(heights)
382 382
383 383 if (minIndex < 0) or (minIndex > maxIndex):
384 384 raise ValueError("some value in (%d,%d) is not valid" % (
385 385 minIndex, maxIndex))
386 386
387 387 if (maxIndex >= self.dataOut.nHeights):
388 388 maxIndex = self.dataOut.nHeights - 1
389 389
390 390 # seleccion de indices para velocidades
391 391 indminvel = numpy.where(velrange >= minVel)
392 392 indmaxvel = numpy.where(velrange <= maxVel)
393 393 try:
394 394 minIndexVel = indminvel[0][0]
395 395 except:
396 396 minIndexVel = 0
397 397
398 398 try:
399 399 maxIndexVel = indmaxvel[0][-1]
400 400 except:
401 401 maxIndexVel = len(velrange)
402 402
403 403 # seleccion del espectro
404 404 data_spc = self.dataOut.data_spc[:,
405 405 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
406 406 # estimacion de ruido
407 407 noise = numpy.zeros(self.dataOut.nChannels)
408 408
409 409 for channel in range(self.dataOut.nChannels):
410 410 daux = data_spc[channel, :, :]
411 411 sortdata = numpy.sort(daux, axis=None)
412 412 noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt)
413 413
414 414 self.dataOut.noise_estimation = noise.copy()
415 415
416 416 return 1
417 417
418 418 class removeDC(Operation):
419 419
420 420 def run(self, dataOut, mode=2):
421 421 self.dataOut = dataOut
422 422 jspectra = self.dataOut.data_spc
423 423 jcspectra = self.dataOut.data_cspc
424 424
425 425 num_chan = jspectra.shape[0]
426 426 num_hei = jspectra.shape[2]
427 427
428 428 if jcspectra is not None:
429 429 jcspectraExist = True
430 430 num_pairs = jcspectra.shape[0]
431 431 else:
432 432 jcspectraExist = False
433 433
434 434 freq_dc = int(jspectra.shape[1] / 2)
435 435 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
436 436 ind_vel = ind_vel.astype(int)
437 437
438 438 if ind_vel[0] < 0:
439 439 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
440 440
441 441 if mode == 1:
442 442 jspectra[:, freq_dc, :] = (
443 443 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
444 444
445 445 if jcspectraExist:
446 446 jcspectra[:, freq_dc, :] = (
447 447 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
448 448
449 449 if mode == 2:
450 450
451 451 vel = numpy.array([-2, -1, 1, 2])
452 452 xx = numpy.zeros([4, 4])
453 453
454 454 for fil in range(4):
455 455 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
456 456
457 457 xx_inv = numpy.linalg.inv(xx)
458 458 xx_aux = xx_inv[0, :]
459 459
460 460 for ich in range(num_chan):
461 461 yy = jspectra[ich, ind_vel, :]
462 462 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
463 463
464 464 junkid = jspectra[ich, freq_dc, :] <= 0
465 465 cjunkid = sum(junkid)
466 466
467 467 if cjunkid.any():
468 468 jspectra[ich, freq_dc, junkid.nonzero()] = (
469 469 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
470 470
471 471 if jcspectraExist:
472 472 for ip in range(num_pairs):
473 473 yy = jcspectra[ip, ind_vel, :]
474 474 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
475 475
476 476 self.dataOut.data_spc = jspectra
477 477 self.dataOut.data_cspc = jcspectra
478 478
479 479 return self.dataOut
480 480
481 481 class removeInterference(Operation):
482 482
483 483 def removeInterference2(self):
484 484
485 485 cspc = self.dataOut.data_cspc
486 486 spc = self.dataOut.data_spc
487 487 Heights = numpy.arange(cspc.shape[2])
488 488 realCspc = numpy.abs(cspc)
489 489
490 490 for i in range(cspc.shape[0]):
491 491 LinePower= numpy.sum(realCspc[i], axis=0)
492 492 Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
493 493 SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
494 494 InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
495 495 InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
496 496 InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
497 497
498 498
499 499 InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
500 500 #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
501 501 if len(InterferenceRange)<int(cspc.shape[1]*0.3):
502 502 cspc[i,InterferenceRange,:] = numpy.NaN
503 503
504 504 self.dataOut.data_cspc = cspc
505 505
506 506 def removeInterference(self, interf = 2, hei_interf = None, nhei_interf = None, offhei_interf = None):
507 507
508 508 jspectra = self.dataOut.data_spc
509 509 jcspectra = self.dataOut.data_cspc
510 510 jnoise = self.dataOut.getNoise()
511 511 num_incoh = self.dataOut.nIncohInt
512 512
513 513 num_channel = jspectra.shape[0]
514 514 num_prof = jspectra.shape[1]
515 515 num_hei = jspectra.shape[2]
516 516
517 517 # hei_interf
518 518 if hei_interf is None:
519 519 count_hei = int(num_hei / 2)
520 520 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
521 521 hei_interf = numpy.asarray(hei_interf)[0]
522 522 # nhei_interf
523 523 if (nhei_interf == None):
524 524 nhei_interf = 5
525 525 if (nhei_interf < 1):
526 526 nhei_interf = 1
527 527 if (nhei_interf > count_hei):
528 528 nhei_interf = count_hei
529 529 if (offhei_interf == None):
530 530 offhei_interf = 0
531 531
532 532 ind_hei = list(range(num_hei))
533 533 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
534 534 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
535 535 mask_prof = numpy.asarray(list(range(num_prof)))
536 536 num_mask_prof = mask_prof.size
537 537 comp_mask_prof = [0, num_prof / 2]
538 538
539 539 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
540 540 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
541 541 jnoise = numpy.nan
542 542 noise_exist = jnoise[0] < numpy.Inf
543 543
544 544 # Subrutina de Remocion de la Interferencia
545 545 for ich in range(num_channel):
546 546 # Se ordena los espectros segun su potencia (menor a mayor)
547 547 power = jspectra[ich, mask_prof, :]
548 548 power = power[:, hei_interf]
549 549 power = power.sum(axis=0)
550 550 psort = power.ravel().argsort()
551 551
552 552 # Se estima la interferencia promedio en los Espectros de Potencia empleando
553 553 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
554 554 offhei_interf, nhei_interf + offhei_interf))]]]
555 555
556 556 if noise_exist:
557 557 # tmp_noise = jnoise[ich] / num_prof
558 558 tmp_noise = jnoise[ich]
559 559 junkspc_interf = junkspc_interf - tmp_noise
560 560 #junkspc_interf[:,comp_mask_prof] = 0
561 561
562 562 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
563 563 jspc_interf = jspc_interf.transpose()
564 564 # Calculando el espectro de interferencia promedio
565 565 noiseid = numpy.where(
566 566 jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
567 567 noiseid = noiseid[0]
568 568 cnoiseid = noiseid.size
569 569 interfid = numpy.where(
570 570 jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
571 571 interfid = interfid[0]
572 572 cinterfid = interfid.size
573 573
574 574 if (cnoiseid > 0):
575 575 jspc_interf[noiseid] = 0
576 576
577 577 # Expandiendo los perfiles a limpiar
578 578 if (cinterfid > 0):
579 579 new_interfid = (
580 580 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
581 581 new_interfid = numpy.asarray(new_interfid)
582 582 new_interfid = {x for x in new_interfid}
583 583 new_interfid = numpy.array(list(new_interfid))
584 584 new_cinterfid = new_interfid.size
585 585 else:
586 586 new_cinterfid = 0
587 587
588 588 for ip in range(new_cinterfid):
589 589 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
590 590 jspc_interf[new_interfid[ip]
591 591 ] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
592 592
593 593 jspectra[ich, :, ind_hei] = jspectra[ich, :,
594 594 ind_hei] - jspc_interf # Corregir indices
595 595
596 596 # Removiendo la interferencia del punto de mayor interferencia
597 597 ListAux = jspc_interf[mask_prof].tolist()
598 598 maxid = ListAux.index(max(ListAux))
599 599
600 600 if cinterfid > 0:
601 601 for ip in range(cinterfid * (interf == 2) - 1):
602 602 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
603 603 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
604 604 cind = len(ind)
605 605
606 606 if (cind > 0):
607 607 jspectra[ich, interfid[ip], ind] = tmp_noise * \
608 608 (1 + (numpy.random.uniform(cind) - 0.5) /
609 609 numpy.sqrt(num_incoh))
610 610
611 611 ind = numpy.array([-2, -1, 1, 2])
612 612 xx = numpy.zeros([4, 4])
613 613
614 614 for id1 in range(4):
615 615 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
616 616
617 617 xx_inv = numpy.linalg.inv(xx)
618 618 xx = xx_inv[:, 0]
619 619 ind = (ind + maxid + num_mask_prof) % num_mask_prof
620 620 yy = jspectra[ich, mask_prof[ind], :]
621 621 jspectra[ich, mask_prof[maxid], :] = numpy.dot(
622 622 yy.transpose(), xx)
623 623
624 624 indAux = (jspectra[ich, :, :] < tmp_noise *
625 625 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
626 626 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
627 627 (1 - 1 / numpy.sqrt(num_incoh))
628 628
629 629 # Remocion de Interferencia en el Cross Spectra
630 630 if jcspectra is None:
631 631 return jspectra, jcspectra
632 632 num_pairs = int(jcspectra.size / (num_prof * num_hei))
633 633 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
634 634
635 635 for ip in range(num_pairs):
636 636
637 637 #-------------------------------------------
638 638
639 639 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
640 640 cspower = cspower[:, hei_interf]
641 641 cspower = cspower.sum(axis=0)
642 642
643 643 cspsort = cspower.ravel().argsort()
644 644 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
645 645 offhei_interf, nhei_interf + offhei_interf))]]]
646 646 junkcspc_interf = junkcspc_interf.transpose()
647 647 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
648 648
649 649 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
650 650
651 651 median_real = int(numpy.median(numpy.real(
652 652 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
653 653 median_imag = int(numpy.median(numpy.imag(
654 654 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
655 655 comp_mask_prof = [int(e) for e in comp_mask_prof]
656 656 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
657 657 median_real, median_imag)
658 658
659 659 for iprof in range(num_prof):
660 660 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
661 661 jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
662 662
663 663 # Removiendo la Interferencia
664 664 jcspectra[ip, :, ind_hei] = jcspectra[ip,
665 665 :, ind_hei] - jcspc_interf
666 666
667 667 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
668 668 maxid = ListAux.index(max(ListAux))
669 669
670 670 ind = numpy.array([-2, -1, 1, 2])
671 671 xx = numpy.zeros([4, 4])
672 672
673 673 for id1 in range(4):
674 674 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
675 675
676 676 xx_inv = numpy.linalg.inv(xx)
677 677 xx = xx_inv[:, 0]
678 678
679 679 ind = (ind + maxid + num_mask_prof) % num_mask_prof
680 680 yy = jcspectra[ip, mask_prof[ind], :]
681 681 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
682 682
683 683 # Guardar Resultados
684 684 self.dataOut.data_spc = jspectra
685 685 self.dataOut.data_cspc = jcspectra
686 686
687 687 return 1
688 688
689 689 def run(self, dataOut, interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None, mode=1):
690 690
691 691 self.dataOut = dataOut
692 692
693 693 if mode == 1:
694 694 self.removeInterference(interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None)
695 695 elif mode == 2:
696 696 self.removeInterference2()
697 697
698 698 return self.dataOut
699 699
700 700
701 701 class IncohInt(Operation):
702 702
703 703 __profIndex = 0
704 704 __withOverapping = False
705 705
706 706 __byTime = False
707 707 __initime = None
708 708 __lastdatatime = None
709 709 __integrationtime = None
710 710
711 711 __buffer_spc = None
712 712 __buffer_cspc = None
713 713 __buffer_dc = None
714 714
715 715 __dataReady = False
716 716
717 717 __timeInterval = None
718 718
719 719 n = None
720 720
721 721 def __init__(self):
722 722
723 723 Operation.__init__(self)
724 724
725 725 def setup(self, n=None, timeInterval=None, overlapping=False):
726 726 """
727 727 Set the parameters of the integration class.
728 728
729 729 Inputs:
730 730
731 731 n : Number of coherent integrations
732 732 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
733 733 overlapping :
734 734
735 735 """
736 736
737 737 self.__initime = None
738 738 self.__lastdatatime = 0
739 739
740 740 self.__buffer_spc = 0
741 741 self.__buffer_cspc = 0
742 742 self.__buffer_dc = 0
743 743
744 744 self.__profIndex = 0
745 745 self.__dataReady = False
746 746 self.__byTime = False
747 747
748 748 if n is None and timeInterval is None:
749 749 raise ValueError("n or timeInterval should be specified ...")
750 750
751 751 if n is not None:
752 752 self.n = int(n)
753 753 else:
754 754
755 755 self.__integrationtime = int(timeInterval)
756 756 self.n = None
757 757 self.__byTime = True
758 758
759 759 def putData(self, data_spc, data_cspc, data_dc):
760 760 """
761 761 Add a profile to the __buffer_spc and increase in one the __profileIndex
762 762
763 763 """
764 764
765 765 self.__buffer_spc += data_spc
766 766
767 767 if data_cspc is None:
768 768 self.__buffer_cspc = None
769 769 else:
770 770 self.__buffer_cspc += data_cspc
771 771
772 772 if data_dc is None:
773 773 self.__buffer_dc = None
774 774 else:
775 775 self.__buffer_dc += data_dc
776 776
777 777 self.__profIndex += 1
778 778
779 779 return
780 780
781 781 def pushData(self):
782 782 """
783 783 Return the sum of the last profiles and the profiles used in the sum.
784 784
785 785 Affected:
786 786
787 787 self.__profileIndex
788 788
789 789 """
790 790
791 791 data_spc = self.__buffer_spc
792 792 data_cspc = self.__buffer_cspc
793 793 data_dc = self.__buffer_dc
794 794 n = self.__profIndex
795 795
796 796 self.__buffer_spc = 0
797 797 self.__buffer_cspc = 0
798 798 self.__buffer_dc = 0
799 799 self.__profIndex = 0
800 800
801 801 return data_spc, data_cspc, data_dc, n
802 802
803 803 def byProfiles(self, *args):
804 804
805 805 self.__dataReady = False
806 806 avgdata_spc = None
807 807 avgdata_cspc = None
808 808 avgdata_dc = None
809 809
810 810 self.putData(*args)
811 811
812 812 if self.__profIndex == self.n:
813 813
814 814 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
815 815 self.n = n
816 816 self.__dataReady = True
817 817
818 818 return avgdata_spc, avgdata_cspc, avgdata_dc
819 819
820 820 def byTime(self, datatime, *args):
821 821
822 822 self.__dataReady = False
823 823 avgdata_spc = None
824 824 avgdata_cspc = None
825 825 avgdata_dc = None
826 826
827 827 self.putData(*args)
828 828
829 829 if (datatime - self.__initime) >= self.__integrationtime:
830 830 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
831 831 self.n = n
832 832 self.__dataReady = True
833 833
834 834 return avgdata_spc, avgdata_cspc, avgdata_dc
835 835
836 836 def integrate(self, datatime, *args):
837 837
838 838 if self.__profIndex == 0:
839 839 self.__initime = datatime
840 840
841 841 if self.__byTime:
842 842 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
843 843 datatime, *args)
844 844 else:
845 845 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
846 846
847 847 if not self.__dataReady:
848 848 return None, None, None, None
849 849
850 850 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
851 851
852 852 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
853 853 if n == 1:
854 854 return dataOut
855 855
856 856 dataOut.flagNoData = True
857 857
858 858 if not self.isConfig:
859 859 self.setup(n, timeInterval, overlapping)
860 860 self.isConfig = True
861 861
862 862 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
863 863 dataOut.data_spc,
864 864 dataOut.data_cspc,
865 865 dataOut.data_dc)
866 866
867 867 if self.__dataReady:
868 868
869 869 dataOut.data_spc = avgdata_spc
870 870 dataOut.data_cspc = avgdata_cspc
871 871 dataOut.data_dc = avgdata_dc
872 872 dataOut.nIncohInt *= self.n
873 873 dataOut.utctime = avgdatatime
874 874 dataOut.flagNoData = False
875 875
876 return dataOut No newline at end of file
876 return dataOut
877
878 class dopplerFlip(Operation):
879
880 def run(self, dataOut):
881 # arreglo 1: (num_chan, num_profiles, num_heights)
882 self.dataOut = dataOut
883 # JULIA-oblicua, indice 2
884 # arreglo 2: (num_profiles, num_heights)
885 jspectra = self.dataOut.data_spc[2]
886 jspectra_tmp = numpy.zeros(jspectra.shape)
887 num_profiles = jspectra.shape[0]
888 freq_dc = int(num_profiles / 2)
889 # Flip con for
890 for j in range(num_profiles):
891 jspectra_tmp[num_profiles-j-1]= jspectra[j]
892 # Intercambio perfil de DC con perfil inmediato anterior
893 jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1]
894 jspectra_tmp[freq_dc]= jspectra[freq_dc]
895 # canal modificado es re-escrito en el arreglo de canales
896 self.dataOut.data_spc[2] = jspectra_tmp
897
898 return self.dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1629 +1,1625
1 1 import sys
2 2 import numpy,math
3 3 from scipy import interpolate
4 4 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
5 5 from schainpy.model.data.jrodata import Voltage,hildebrand_sekhon
6 6 from schainpy.utils import log
7 7 from time import time
8 8
9 9
10 10
11 11 class VoltageProc(ProcessingUnit):
12 12
13 13 def __init__(self):
14 14
15 15 ProcessingUnit.__init__(self)
16 16
17 17 self.dataOut = Voltage()
18 18 self.flip = 1
19 19 self.setupReq = False
20 20
21 21 def run(self):
22 22
23 23 if self.dataIn.type == 'AMISR':
24 24 self.__updateObjFromAmisrInput()
25 25
26 26 if self.dataIn.type == 'Voltage':
27 27 self.dataOut.copy(self.dataIn)
28 28
29 29 def __updateObjFromAmisrInput(self):
30 30
31 31 self.dataOut.timeZone = self.dataIn.timeZone
32 32 self.dataOut.dstFlag = self.dataIn.dstFlag
33 33 self.dataOut.errorCount = self.dataIn.errorCount
34 34 self.dataOut.useLocalTime = self.dataIn.useLocalTime
35 35
36 36 self.dataOut.flagNoData = self.dataIn.flagNoData
37 37 self.dataOut.data = self.dataIn.data
38 38 self.dataOut.utctime = self.dataIn.utctime
39 39 self.dataOut.channelList = self.dataIn.channelList
40 40 #self.dataOut.timeInterval = self.dataIn.timeInterval
41 41 self.dataOut.heightList = self.dataIn.heightList
42 42 self.dataOut.nProfiles = self.dataIn.nProfiles
43 43
44 44 self.dataOut.nCohInt = self.dataIn.nCohInt
45 45 self.dataOut.ippSeconds = self.dataIn.ippSeconds
46 46 self.dataOut.frequency = self.dataIn.frequency
47 47
48 48 self.dataOut.azimuth = self.dataIn.azimuth
49 49 self.dataOut.zenith = self.dataIn.zenith
50 50
51 51 self.dataOut.beam.codeList = self.dataIn.beam.codeList
52 52 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
53 53 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
54 54
55 55
56 56 class selectChannels(Operation):
57 57
58 58 def run(self, dataOut, channelList):
59 59
60 60 channelIndexList = []
61 61 self.dataOut = dataOut
62 62 for channel in channelList:
63 63 if channel not in self.dataOut.channelList:
64 64 raise ValueError("Channel %d is not in %s" %(channel, str(self.dataOut.channelList)))
65 65
66 66 index = self.dataOut.channelList.index(channel)
67 67 channelIndexList.append(index)
68 68 self.selectChannelsByIndex(channelIndexList)
69 69 return self.dataOut
70 70
71 71 def selectChannelsByIndex(self, channelIndexList):
72 72 """
73 73 Selecciona un bloque de datos en base a canales segun el channelIndexList
74 74
75 75 Input:
76 76 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
77 77
78 78 Affected:
79 79 self.dataOut.data
80 80 self.dataOut.channelIndexList
81 81 self.dataOut.nChannels
82 82 self.dataOut.m_ProcessingHeader.totalSpectra
83 83 self.dataOut.systemHeaderObj.numChannels
84 84 self.dataOut.m_ProcessingHeader.blockSize
85 85
86 86 Return:
87 87 None
88 88 """
89 89
90 90 for channelIndex in channelIndexList:
91 91 if channelIndex not in self.dataOut.channelIndexList:
92 92 raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
93 93
94 94 if self.dataOut.type == 'Voltage':
95 95 if self.dataOut.flagDataAsBlock:
96 96 """
97 97 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
98 98 """
99 99 data = self.dataOut.data[channelIndexList,:,:]
100 100 else:
101 101 data = self.dataOut.data[channelIndexList,:]
102 102
103 103 self.dataOut.data = data
104 104 # self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
105 105 self.dataOut.channelList = range(len(channelIndexList))
106 106
107 107 elif self.dataOut.type == 'Spectra':
108 108 data_spc = self.dataOut.data_spc[channelIndexList, :]
109 109 data_dc = self.dataOut.data_dc[channelIndexList, :]
110 110
111 111 self.dataOut.data_spc = data_spc
112 112 self.dataOut.data_dc = data_dc
113 113
114 114 # self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
115 115 self.dataOut.channelList = range(len(channelIndexList))
116 116 self.__selectPairsByChannel(channelIndexList)
117 117
118 118 return 1
119 119
120 120 def __selectPairsByChannel(self, channelList=None):
121 121
122 122 if channelList == None:
123 123 return
124 124
125 125 pairsIndexListSelected = []
126 126 for pairIndex in self.dataOut.pairsIndexList:
127 127 # First pair
128 128 if self.dataOut.pairsList[pairIndex][0] not in channelList:
129 129 continue
130 130 # Second pair
131 131 if self.dataOut.pairsList[pairIndex][1] not in channelList:
132 132 continue
133 133
134 134 pairsIndexListSelected.append(pairIndex)
135 135
136 136 if not pairsIndexListSelected:
137 137 self.dataOut.data_cspc = None
138 138 self.dataOut.pairsList = []
139 139 return
140 140
141 141 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndexListSelected]
142 142 self.dataOut.pairsList = [self.dataOut.pairsList[i]
143 143 for i in pairsIndexListSelected]
144 144
145 145 return
146 146
147 147 class selectHeights(Operation):
148 148
149 def run(self, dataOut, minHei=None, maxHei=None):
149 def run(self, dataOut, minHei=None, maxHei=None, minIndex=None, maxIndex=None):
150 150 """
151 151 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
152 152 minHei <= height <= maxHei
153 153
154 154 Input:
155 155 minHei : valor minimo de altura a considerar
156 156 maxHei : valor maximo de altura a considerar
157 157
158 158 Affected:
159 159 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
160 160
161 161 Return:
162 162 1 si el metodo se ejecuto con exito caso contrario devuelve 0
163 163 """
164 164
165 165 self.dataOut = dataOut
166 166
167 if minHei == None:
168 minHei = self.dataOut.heightList[0]
167 if minHei and maxHei:
169 168
170 if maxHei == None:
171 maxHei = self.dataOut.heightList[-1]
169 if (minHei < self.dataOut.heightList[0]):
170 minHei = self.dataOut.heightList[0]
172 171
173 if (minHei < self.dataOut.heightList[0]):
174 minHei = self.dataOut.heightList[0]
172 if (maxHei > self.dataOut.heightList[-1]):
173 maxHei = self.dataOut.heightList[-1]
175 174
176 if (maxHei > self.dataOut.heightList[-1]):
177 maxHei = self.dataOut.heightList[-1]
178
179 minIndex = 0
180 maxIndex = 0
181 heights = self.dataOut.heightList
175 minIndex = 0
176 maxIndex = 0
177 heights = self.dataOut.heightList
182 178
183 inda = numpy.where(heights >= minHei)
184 indb = numpy.where(heights <= maxHei)
179 inda = numpy.where(heights >= minHei)
180 indb = numpy.where(heights <= maxHei)
185 181
186 try:
187 minIndex = inda[0][0]
188 except:
189 minIndex = 0
182 try:
183 minIndex = inda[0][0]
184 except:
185 minIndex = 0
190 186
191 try:
192 maxIndex = indb[0][-1]
193 except:
194 maxIndex = len(heights)
187 try:
188 maxIndex = indb[0][-1]
189 except:
190 maxIndex = len(heights)
195 191
196 192 self.selectHeightsByIndex(minIndex, maxIndex)
197 193
198 194 return self.dataOut
199 195
200 196 def selectHeightsByIndex(self, minIndex, maxIndex):
201 197 """
202 198 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
203 199 minIndex <= index <= maxIndex
204 200
205 201 Input:
206 202 minIndex : valor de indice minimo de altura a considerar
207 203 maxIndex : valor de indice maximo de altura a considerar
208 204
209 205 Affected:
210 206 self.dataOut.data
211 207 self.dataOut.heightList
212 208
213 209 Return:
214 210 1 si el metodo se ejecuto con exito caso contrario devuelve 0
215 211 """
216 212
217 213 if self.dataOut.type == 'Voltage':
218 214 if (minIndex < 0) or (minIndex > maxIndex):
219 215 raise ValueError("Height index range (%d,%d) is not valid" % (minIndex, maxIndex))
220 216
221 217 if (maxIndex >= self.dataOut.nHeights):
222 218 maxIndex = self.dataOut.nHeights
223 219
224 220 #voltage
225 221 if self.dataOut.flagDataAsBlock:
226 222 """
227 223 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
228 224 """
229 225 data = self.dataOut.data[:,:, minIndex:maxIndex]
230 226 else:
231 227 data = self.dataOut.data[:, minIndex:maxIndex]
232 228
233 229 # firstHeight = self.dataOut.heightList[minIndex]
234 230
235 231 self.dataOut.data = data
236 232 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
237 233
238 234 if self.dataOut.nHeights <= 1:
239 235 raise ValueError("selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights))
240 236 elif self.dataOut.type == 'Spectra':
241 237 if (minIndex < 0) or (minIndex > maxIndex):
242 238 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (
243 239 minIndex, maxIndex))
244 240
245 241 if (maxIndex >= self.dataOut.nHeights):
246 242 maxIndex = self.dataOut.nHeights - 1
247 243
248 244 # Spectra
249 245 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
250 246
251 247 data_cspc = None
252 248 if self.dataOut.data_cspc is not None:
253 249 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
254 250
255 251 data_dc = None
256 252 if self.dataOut.data_dc is not None:
257 253 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
258 254
259 255 self.dataOut.data_spc = data_spc
260 256 self.dataOut.data_cspc = data_cspc
261 257 self.dataOut.data_dc = data_dc
262 258
263 259 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
264 260
265 261 return 1
266 262
267 263
268 264 class filterByHeights(Operation):
269 265
270 266 def run(self, dataOut, window):
271 267
272 268 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
273 269
274 270 if window == None:
275 271 window = (dataOut.radarControllerHeaderObj.txA/dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
276 272
277 273 newdelta = deltaHeight * window
278 274 r = dataOut.nHeights % window
279 275 newheights = (dataOut.nHeights-r)/window
280 276
281 277 if newheights <= 1:
282 278 raise ValueError("filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(dataOut.nHeights, window))
283 279
284 280 if dataOut.flagDataAsBlock:
285 281 """
286 282 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
287 283 """
288 284 buffer = dataOut.data[:, :, 0:int(dataOut.nHeights-r)]
289 285 buffer = buffer.reshape(dataOut.nChannels, dataOut.nProfiles, int(dataOut.nHeights/window), window)
290 286 buffer = numpy.sum(buffer,3)
291 287
292 288 else:
293 289 buffer = dataOut.data[:,0:int(dataOut.nHeights-r)]
294 290 buffer = buffer.reshape(dataOut.nChannels,int(dataOut.nHeights/window),int(window))
295 291 buffer = numpy.sum(buffer,2)
296 292
297 293 dataOut.data = buffer
298 294 dataOut.heightList = dataOut.heightList[0] + numpy.arange( newheights )*newdelta
299 295 dataOut.windowOfFilter = window
300 296
301 297 return dataOut
302 298
303 299
304 300 class setH0(Operation):
305 301
306 302 def run(self, dataOut, h0, deltaHeight = None):
307 303
308 304 if not deltaHeight:
309 305 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
310 306
311 307 nHeights = dataOut.nHeights
312 308
313 309 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
314 310
315 311 dataOut.heightList = newHeiRange
316 312
317 313 return dataOut
318 314
319 315
320 316 class deFlip(Operation):
321 317
322 318 def run(self, dataOut, channelList = []):
323 319
324 320 data = dataOut.data.copy()
325 321
326 322 if dataOut.flagDataAsBlock:
327 323 flip = self.flip
328 324 profileList = list(range(dataOut.nProfiles))
329 325
330 326 if not channelList:
331 327 for thisProfile in profileList:
332 328 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
333 329 flip *= -1.0
334 330 else:
335 331 for thisChannel in channelList:
336 332 if thisChannel not in dataOut.channelList:
337 333 continue
338 334
339 335 for thisProfile in profileList:
340 336 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
341 337 flip *= -1.0
342 338
343 339 self.flip = flip
344 340
345 341 else:
346 342 if not channelList:
347 343 data[:,:] = data[:,:]*self.flip
348 344 else:
349 345 for thisChannel in channelList:
350 346 if thisChannel not in dataOut.channelList:
351 347 continue
352 348
353 349 data[thisChannel,:] = data[thisChannel,:]*self.flip
354 350
355 351 self.flip *= -1.
356 352
357 353 dataOut.data = data
358 354
359 355 return dataOut
360 356
361 357
362 358 class setAttribute(Operation):
363 359 '''
364 360 Set an arbitrary attribute(s) to dataOut
365 361 '''
366 362
367 363 def __init__(self):
368 364
369 365 Operation.__init__(self)
370 366 self._ready = False
371 367
372 368 def run(self, dataOut, **kwargs):
373 369
374 370 for key, value in kwargs.items():
375 371 setattr(dataOut, key, value)
376 372
377 373 return dataOut
378 374
379 375
380 376 @MPDecorator
381 377 class printAttribute(Operation):
382 378 '''
383 379 Print an arbitrary attribute of dataOut
384 380 '''
385 381
386 382 def __init__(self):
387 383
388 384 Operation.__init__(self)
389 385
390 386 def run(self, dataOut, attributes):
391 387
392 388 if isinstance(attributes, str):
393 389 attributes = [attributes]
394 390 for attr in attributes:
395 391 if hasattr(dataOut, attr):
396 392 log.log(getattr(dataOut, attr), attr)
397 393
398 394
399 395 class interpolateHeights(Operation):
400 396
401 397 def run(self, dataOut, topLim, botLim):
402 398 #69 al 72 para julia
403 399 #82-84 para meteoros
404 400 if len(numpy.shape(dataOut.data))==2:
405 401 sampInterp = (dataOut.data[:,botLim-1] + dataOut.data[:,topLim+1])/2
406 402 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
407 403 #dataOut.data[:,botLim:limSup+1] = sampInterp
408 404 dataOut.data[:,botLim:topLim+1] = sampInterp
409 405 else:
410 406 nHeights = dataOut.data.shape[2]
411 407 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
412 408 y = dataOut.data[:,:,list(range(botLim))+list(range(topLim+1,nHeights))]
413 409 f = interpolate.interp1d(x, y, axis = 2)
414 410 xnew = numpy.arange(botLim,topLim+1)
415 411 ynew = f(xnew)
416 412 dataOut.data[:,:,botLim:topLim+1] = ynew
417 413
418 414 return dataOut
419 415
420 416
421 417 class CohInt(Operation):
422 418
423 419 isConfig = False
424 420 __profIndex = 0
425 421 __byTime = False
426 422 __initime = None
427 423 __lastdatatime = None
428 424 __integrationtime = None
429 425 __buffer = None
430 426 __bufferStride = []
431 427 __dataReady = False
432 428 __profIndexStride = 0
433 429 __dataToPutStride = False
434 430 n = None
435 431
436 432 def __init__(self, **kwargs):
437 433
438 434 Operation.__init__(self, **kwargs)
439 435
440 436 def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False):
441 437 """
442 438 Set the parameters of the integration class.
443 439
444 440 Inputs:
445 441
446 442 n : Number of coherent integrations
447 443 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
448 444 overlapping :
449 445 """
450 446
451 447 self.__initime = None
452 448 self.__lastdatatime = 0
453 449 self.__buffer = None
454 450 self.__dataReady = False
455 451 self.byblock = byblock
456 452 self.stride = stride
457 453
458 454 if n == None and timeInterval == None:
459 455 raise ValueError("n or timeInterval should be specified ...")
460 456
461 457 if n != None:
462 458 self.n = n
463 459 self.__byTime = False
464 460 else:
465 461 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
466 462 self.n = 9999
467 463 self.__byTime = True
468 464
469 465 if overlapping:
470 466 self.__withOverlapping = True
471 467 self.__buffer = None
472 468 else:
473 469 self.__withOverlapping = False
474 470 self.__buffer = 0
475 471
476 472 self.__profIndex = 0
477 473
478 474 def putData(self, data):
479 475
480 476 """
481 477 Add a profile to the __buffer and increase in one the __profileIndex
482 478
483 479 """
484 480
485 481 if not self.__withOverlapping:
486 482 self.__buffer += data.copy()
487 483 self.__profIndex += 1
488 484 return
489 485
490 486 #Overlapping data
491 487 nChannels, nHeis = data.shape
492 488 data = numpy.reshape(data, (1, nChannels, nHeis))
493 489
494 490 #If the buffer is empty then it takes the data value
495 491 if self.__buffer is None:
496 492 self.__buffer = data
497 493 self.__profIndex += 1
498 494 return
499 495
500 496 #If the buffer length is lower than n then stakcing the data value
501 497 if self.__profIndex < self.n:
502 498 self.__buffer = numpy.vstack((self.__buffer, data))
503 499 self.__profIndex += 1
504 500 return
505 501
506 502 #If the buffer length is equal to n then replacing the last buffer value with the data value
507 503 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
508 504 self.__buffer[self.n-1] = data
509 505 self.__profIndex = self.n
510 506 return
511 507
512 508
513 509 def pushData(self):
514 510 """
515 511 Return the sum of the last profiles and the profiles used in the sum.
516 512
517 513 Affected:
518 514
519 515 self.__profileIndex
520 516
521 517 """
522 518
523 519 if not self.__withOverlapping:
524 520 data = self.__buffer
525 521 n = self.__profIndex
526 522
527 523 self.__buffer = 0
528 524 self.__profIndex = 0
529 525
530 526 return data, n
531 527
532 528 #Integration with Overlapping
533 529 data = numpy.sum(self.__buffer, axis=0)
534 530 # print data
535 531 # raise
536 532 n = self.__profIndex
537 533
538 534 return data, n
539 535
540 536 def byProfiles(self, data):
541 537
542 538 self.__dataReady = False
543 539 avgdata = None
544 540 # n = None
545 541 # print data
546 542 # raise
547 543 self.putData(data)
548 544
549 545 if self.__profIndex == self.n:
550 546 avgdata, n = self.pushData()
551 547 self.__dataReady = True
552 548
553 549 return avgdata
554 550
555 551 def byTime(self, data, datatime):
556 552
557 553 self.__dataReady = False
558 554 avgdata = None
559 555 n = None
560 556
561 557 self.putData(data)
562 558
563 559 if (datatime - self.__initime) >= self.__integrationtime:
564 560 avgdata, n = self.pushData()
565 561 self.n = n
566 562 self.__dataReady = True
567 563
568 564 return avgdata
569 565
570 566 def integrateByStride(self, data, datatime):
571 567 # print data
572 568 if self.__profIndex == 0:
573 569 self.__buffer = [[data.copy(), datatime]]
574 570 else:
575 571 self.__buffer.append([data.copy(),datatime])
576 572 self.__profIndex += 1
577 573 self.__dataReady = False
578 574
579 575 if self.__profIndex == self.n * self.stride :
580 576 self.__dataToPutStride = True
581 577 self.__profIndexStride = 0
582 578 self.__profIndex = 0
583 579 self.__bufferStride = []
584 580 for i in range(self.stride):
585 581 current = self.__buffer[i::self.stride]
586 582 data = numpy.sum([t[0] for t in current], axis=0)
587 583 avgdatatime = numpy.average([t[1] for t in current])
588 584 # print data
589 585 self.__bufferStride.append((data, avgdatatime))
590 586
591 587 if self.__dataToPutStride:
592 588 self.__dataReady = True
593 589 self.__profIndexStride += 1
594 590 if self.__profIndexStride == self.stride:
595 591 self.__dataToPutStride = False
596 592 # print self.__bufferStride[self.__profIndexStride - 1]
597 593 # raise
598 594 return self.__bufferStride[self.__profIndexStride - 1]
599 595
600 596
601 597 return None, None
602 598
603 599 def integrate(self, data, datatime=None):
604 600
605 601 if self.__initime == None:
606 602 self.__initime = datatime
607 603
608 604 if self.__byTime:
609 605 avgdata = self.byTime(data, datatime)
610 606 else:
611 607 avgdata = self.byProfiles(data)
612 608
613 609
614 610 self.__lastdatatime = datatime
615 611
616 612 if avgdata is None:
617 613 return None, None
618 614
619 615 avgdatatime = self.__initime
620 616
621 617 deltatime = datatime - self.__lastdatatime
622 618
623 619 if not self.__withOverlapping:
624 620 self.__initime = datatime
625 621 else:
626 622 self.__initime += deltatime
627 623
628 624 return avgdata, avgdatatime
629 625
630 626 def integrateByBlock(self, dataOut):
631 627
632 628 times = int(dataOut.data.shape[1]/self.n)
633 629 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
634 630
635 631 id_min = 0
636 632 id_max = self.n
637 633
638 634 for i in range(times):
639 635 junk = dataOut.data[:,id_min:id_max,:]
640 636 avgdata[:,i,:] = junk.sum(axis=1)
641 637 id_min += self.n
642 638 id_max += self.n
643 639
644 640 timeInterval = dataOut.ippSeconds*self.n
645 641 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
646 642 self.__dataReady = True
647 643 return avgdata, avgdatatime
648 644
649 645 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
650 646
651 647 if not self.isConfig:
652 648 self.setup(n=n, stride=stride, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
653 649 self.isConfig = True
654 650
655 651 if dataOut.flagDataAsBlock:
656 652 """
657 653 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
658 654 """
659 655 avgdata, avgdatatime = self.integrateByBlock(dataOut)
660 656 dataOut.nProfiles /= self.n
661 657 else:
662 658 if stride is None:
663 659 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
664 660 else:
665 661 avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
666 662
667 663
668 664 # dataOut.timeInterval *= n
669 665 dataOut.flagNoData = True
670 666
671 667 if self.__dataReady:
672 668 dataOut.data = avgdata
673 669 if not dataOut.flagCohInt:
674 670 dataOut.nCohInt *= self.n
675 671 dataOut.flagCohInt = True
676 672 dataOut.utctime = avgdatatime
677 673 # print avgdata, avgdatatime
678 674 # raise
679 675 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
680 676 dataOut.flagNoData = False
681 677 return dataOut
682 678
683 679 class Decoder(Operation):
684 680
685 681 isConfig = False
686 682 __profIndex = 0
687 683
688 684 code = None
689 685
690 686 nCode = None
691 687 nBaud = None
692 688
693 689 def __init__(self, **kwargs):
694 690
695 691 Operation.__init__(self, **kwargs)
696 692
697 693 self.times = None
698 694 self.osamp = None
699 695 # self.__setValues = False
700 696 self.isConfig = False
701 697 self.setupReq = False
702 698 def setup(self, code, osamp, dataOut):
703 699
704 700 self.__profIndex = 0
705 701
706 702 self.code = code
707 703
708 704 self.nCode = len(code)
709 705 self.nBaud = len(code[0])
710 706
711 707 if (osamp != None) and (osamp >1):
712 708 self.osamp = osamp
713 709 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
714 710 self.nBaud = self.nBaud*self.osamp
715 711
716 712 self.__nChannels = dataOut.nChannels
717 713 self.__nProfiles = dataOut.nProfiles
718 714 self.__nHeis = dataOut.nHeights
719 715
720 716 if self.__nHeis < self.nBaud:
721 717 raise ValueError('Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud))
722 718
723 719 #Frequency
724 720 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
725 721
726 722 __codeBuffer[:,0:self.nBaud] = self.code
727 723
728 724 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
729 725
730 726 if dataOut.flagDataAsBlock:
731 727
732 728 self.ndatadec = self.__nHeis #- self.nBaud + 1
733 729
734 730 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
735 731
736 732 else:
737 733
738 734 #Time
739 735 self.ndatadec = self.__nHeis #- self.nBaud + 1
740 736
741 737 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
742 738
743 739 def __convolutionInFreq(self, data):
744 740
745 741 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
746 742
747 743 fft_data = numpy.fft.fft(data, axis=1)
748 744
749 745 conv = fft_data*fft_code
750 746
751 747 data = numpy.fft.ifft(conv,axis=1)
752 748
753 749 return data
754 750
755 751 def __convolutionInFreqOpt(self, data):
756 752
757 753 raise NotImplementedError
758 754
759 755 def __convolutionInTime(self, data):
760 756
761 757 code = self.code[self.__profIndex]
762 758 for i in range(self.__nChannels):
763 759 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
764 760
765 761 return self.datadecTime
766 762
767 763 def __convolutionByBlockInTime(self, data):
768 764
769 765 repetitions = int(self.__nProfiles / self.nCode)
770 766 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
771 767 junk = junk.flatten()
772 768 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
773 769 profilesList = range(self.__nProfiles)
774 770
775 771 for i in range(self.__nChannels):
776 772 for j in profilesList:
777 773 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
778 774 return self.datadecTime
779 775
780 776 def __convolutionByBlockInFreq(self, data):
781 777
782 778 raise NotImplementedError("Decoder by frequency fro Blocks not implemented")
783 779
784 780
785 781 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
786 782
787 783 fft_data = numpy.fft.fft(data, axis=2)
788 784
789 785 conv = fft_data*fft_code
790 786
791 787 data = numpy.fft.ifft(conv,axis=2)
792 788
793 789 return data
794 790
795 791
796 792 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
797 793
798 794 if dataOut.flagDecodeData:
799 795 print("This data is already decoded, recoding again ...")
800 796
801 797 if not self.isConfig:
802 798
803 799 if code is None:
804 800 if dataOut.code is None:
805 801 raise ValueError("Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type)
806 802
807 803 code = dataOut.code
808 804 else:
809 805 code = numpy.array(code).reshape(nCode,nBaud)
810 806 self.setup(code, osamp, dataOut)
811 807
812 808 self.isConfig = True
813 809
814 810 if mode == 3:
815 811 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
816 812
817 813 if times != None:
818 814 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
819 815
820 816 if self.code is None:
821 817 print("Fail decoding: Code is not defined.")
822 818 return
823 819
824 820 self.__nProfiles = dataOut.nProfiles
825 821 datadec = None
826 822
827 823 if mode == 3:
828 824 mode = 0
829 825
830 826 if dataOut.flagDataAsBlock:
831 827 """
832 828 Decoding when data have been read as block,
833 829 """
834 830
835 831 if mode == 0:
836 832 datadec = self.__convolutionByBlockInTime(dataOut.data)
837 833 if mode == 1:
838 834 datadec = self.__convolutionByBlockInFreq(dataOut.data)
839 835 else:
840 836 """
841 837 Decoding when data have been read profile by profile
842 838 """
843 839 if mode == 0:
844 840 datadec = self.__convolutionInTime(dataOut.data)
845 841
846 842 if mode == 1:
847 843 datadec = self.__convolutionInFreq(dataOut.data)
848 844
849 845 if mode == 2:
850 846 datadec = self.__convolutionInFreqOpt(dataOut.data)
851 847
852 848 if datadec is None:
853 849 raise ValueError("Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode)
854 850
855 851 dataOut.code = self.code
856 852 dataOut.nCode = self.nCode
857 853 dataOut.nBaud = self.nBaud
858 854
859 855 dataOut.data = datadec
860 856
861 857 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
862 858
863 859 dataOut.flagDecodeData = True #asumo q la data esta decodificada
864 860
865 861 if self.__profIndex == self.nCode-1:
866 862 self.__profIndex = 0
867 863 return dataOut
868 864
869 865 self.__profIndex += 1
870 866
871 867 return dataOut
872 868 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
873 869
874 870
875 871 class ProfileConcat(Operation):
876 872
877 873 isConfig = False
878 874 buffer = None
879 875
880 876 def __init__(self, **kwargs):
881 877
882 878 Operation.__init__(self, **kwargs)
883 879 self.profileIndex = 0
884 880
885 881 def reset(self):
886 882 self.buffer = numpy.zeros_like(self.buffer)
887 883 self.start_index = 0
888 884 self.times = 1
889 885
890 886 def setup(self, data, m, n=1):
891 887 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
892 888 self.nHeights = data.shape[1]#.nHeights
893 889 self.start_index = 0
894 890 self.times = 1
895 891
896 892 def concat(self, data):
897 893
898 894 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
899 895 self.start_index = self.start_index + self.nHeights
900 896
901 897 def run(self, dataOut, m):
902 898 dataOut.flagNoData = True
903 899
904 900 if not self.isConfig:
905 901 self.setup(dataOut.data, m, 1)
906 902 self.isConfig = True
907 903
908 904 if dataOut.flagDataAsBlock:
909 905 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
910 906
911 907 else:
912 908 self.concat(dataOut.data)
913 909 self.times += 1
914 910 if self.times > m:
915 911 dataOut.data = self.buffer
916 912 self.reset()
917 913 dataOut.flagNoData = False
918 914 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
919 915 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
920 916 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
921 917 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
922 918 dataOut.ippSeconds *= m
923 919 return dataOut
924 920
925 921 class ProfileSelector(Operation):
926 922
927 923 profileIndex = None
928 924 # Tamanho total de los perfiles
929 925 nProfiles = None
930 926
931 927 def __init__(self, **kwargs):
932 928
933 929 Operation.__init__(self, **kwargs)
934 930 self.profileIndex = 0
935 931
936 932 def incProfileIndex(self):
937 933
938 934 self.profileIndex += 1
939 935
940 936 if self.profileIndex >= self.nProfiles:
941 937 self.profileIndex = 0
942 938
943 939 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
944 940
945 941 if profileIndex < minIndex:
946 942 return False
947 943
948 944 if profileIndex > maxIndex:
949 945 return False
950 946
951 947 return True
952 948
953 949 def isThisProfileInList(self, profileIndex, profileList):
954 950
955 951 if profileIndex not in profileList:
956 952 return False
957 953
958 954 return True
959 955
960 956 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
961 957
962 958 """
963 959 ProfileSelector:
964 960
965 961 Inputs:
966 962 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
967 963
968 964 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
969 965
970 966 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
971 967
972 968 """
973 969
974 970 if rangeList is not None:
975 971 if type(rangeList[0]) not in (tuple, list):
976 972 rangeList = [rangeList]
977 973
978 974 dataOut.flagNoData = True
979 975
980 976 if dataOut.flagDataAsBlock:
981 977 """
982 978 data dimension = [nChannels, nProfiles, nHeis]
983 979 """
984 980 if profileList != None:
985 981 dataOut.data = dataOut.data[:,profileList,:]
986 982
987 983 if profileRangeList != None:
988 984 minIndex = profileRangeList[0]
989 985 maxIndex = profileRangeList[1]
990 986 profileList = list(range(minIndex, maxIndex+1))
991 987
992 988 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
993 989
994 990 if rangeList != None:
995 991
996 992 profileList = []
997 993
998 994 for thisRange in rangeList:
999 995 minIndex = thisRange[0]
1000 996 maxIndex = thisRange[1]
1001 997
1002 998 profileList.extend(list(range(minIndex, maxIndex+1)))
1003 999
1004 1000 dataOut.data = dataOut.data[:,profileList,:]
1005 1001
1006 1002 dataOut.nProfiles = len(profileList)
1007 1003 dataOut.profileIndex = dataOut.nProfiles - 1
1008 1004 dataOut.flagNoData = False
1009 1005
1010 1006 return dataOut
1011 1007
1012 1008 """
1013 1009 data dimension = [nChannels, nHeis]
1014 1010 """
1015 1011
1016 1012 if profileList != None:
1017 1013
1018 1014 if self.isThisProfileInList(dataOut.profileIndex, profileList):
1019 1015
1020 1016 self.nProfiles = len(profileList)
1021 1017 dataOut.nProfiles = self.nProfiles
1022 1018 dataOut.profileIndex = self.profileIndex
1023 1019 dataOut.flagNoData = False
1024 1020
1025 1021 self.incProfileIndex()
1026 1022 return dataOut
1027 1023
1028 1024 if profileRangeList != None:
1029 1025
1030 1026 minIndex = profileRangeList[0]
1031 1027 maxIndex = profileRangeList[1]
1032 1028
1033 1029 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1034 1030
1035 1031 self.nProfiles = maxIndex - minIndex + 1
1036 1032 dataOut.nProfiles = self.nProfiles
1037 1033 dataOut.profileIndex = self.profileIndex
1038 1034 dataOut.flagNoData = False
1039 1035
1040 1036 self.incProfileIndex()
1041 1037 return dataOut
1042 1038
1043 1039 if rangeList != None:
1044 1040
1045 1041 nProfiles = 0
1046 1042
1047 1043 for thisRange in rangeList:
1048 1044 minIndex = thisRange[0]
1049 1045 maxIndex = thisRange[1]
1050 1046
1051 1047 nProfiles += maxIndex - minIndex + 1
1052 1048
1053 1049 for thisRange in rangeList:
1054 1050
1055 1051 minIndex = thisRange[0]
1056 1052 maxIndex = thisRange[1]
1057 1053
1058 1054 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1059 1055
1060 1056 self.nProfiles = nProfiles
1061 1057 dataOut.nProfiles = self.nProfiles
1062 1058 dataOut.profileIndex = self.profileIndex
1063 1059 dataOut.flagNoData = False
1064 1060
1065 1061 self.incProfileIndex()
1066 1062
1067 1063 break
1068 1064
1069 1065 return dataOut
1070 1066
1071 1067
1072 1068 if beam != None: #beam is only for AMISR data
1073 1069 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
1074 1070 dataOut.flagNoData = False
1075 1071 dataOut.profileIndex = self.profileIndex
1076 1072
1077 1073 self.incProfileIndex()
1078 1074
1079 1075 return dataOut
1080 1076
1081 1077 raise ValueError("ProfileSelector needs profileList, profileRangeList or rangeList parameter")
1082 1078
1083 1079
1084 1080 class Reshaper(Operation):
1085 1081
1086 1082 def __init__(self, **kwargs):
1087 1083
1088 1084 Operation.__init__(self, **kwargs)
1089 1085
1090 1086 self.__buffer = None
1091 1087 self.__nitems = 0
1092 1088
1093 1089 def __appendProfile(self, dataOut, nTxs):
1094 1090
1095 1091 if self.__buffer is None:
1096 1092 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
1097 1093 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
1098 1094
1099 1095 ini = dataOut.nHeights * self.__nitems
1100 1096 end = ini + dataOut.nHeights
1101 1097
1102 1098 self.__buffer[:, ini:end] = dataOut.data
1103 1099
1104 1100 self.__nitems += 1
1105 1101
1106 1102 return int(self.__nitems*nTxs)
1107 1103
1108 1104 def __getBuffer(self):
1109 1105
1110 1106 if self.__nitems == int(1./self.__nTxs):
1111 1107
1112 1108 self.__nitems = 0
1113 1109
1114 1110 return self.__buffer.copy()
1115 1111
1116 1112 return None
1117 1113
1118 1114 def __checkInputs(self, dataOut, shape, nTxs):
1119 1115
1120 1116 if shape is None and nTxs is None:
1121 1117 raise ValueError("Reshaper: shape of factor should be defined")
1122 1118
1123 1119 if nTxs:
1124 1120 if nTxs < 0:
1125 1121 raise ValueError("nTxs should be greater than 0")
1126 1122
1127 1123 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
1128 1124 raise ValueError("nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs)))
1129 1125
1130 1126 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
1131 1127
1132 1128 return shape, nTxs
1133 1129
1134 1130 if len(shape) != 2 and len(shape) != 3:
1135 1131 raise ValueError("shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights))
1136 1132
1137 1133 if len(shape) == 2:
1138 1134 shape_tuple = [dataOut.nChannels]
1139 1135 shape_tuple.extend(shape)
1140 1136 else:
1141 1137 shape_tuple = list(shape)
1142 1138
1143 1139 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1144 1140
1145 1141 return shape_tuple, nTxs
1146 1142
1147 1143 def run(self, dataOut, shape=None, nTxs=None):
1148 1144
1149 1145 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1150 1146
1151 1147 dataOut.flagNoData = True
1152 1148 profileIndex = None
1153 1149
1154 1150 if dataOut.flagDataAsBlock:
1155 1151
1156 1152 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1157 1153 dataOut.flagNoData = False
1158 1154
1159 1155 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1160 1156
1161 1157 else:
1162 1158
1163 1159 if self.__nTxs < 1:
1164 1160
1165 1161 self.__appendProfile(dataOut, self.__nTxs)
1166 1162 new_data = self.__getBuffer()
1167 1163
1168 1164 if new_data is not None:
1169 1165 dataOut.data = new_data
1170 1166 dataOut.flagNoData = False
1171 1167
1172 1168 profileIndex = dataOut.profileIndex*nTxs
1173 1169
1174 1170 else:
1175 1171 raise ValueError("nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)")
1176 1172
1177 1173 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1178 1174
1179 1175 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1180 1176
1181 1177 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1182 1178
1183 1179 dataOut.profileIndex = profileIndex
1184 1180
1185 1181 dataOut.ippSeconds /= self.__nTxs
1186 1182
1187 1183 return dataOut
1188 1184
1189 1185 class SplitProfiles(Operation):
1190 1186
1191 1187 def __init__(self, **kwargs):
1192 1188
1193 1189 Operation.__init__(self, **kwargs)
1194 1190
1195 1191 def run(self, dataOut, n):
1196 1192
1197 1193 dataOut.flagNoData = True
1198 1194 profileIndex = None
1199 1195
1200 1196 if dataOut.flagDataAsBlock:
1201 1197
1202 1198 #nchannels, nprofiles, nsamples
1203 1199 shape = dataOut.data.shape
1204 1200
1205 1201 if shape[2] % n != 0:
1206 1202 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
1207 1203
1208 1204 new_shape = shape[0], shape[1]*n, int(shape[2]/n)
1209 1205
1210 1206 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1211 1207 dataOut.flagNoData = False
1212 1208
1213 1209 profileIndex = int(dataOut.nProfiles/n) - 1
1214 1210
1215 1211 else:
1216 1212
1217 1213 raise ValueError("Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)")
1218 1214
1219 1215 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1220 1216
1221 1217 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1222 1218
1223 1219 dataOut.nProfiles = int(dataOut.nProfiles*n)
1224 1220
1225 1221 dataOut.profileIndex = profileIndex
1226 1222
1227 1223 dataOut.ippSeconds /= n
1228 1224
1229 1225 return dataOut
1230 1226
1231 1227 class CombineProfiles(Operation):
1232 1228 def __init__(self, **kwargs):
1233 1229
1234 1230 Operation.__init__(self, **kwargs)
1235 1231
1236 1232 self.__remData = None
1237 1233 self.__profileIndex = 0
1238 1234
1239 1235 def run(self, dataOut, n):
1240 1236
1241 1237 dataOut.flagNoData = True
1242 1238 profileIndex = None
1243 1239
1244 1240 if dataOut.flagDataAsBlock:
1245 1241
1246 1242 #nchannels, nprofiles, nsamples
1247 1243 shape = dataOut.data.shape
1248 1244 new_shape = shape[0], shape[1]/n, shape[2]*n
1249 1245
1250 1246 if shape[1] % n != 0:
1251 1247 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[1]))
1252 1248
1253 1249 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1254 1250 dataOut.flagNoData = False
1255 1251
1256 1252 profileIndex = int(dataOut.nProfiles*n) - 1
1257 1253
1258 1254 else:
1259 1255
1260 1256 #nchannels, nsamples
1261 1257 if self.__remData is None:
1262 1258 newData = dataOut.data
1263 1259 else:
1264 1260 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1265 1261
1266 1262 self.__profileIndex += 1
1267 1263
1268 1264 if self.__profileIndex < n:
1269 1265 self.__remData = newData
1270 1266 #continue
1271 1267 return
1272 1268
1273 1269 self.__profileIndex = 0
1274 1270 self.__remData = None
1275 1271
1276 1272 dataOut.data = newData
1277 1273 dataOut.flagNoData = False
1278 1274
1279 1275 profileIndex = dataOut.profileIndex/n
1280 1276
1281 1277
1282 1278 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1283 1279
1284 1280 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1285 1281
1286 1282 dataOut.nProfiles = int(dataOut.nProfiles/n)
1287 1283
1288 1284 dataOut.profileIndex = profileIndex
1289 1285
1290 1286 dataOut.ippSeconds *= n
1291 1287
1292 1288 return dataOut
1293 1289
1294 1290 class PulsePairVoltage(Operation):
1295 1291 '''
1296 1292 Function PulsePair(Signal Power, Velocity)
1297 1293 The real component of Lag[0] provides Intensity Information
1298 1294 The imag component of Lag[1] Phase provides Velocity Information
1299 1295
1300 1296 Configuration Parameters:
1301 1297 nPRF = Number of Several PRF
1302 1298 theta = Degree Azimuth angel Boundaries
1303 1299
1304 1300 Input:
1305 1301 self.dataOut
1306 1302 lag[N]
1307 1303 Affected:
1308 1304 self.dataOut.spc
1309 1305 '''
1310 1306 isConfig = False
1311 1307 __profIndex = 0
1312 1308 __initime = None
1313 1309 __lastdatatime = None
1314 1310 __buffer = None
1315 1311 noise = None
1316 1312 __dataReady = False
1317 1313 n = None
1318 1314 __nch = 0
1319 1315 __nHeis = 0
1320 1316 removeDC = False
1321 1317 ipp = None
1322 1318 lambda_ = 0
1323 1319
1324 1320 def __init__(self,**kwargs):
1325 1321 Operation.__init__(self,**kwargs)
1326 1322
1327 1323 def setup(self, dataOut, n = None, removeDC=False):
1328 1324 '''
1329 1325 n= Numero de PRF's de entrada
1330 1326 '''
1331 1327 self.__initime = None
1332 1328 self.__lastdatatime = 0
1333 1329 self.__dataReady = False
1334 1330 self.__buffer = 0
1335 1331 self.__profIndex = 0
1336 1332 self.noise = None
1337 1333 self.__nch = dataOut.nChannels
1338 1334 self.__nHeis = dataOut.nHeights
1339 1335 self.removeDC = removeDC
1340 1336 self.lambda_ = 3.0e8/(9345.0e6)
1341 1337 self.ippSec = dataOut.ippSeconds
1342 1338 self.nCohInt = dataOut.nCohInt
1343 1339 print("IPPseconds",dataOut.ippSeconds)
1344 1340
1345 1341 print("ELVALOR DE n es:", n)
1346 1342 if n == None:
1347 1343 raise ValueError("n should be specified.")
1348 1344
1349 1345 if n != None:
1350 1346 if n<2:
1351 1347 raise ValueError("n should be greater than 2")
1352 1348
1353 1349 self.n = n
1354 1350 self.__nProf = n
1355 1351
1356 1352 self.__buffer = numpy.zeros((dataOut.nChannels,
1357 1353 n,
1358 1354 dataOut.nHeights),
1359 1355 dtype='complex')
1360 1356
1361 1357 def putData(self,data):
1362 1358 '''
1363 1359 Add a profile to he __buffer and increase in one the __profiel Index
1364 1360 '''
1365 1361 self.__buffer[:,self.__profIndex,:]= data
1366 1362 self.__profIndex += 1
1367 1363 return
1368 1364
1369 1365 def pushData(self,dataOut):
1370 1366 '''
1371 1367 Return the PULSEPAIR and the profiles used in the operation
1372 1368 Affected : self.__profileIndex
1373 1369 '''
1374 1370 #----------------- Remove DC-----------------------------------
1375 1371 if self.removeDC==True:
1376 1372 mean = numpy.mean(self.__buffer,1)
1377 1373 tmp = mean.reshape(self.__nch,1,self.__nHeis)
1378 1374 dc= numpy.tile(tmp,[1,self.__nProf,1])
1379 1375 self.__buffer = self.__buffer - dc
1380 1376 #------------------Calculo de Potencia ------------------------
1381 1377 pair0 = self.__buffer*numpy.conj(self.__buffer)
1382 1378 pair0 = pair0.real
1383 1379 lag_0 = numpy.sum(pair0,1)
1384 1380 #------------------Calculo de Ruido x canal--------------------
1385 1381 self.noise = numpy.zeros(self.__nch)
1386 1382 for i in range(self.__nch):
1387 1383 daux = numpy.sort(pair0[i,:,:],axis= None)
1388 1384 self.noise[i]=hildebrand_sekhon( daux ,self.nCohInt)
1389 1385
1390 1386 self.noise = self.noise.reshape(self.__nch,1)
1391 1387 self.noise = numpy.tile(self.noise,[1,self.__nHeis])
1392 1388 noise_buffer = self.noise.reshape(self.__nch,1,self.__nHeis)
1393 1389 noise_buffer = numpy.tile(noise_buffer,[1,self.__nProf,1])
1394 1390 #------------------ Potencia recibida= P , Potencia senal = S , Ruido= N--
1395 1391 #------------------ P= S+N ,P=lag_0/N ---------------------------------
1396 1392 #-------------------- Power --------------------------------------------------
1397 1393 data_power = lag_0/(self.n*self.nCohInt)
1398 1394 #------------------ Senal ---------------------------------------------------
1399 1395 data_intensity = pair0 - noise_buffer
1400 1396 data_intensity = numpy.sum(data_intensity,axis=1)*(self.n*self.nCohInt)#*self.nCohInt)
1401 1397 #data_intensity = (lag_0-self.noise*self.n)*(self.n*self.nCohInt)
1402 1398 for i in range(self.__nch):
1403 1399 for j in range(self.__nHeis):
1404 1400 if data_intensity[i][j] < 0:
1405 1401 data_intensity[i][j] = numpy.min(numpy.absolute(data_intensity[i][j]))
1406 1402
1407 1403 #----------------- Calculo de Frecuencia y Velocidad doppler--------
1408 1404 pair1 = self.__buffer[:,:-1,:]*numpy.conjugate(self.__buffer[:,1:,:])
1409 1405 lag_1 = numpy.sum(pair1,1)
1410 1406 data_freq = (-1/(2.0*math.pi*self.ippSec*self.nCohInt))*numpy.angle(lag_1)
1411 1407 data_velocity = (self.lambda_/2.0)*data_freq
1412 1408
1413 1409 #---------------- Potencia promedio estimada de la Senal-----------
1414 1410 lag_0 = lag_0/self.n
1415 1411 S = lag_0-self.noise
1416 1412
1417 1413 #---------------- Frecuencia Doppler promedio ---------------------
1418 1414 lag_1 = lag_1/(self.n-1)
1419 1415 R1 = numpy.abs(lag_1)
1420 1416
1421 1417 #---------------- Calculo del SNR----------------------------------
1422 1418 data_snrPP = S/self.noise
1423 1419 for i in range(self.__nch):
1424 1420 for j in range(self.__nHeis):
1425 1421 if data_snrPP[i][j] < 1.e-20:
1426 1422 data_snrPP[i][j] = 1.e-20
1427 1423
1428 1424 #----------------- Calculo del ancho espectral ----------------------
1429 1425 L = S/R1
1430 1426 L = numpy.where(L<0,1,L)
1431 1427 L = numpy.log(L)
1432 1428 tmp = numpy.sqrt(numpy.absolute(L))
1433 1429 data_specwidth = (self.lambda_/(2*math.sqrt(2)*math.pi*self.ippSec*self.nCohInt))*tmp*numpy.sign(L)
1434 1430 n = self.__profIndex
1435 1431
1436 1432 self.__buffer = numpy.zeros((self.__nch, self.__nProf,self.__nHeis), dtype='complex')
1437 1433 self.__profIndex = 0
1438 1434 return data_power,data_intensity,data_velocity,data_snrPP,data_specwidth,n
1439 1435
1440 1436
1441 1437 def pulsePairbyProfiles(self,dataOut):
1442 1438
1443 1439 self.__dataReady = False
1444 1440 data_power = None
1445 1441 data_intensity = None
1446 1442 data_velocity = None
1447 1443 data_specwidth = None
1448 1444 data_snrPP = None
1449 1445 self.putData(data=dataOut.data)
1450 1446 if self.__profIndex == self.n:
1451 1447 data_power,data_intensity, data_velocity,data_snrPP,data_specwidth, n = self.pushData(dataOut=dataOut)
1452 1448 self.__dataReady = True
1453 1449
1454 1450 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth
1455 1451
1456 1452
1457 1453 def pulsePairOp(self, dataOut, datatime= None):
1458 1454
1459 1455 if self.__initime == None:
1460 1456 self.__initime = datatime
1461 1457 data_power, data_intensity, data_velocity, data_snrPP, data_specwidth = self.pulsePairbyProfiles(dataOut)
1462 1458 self.__lastdatatime = datatime
1463 1459
1464 1460 if data_power is None:
1465 1461 return None, None, None,None,None,None
1466 1462
1467 1463 avgdatatime = self.__initime
1468 1464 deltatime = datatime - self.__lastdatatime
1469 1465 self.__initime = datatime
1470 1466
1471 1467 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth, avgdatatime
1472 1468
1473 1469 def run(self, dataOut,n = None,removeDC= False, overlapping= False,**kwargs):
1474 1470
1475 1471 if not self.isConfig:
1476 1472 self.setup(dataOut = dataOut, n = n , removeDC=removeDC , **kwargs)
1477 1473 self.isConfig = True
1478 1474 data_power, data_intensity, data_velocity,data_snrPP,data_specwidth, avgdatatime = self.pulsePairOp(dataOut, dataOut.utctime)
1479 1475 dataOut.flagNoData = True
1480 1476
1481 1477 if self.__dataReady:
1482 1478 dataOut.nCohInt *= self.n
1483 1479 dataOut.dataPP_POW = data_intensity # S
1484 1480 dataOut.dataPP_POWER = data_power # P
1485 1481 dataOut.dataPP_DOP = data_velocity
1486 1482 dataOut.dataPP_SNR = data_snrPP
1487 1483 dataOut.dataPP_WIDTH = data_specwidth
1488 1484 dataOut.PRFbyAngle = self.n #numero de PRF*cada angulo rotado que equivale a un tiempo.
1489 1485 dataOut.utctime = avgdatatime
1490 1486 dataOut.flagNoData = False
1491 1487 return dataOut
1492 1488
1493 1489
1494 1490
1495 1491 # import collections
1496 1492 # from scipy.stats import mode
1497 1493 #
1498 1494 # class Synchronize(Operation):
1499 1495 #
1500 1496 # isConfig = False
1501 1497 # __profIndex = 0
1502 1498 #
1503 1499 # def __init__(self, **kwargs):
1504 1500 #
1505 1501 # Operation.__init__(self, **kwargs)
1506 1502 # # self.isConfig = False
1507 1503 # self.__powBuffer = None
1508 1504 # self.__startIndex = 0
1509 1505 # self.__pulseFound = False
1510 1506 #
1511 1507 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1512 1508 #
1513 1509 # #Read data
1514 1510 #
1515 1511 # powerdB = dataOut.getPower(channel = channel)
1516 1512 # noisedB = dataOut.getNoise(channel = channel)[0]
1517 1513 #
1518 1514 # self.__powBuffer.extend(powerdB.flatten())
1519 1515 #
1520 1516 # dataArray = numpy.array(self.__powBuffer)
1521 1517 #
1522 1518 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1523 1519 #
1524 1520 # maxValue = numpy.nanmax(filteredPower)
1525 1521 #
1526 1522 # if maxValue < noisedB + 10:
1527 1523 # #No se encuentra ningun pulso de transmision
1528 1524 # return None
1529 1525 #
1530 1526 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1531 1527 #
1532 1528 # if len(maxValuesIndex) < 2:
1533 1529 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1534 1530 # return None
1535 1531 #
1536 1532 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1537 1533 #
1538 1534 # #Seleccionar solo valores con un espaciamiento de nSamples
1539 1535 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1540 1536 #
1541 1537 # if len(pulseIndex) < 2:
1542 1538 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1543 1539 # return None
1544 1540 #
1545 1541 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1546 1542 #
1547 1543 # #remover senales que se distancien menos de 10 unidades o muestras
1548 1544 # #(No deberian existir IPP menor a 10 unidades)
1549 1545 #
1550 1546 # realIndex = numpy.where(spacing > 10 )[0]
1551 1547 #
1552 1548 # if len(realIndex) < 2:
1553 1549 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1554 1550 # return None
1555 1551 #
1556 1552 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1557 1553 # realPulseIndex = pulseIndex[realIndex]
1558 1554 #
1559 1555 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1560 1556 #
1561 1557 # print "IPP = %d samples" %period
1562 1558 #
1563 1559 # self.__newNSamples = dataOut.nHeights #int(period)
1564 1560 # self.__startIndex = int(realPulseIndex[0])
1565 1561 #
1566 1562 # return 1
1567 1563 #
1568 1564 #
1569 1565 # def setup(self, nSamples, nChannels, buffer_size = 4):
1570 1566 #
1571 1567 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1572 1568 # maxlen = buffer_size*nSamples)
1573 1569 #
1574 1570 # bufferList = []
1575 1571 #
1576 1572 # for i in range(nChannels):
1577 1573 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1578 1574 # maxlen = buffer_size*nSamples)
1579 1575 #
1580 1576 # bufferList.append(bufferByChannel)
1581 1577 #
1582 1578 # self.__nSamples = nSamples
1583 1579 # self.__nChannels = nChannels
1584 1580 # self.__bufferList = bufferList
1585 1581 #
1586 1582 # def run(self, dataOut, channel = 0):
1587 1583 #
1588 1584 # if not self.isConfig:
1589 1585 # nSamples = dataOut.nHeights
1590 1586 # nChannels = dataOut.nChannels
1591 1587 # self.setup(nSamples, nChannels)
1592 1588 # self.isConfig = True
1593 1589 #
1594 1590 # #Append new data to internal buffer
1595 1591 # for thisChannel in range(self.__nChannels):
1596 1592 # bufferByChannel = self.__bufferList[thisChannel]
1597 1593 # bufferByChannel.extend(dataOut.data[thisChannel])
1598 1594 #
1599 1595 # if self.__pulseFound:
1600 1596 # self.__startIndex -= self.__nSamples
1601 1597 #
1602 1598 # #Finding Tx Pulse
1603 1599 # if not self.__pulseFound:
1604 1600 # indexFound = self.__findTxPulse(dataOut, channel)
1605 1601 #
1606 1602 # if indexFound == None:
1607 1603 # dataOut.flagNoData = True
1608 1604 # return
1609 1605 #
1610 1606 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1611 1607 # self.__pulseFound = True
1612 1608 # self.__startIndex = indexFound
1613 1609 #
1614 1610 # #If pulse was found ...
1615 1611 # for thisChannel in range(self.__nChannels):
1616 1612 # bufferByChannel = self.__bufferList[thisChannel]
1617 1613 # #print self.__startIndex
1618 1614 # x = numpy.array(bufferByChannel)
1619 1615 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1620 1616 #
1621 1617 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1622 1618 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1623 1619 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1624 1620 #
1625 1621 # dataOut.data = self.__arrayBuffer
1626 1622 #
1627 1623 # self.__startIndex += self.__newNSamples
1628 1624 #
1629 1625 # return
General Comments 0
You need to be logged in to leave comments. Login now