##// END OF EJS Templates
schain-jc
RSS
Multiprocessing for writing Units(Spectral, Voltage and Parameters)
George Yong -
r1179:6414333e2ace
parent child
Show More
Side by Side Unified
Context file:
r1179:6414333e2ace Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,1234 +1,1251
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
5 5 '''
6 6
7 7 import copy
8 8 import numpy
9 9 import datetime
10 10
11 11 from .jroheaderIO import SystemHeader, RadarControllerHeader
12 12
13 13
14 14 def getNumpyDtype(dataTypeCode):
15 15
16 16 if dataTypeCode == 0:
17 17 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
18 18 elif dataTypeCode == 1:
19 19 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
20 20 elif dataTypeCode == 2:
21 21 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
22 22 elif dataTypeCode == 3:
23 23 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
24 24 elif dataTypeCode == 4:
25 25 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
26 26 elif dataTypeCode == 5:
27 27 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
28 28 else:
29 29 raise ValueError('dataTypeCode was not defined')
30 30
31 31 return numpyDtype
32 32
33 33
34 34 def getDataTypeCode(numpyDtype):
35 35
36 36 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
37 37 datatype = 0
38 38 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
39 39 datatype = 1
40 40 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
41 41 datatype = 2
42 42 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
43 43 datatype = 3
44 44 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
45 45 datatype = 4
46 46 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
47 47 datatype = 5
48 48 else:
49 49 datatype = None
50 50
51 51 return datatype
52 52
53 53
54 54 def hildebrand_sekhon(data, navg):
55 55 """
56 56 This method is for the objective determination of the noise level in Doppler spectra. This
57 57 implementation technique is based on the fact that the standard deviation of the spectral
58 58 densities is equal to the mean spectral density for white Gaussian noise
59 59
60 60 Inputs:
61 61 Data : heights
62 62 navg : numbers of averages
63 63
64 64 Return:
65 65 mean : noise's level
66 66 """
67 67
68 sorted_spectrum = numpy.sort(data, axis=None)
69 nnoise = len(sorted_spectrum) # default to all points in the spectrum as noise
70 for npts in range(1, len(sorted_spectrum)+1):
71 partial = sorted_spectrum[:npts]
72 mean = partial.mean()
73 var = partial.var()
74 if var * navg < mean**2.:
75 nnoise = npts
76 else:
77 # partial spectrum no longer has characteristics of white noise
78 break
68 sortdata = numpy.sort(data, axis=None)
69 lenOfData = len(sortdata)
70 nums_min = lenOfData*0.2
71
72 if nums_min <= 5:
73
74 nums_min = 5
75
76 sump = 0.
77 sumq = 0.
78
79 j = 0
80 cont = 1
81
82 while((cont==1)and(j<lenOfData)):
83
84 sump += sortdata[j]
85 sumq += sortdata[j]**2
86
87 if j > nums_min:
88 rtest = float(j)/(j-1) + 1.0/navg
89 if ((sumq*j) > (rtest*sump**2)):
90 j = j - 1
91 sump = sump - sortdata[j]
92 sumq = sumq - sortdata[j]**2
93 cont = 0
94
95 j += 1
96
97 lnoise = sump /j
79 98
80 noise_spectrum = sorted_spectrum[:nnoise]
81 mean = noise_spectrum.mean()
82 return mean
99 return lnoise
83 100
84 101
85 102 class Beam:
86 103
87 104 def __init__(self):
88 105 self.codeList = []
89 106 self.azimuthList = []
90 107 self.zenithList = []
91 108
92 109
93 110 class GenericData(object):
94 111
95 112 flagNoData = True
96 113
97 114 def copy(self, inputObj=None):
98 115
99 116 if inputObj == None:
100 117 return copy.deepcopy(self)
101 118
102 119 for key in list(inputObj.__dict__.keys()):
103 120
104 121 attribute = inputObj.__dict__[key]
105 122
106 123 # If this attribute is a tuple or list
107 124 if type(inputObj.__dict__[key]) in (tuple, list):
108 125 self.__dict__[key] = attribute[:]
109 126 continue
110 127
111 128 # If this attribute is another object or instance
112 129 if hasattr(attribute, '__dict__'):
113 130 self.__dict__[key] = attribute.copy()
114 131 continue
115 132
116 133 self.__dict__[key] = inputObj.__dict__[key]
117 134
118 135 def deepcopy(self):
119 136
120 137 return copy.deepcopy(self)
121 138
122 139 def isEmpty(self):
123 140
124 141 return self.flagNoData
125 142
126 143
127 144 class JROData(GenericData):
128 145
129 146 # m_BasicHeader = BasicHeader()
130 147 # m_ProcessingHeader = ProcessingHeader()
131 148
132 149 systemHeaderObj = SystemHeader()
133 150
134 151 radarControllerHeaderObj = RadarControllerHeader()
135 152
136 153 # data = None
137 154
138 155 type = None
139 156
140 157 datatype = None # dtype but in string
141 158
142 159 # dtype = None
143 160
144 161 # nChannels = None
145 162
146 163 # nHeights = None
147 164
148 165 nProfiles = None
149 166
150 167 heightList = None
151 168
152 169 channelList = None
153 170
154 171 flagDiscontinuousBlock = False
155 172
156 173 useLocalTime = False
157 174
158 175 utctime = None
159 176
160 177 timeZone = None
161 178
162 179 dstFlag = None
163 180
164 181 errorCount = None
165 182
166 183 blocksize = None
167 184
168 185 # nCode = None
169 186 #
170 187 # nBaud = None
171 188 #
172 189 # code = None
173 190
174 191 flagDecodeData = False # asumo q la data no esta decodificada
175 192
176 193 flagDeflipData = False # asumo q la data no esta sin flip
177 194
178 195 flagShiftFFT = False
179 196
180 197 # ippSeconds = None
181 198
182 199 # timeInterval = None
183 200
184 201 nCohInt = None
185 202
186 203 # noise = None
187 204
188 205 windowOfFilter = 1
189 206
190 207 # Speed of ligth
191 208 C = 3e8
192 209
193 210 frequency = 49.92e6
194 211
195 212 realtime = False
196 213
197 214 beacon_heiIndexList = None
198 215
199 216 last_block = None
200 217
201 218 blocknow = None
202 219
203 220 azimuth = None
204 221
205 222 zenith = None
206 223
207 224 beam = Beam()
208 225
209 226 profileIndex = None
210 227
211 228 error = (0, '')
212 229
213 230 def __str__(self):
214 231
215 232 return '{} - {}'.format(self.type, self.getDatatime())
216 233
217 234 def getNoise(self):
218 235
219 236 raise NotImplementedError
220 237
221 238 def getNChannels(self):
222 239
223 240 return len(self.channelList)
224 241
225 242 def getChannelIndexList(self):
226 243
227 244 return list(range(self.nChannels))
228 245
229 246 def getNHeights(self):
230 247
231 248 return len(self.heightList)
232 249
233 250 def getHeiRange(self, extrapoints=0):
234 251
235 252 heis = self.heightList
236 253 # deltah = self.heightList[1] - self.heightList[0]
237 254 #
238 255 # heis.append(self.heightList[-1])
239 256
240 257 return heis
241 258
242 259 def getDeltaH(self):
243 260
244 261 delta = self.heightList[1] - self.heightList[0]
245 262
246 263 return delta
247 264
248 265 def getltctime(self):
249 266
250 267 if self.useLocalTime:
251 268 return self.utctime - self.timeZone * 60
252 269
253 270 return self.utctime
254 271
255 272 def getDatatime(self):
256 273
257 274 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
258 275 return datatimeValue
259 276
260 277 def getTimeRange(self):
261 278
262 279 datatime = []
263 280
264 281 datatime.append(self.ltctime)
265 282 datatime.append(self.ltctime + self.timeInterval + 1)
266 283
267 284 datatime = numpy.array(datatime)
268 285
269 286 return datatime
270 287
271 288 def getFmaxTimeResponse(self):
272 289
273 290 period = (10**-6) * self.getDeltaH() / (0.15)
274 291
275 292 PRF = 1. / (period * self.nCohInt)
276 293
277 294 fmax = PRF
278 295
279 296 return fmax
280 297
281 298 def getFmax(self):
282 299 PRF = 1. / (self.ippSeconds * self.nCohInt)
283 300
284 301 fmax = PRF
285 302 return fmax
286 303
287 304 def getVmax(self):
288 305
289 306 _lambda = self.C / self.frequency
290 307
291 308 vmax = self.getFmax() * _lambda / 2
292 309
293 310 return vmax
294 311
295 312 def get_ippSeconds(self):
296 313 '''
297 314 '''
298 315 return self.radarControllerHeaderObj.ippSeconds
299 316
300 317 def set_ippSeconds(self, ippSeconds):
301 318 '''
302 319 '''
303 320
304 321 self.radarControllerHeaderObj.ippSeconds = ippSeconds
305 322
306 323 return
307 324
308 325 def get_dtype(self):
309 326 '''
310 327 '''
311 328 return getNumpyDtype(self.datatype)
312 329
313 330 def set_dtype(self, numpyDtype):
314 331 '''
315 332 '''
316 333
317 334 self.datatype = getDataTypeCode(numpyDtype)
318 335
319 336 def get_code(self):
320 337 '''
321 338 '''
322 339 return self.radarControllerHeaderObj.code
323 340
324 341 def set_code(self, code):
325 342 '''
326 343 '''
327 344 self.radarControllerHeaderObj.code = code
328 345
329 346 return
330 347
331 348 def get_ncode(self):
332 349 '''
333 350 '''
334 351 return self.radarControllerHeaderObj.nCode
335 352
336 353 def set_ncode(self, nCode):
337 354 '''
338 355 '''
339 356 self.radarControllerHeaderObj.nCode = nCode
340 357
341 358 return
342 359
343 360 def get_nbaud(self):
344 361 '''
345 362 '''
346 363 return self.radarControllerHeaderObj.nBaud
347 364
348 365 def set_nbaud(self, nBaud):
349 366 '''
350 367 '''
351 368 self.radarControllerHeaderObj.nBaud = nBaud
352 369
353 370 return
354 371
355 372 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
356 373 channelIndexList = property(
357 374 getChannelIndexList, "I'm the 'channelIndexList' property.")
358 375 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
359 376 #noise = property(getNoise, "I'm the 'nHeights' property.")
360 377 datatime = property(getDatatime, "I'm the 'datatime' property")
361 378 ltctime = property(getltctime, "I'm the 'ltctime' property")
362 379 ippSeconds = property(get_ippSeconds, set_ippSeconds)
363 380 dtype = property(get_dtype, set_dtype)
364 381 # timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
365 382 code = property(get_code, set_code)
366 383 nCode = property(get_ncode, set_ncode)
367 384 nBaud = property(get_nbaud, set_nbaud)
368 385
369 386
370 387 class Voltage(JROData):
371 388
372 389 # data es un numpy array de 2 dmensiones (canales, alturas)
373 390 data = None
374 391
375 392 def __init__(self):
376 393 '''
377 394 Constructor
378 395 '''
379 396
380 397 self.useLocalTime = True
381 398
382 399 self.radarControllerHeaderObj = RadarControllerHeader()
383 400
384 401 self.systemHeaderObj = SystemHeader()
385 402
386 403 self.type = "Voltage"
387 404
388 405 self.data = None
389 406
390 407 # self.dtype = None
391 408
392 409 # self.nChannels = 0
393 410
394 411 # self.nHeights = 0
395 412
396 413 self.nProfiles = None
397 414
398 415 self.heightList = None
399 416
400 417 self.channelList = None
401 418
402 419 # self.channelIndexList = None
403 420
404 421 self.flagNoData = True
405 422
406 423 self.flagDiscontinuousBlock = False
407 424
408 425 self.utctime = None
409 426
410 427 self.timeZone = None
411 428
412 429 self.dstFlag = None
413 430
414 431 self.errorCount = None
415 432
416 433 self.nCohInt = None
417 434
418 435 self.blocksize = None
419 436
420 437 self.flagDecodeData = False # asumo q la data no esta decodificada
421 438
422 439 self.flagDeflipData = False # asumo q la data no esta sin flip
423 440
424 441 self.flagShiftFFT = False
425 442
426 443 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
427 444
428 445 self.profileIndex = 0
429 446
430 447 def getNoisebyHildebrand(self, channel=None):
431 448 """
432 449 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
433 450
434 451 Return:
435 452 noiselevel
436 453 """
437 454
438 455 if channel != None:
439 456 data = self.data[channel]
440 457 nChannels = 1
441 458 else:
442 459 data = self.data
443 460 nChannels = self.nChannels
444 461
445 462 noise = numpy.zeros(nChannels)
446 463 power = data * numpy.conjugate(data)
447 464
448 465 for thisChannel in range(nChannels):
449 466 if nChannels == 1:
450 467 daux = power[:].real
451 468 else:
452 469 daux = power[thisChannel, :].real
453 470 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
454 471
455 472 return noise
456 473
457 474 def getNoise(self, type=1, channel=None):
458 475
459 476 if type == 1:
460 477 noise = self.getNoisebyHildebrand(channel)
461 478
462 479 return noise
463 480
464 481 def getPower(self, channel=None):
465 482
466 483 if channel != None:
467 484 data = self.data[channel]
468 485 else:
469 486 data = self.data
470 487
471 488 power = data * numpy.conjugate(data)
472 489 powerdB = 10 * numpy.log10(power.real)
473 490 powerdB = numpy.squeeze(powerdB)
474 491
475 492 return powerdB
476 493
477 494 def getTimeInterval(self):
478 495
479 496 timeInterval = self.ippSeconds * self.nCohInt
480 497
481 498 return timeInterval
482 499
483 500 noise = property(getNoise, "I'm the 'nHeights' property.")
484 501 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
485 502
486 503
487 504 class Spectra(JROData):
488 505
489 506 # data spc es un numpy array de 2 dmensiones (canales, perfiles, alturas)
490 507 data_spc = None
491 508
492 509 # data cspc es un numpy array de 2 dmensiones (canales, pares, alturas)
493 510 data_cspc = None
494 511
495 512 # data dc es un numpy array de 2 dmensiones (canales, alturas)
496 513 data_dc = None
497 514
498 515 # data power
499 516 data_pwr = None
500 517
501 518 nFFTPoints = None
502 519
503 520 # nPairs = None
504 521
505 522 pairsList = None
506 523
507 524 nIncohInt = None
508 525
509 526 wavelength = None # Necesario para cacular el rango de velocidad desde la frecuencia
510 527
511 528 nCohInt = None # se requiere para determinar el valor de timeInterval
512 529
513 530 ippFactor = None
514 531
515 532 profileIndex = 0
516 533
517 534 plotting = "spectra"
518 535
519 536 def __init__(self):
520 537 '''
521 538 Constructor
522 539 '''
523 540
524 541 self.useLocalTime = True
525 542
526 543 self.radarControllerHeaderObj = RadarControllerHeader()
527 544
528 545 self.systemHeaderObj = SystemHeader()
529 546
530 547 self.type = "Spectra"
531 548
532 549 # self.data = None
533 550
534 551 # self.dtype = None
535 552
536 553 # self.nChannels = 0
537 554
538 555 # self.nHeights = 0
539 556
540 557 self.nProfiles = None
541 558
542 559 self.heightList = None
543 560
544 561 self.channelList = None
545 562
546 563 # self.channelIndexList = None
547 564
548 565 self.pairsList = None
549 566
550 567 self.flagNoData = True
551 568
552 569 self.flagDiscontinuousBlock = False
553 570
554 571 self.utctime = None
555 572
556 573 self.nCohInt = None
557 574
558 575 self.nIncohInt = None
559 576
560 577 self.blocksize = None
561 578
562 579 self.nFFTPoints = None
563 580
564 581 self.wavelength = None
565 582
566 583 self.flagDecodeData = False # asumo q la data no esta decodificada
567 584
568 585 self.flagDeflipData = False # asumo q la data no esta sin flip
569 586
570 587 self.flagShiftFFT = False
571 588
572 589 self.ippFactor = 1
573 590
574 591 #self.noise = None
575 592
576 593 self.beacon_heiIndexList = []
577 594
578 595 self.noise_estimation = None
579 596
580 597 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
581 598 """
582 599 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
583 600
584 601 Return:
585 602 noiselevel
586 603 """
587 604
588 605 noise = numpy.zeros(self.nChannels)
589 606
590 607 for channel in range(self.nChannels):
591 608 daux = self.data_spc[channel,
592 609 xmin_index:xmax_index, ymin_index:ymax_index]
593 610 noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
594 611
595 612 return noise
596 613
597 614 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
598 615
599 616 if self.noise_estimation is not None:
600 617 # this was estimated by getNoise Operation defined in jroproc_spectra.py
601 618 return self.noise_estimation
602 619 else:
603 620 noise = self.getNoisebyHildebrand(
604 621 xmin_index, xmax_index, ymin_index, ymax_index)
605 622 return noise
606 623
607 624 def getFreqRangeTimeResponse(self, extrapoints=0):
608 625
609 626 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
610 627 freqrange = deltafreq * \
611 628 (numpy.arange(self.nFFTPoints + extrapoints) -
612 629 self.nFFTPoints / 2.) - deltafreq / 2
613 630
614 631 return freqrange
615 632
616 633 def getAcfRange(self, extrapoints=0):
617 634
618 635 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
619 636 freqrange = deltafreq * \
620 637 (numpy.arange(self.nFFTPoints + extrapoints) -
621 638 self.nFFTPoints / 2.) - deltafreq / 2
622 639
623 640 return freqrange
624 641
625 642 def getFreqRange(self, extrapoints=0):
626 643
627 644 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
628 645 freqrange = deltafreq * \
629 646 (numpy.arange(self.nFFTPoints + extrapoints) -
630 647 self.nFFTPoints / 2.) - deltafreq / 2
631 648
632 649 return freqrange
633 650
634 651 def getVelRange(self, extrapoints=0):
635 652
636 653 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
637 654 velrange = deltav * (numpy.arange(self.nFFTPoints +
638 655 extrapoints) - self.nFFTPoints / 2.) # - deltav/2
639 656
640 657 return velrange
641 658
642 659 def getNPairs(self):
643 660
644 661 return len(self.pairsList)
645 662
646 663 def getPairsIndexList(self):
647 664
648 665 return list(range(self.nPairs))
649 666
650 667 def getNormFactor(self):
651 668
652 669 pwcode = 1
653 670
654 671 if self.flagDecodeData:
655 672 pwcode = numpy.sum(self.code[0]**2)
656 673 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
657 674 normFactor = self.nProfiles * self.nIncohInt * \
658 675 self.nCohInt * pwcode * self.windowOfFilter
659 676
660 677 return normFactor
661 678
662 679 def getFlagCspc(self):
663 680
664 681 if self.data_cspc is None:
665 682 return True
666 683
667 684 return False
668 685
669 686 def getFlagDc(self):
670 687
671 688 if self.data_dc is None:
672 689 return True
673 690
674 691 return False
675 692
676 693 def getTimeInterval(self):
677 694
678 695 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
679 696
680 697 return timeInterval
681 698
682 699 def getPower(self):
683 700
684 701 factor = self.normFactor
685 702 z = self.data_spc / factor
686 703 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
687 704 avg = numpy.average(z, axis=1)
688 705
689 706 return 10 * numpy.log10(avg)
690 707
691 708 def getCoherence(self, pairsList=None, phase=False):
692 709
693 710 z = []
694 711 if pairsList is None:
695 712 pairsIndexList = self.pairsIndexList
696 713 else:
697 714 pairsIndexList = []
698 715 for pair in pairsList:
699 716 if pair not in self.pairsList:
700 717 raise ValueError("Pair %s is not in dataOut.pairsList" % (
701 718 pair))
702 719 pairsIndexList.append(self.pairsList.index(pair))
703 720 for i in range(len(pairsIndexList)):
704 721 pair = self.pairsList[pairsIndexList[i]]
705 722 ccf = numpy.average(
706 723 self.data_cspc[pairsIndexList[i], :, :], axis=0)
707 724 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
708 725 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
709 726 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
710 727 if phase:
711 728 data = numpy.arctan2(avgcoherenceComplex.imag,
712 729 avgcoherenceComplex.real) * 180 / numpy.pi
713 730 else:
714 731 data = numpy.abs(avgcoherenceComplex)
715 732
716 733 z.append(data)
717 734
718 735 return numpy.array(z)
719 736
720 737 def setValue(self, value):
721 738
722 739 print("This property should not be initialized")
723 740
724 741 return
725 742
726 743 nPairs = property(getNPairs, setValue, "I'm the 'nPairs' property.")
727 744 pairsIndexList = property(
728 745 getPairsIndexList, setValue, "I'm the 'pairsIndexList' property.")
729 746 normFactor = property(getNormFactor, setValue,
730 747 "I'm the 'getNormFactor' property.")
731 748 flag_cspc = property(getFlagCspc, setValue)
732 749 flag_dc = property(getFlagDc, setValue)
733 750 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
734 751 timeInterval = property(getTimeInterval, setValue,
735 752 "I'm the 'timeInterval' property")
736 753
737 754
738 755 class SpectraHeis(Spectra):
739 756
740 757 data_spc = None
741 758
742 759 data_cspc = None
743 760
744 761 data_dc = None
745 762
746 763 nFFTPoints = None
747 764
748 765 # nPairs = None
749 766
750 767 pairsList = None
751 768
752 769 nCohInt = None
753 770
754 771 nIncohInt = None
755 772
756 773 def __init__(self):
757 774
758 775 self.radarControllerHeaderObj = RadarControllerHeader()
759 776
760 777 self.systemHeaderObj = SystemHeader()
761 778
762 779 self.type = "SpectraHeis"
763 780
764 781 # self.dtype = None
765 782
766 783 # self.nChannels = 0
767 784
768 785 # self.nHeights = 0
769 786
770 787 self.nProfiles = None
771 788
772 789 self.heightList = None
773 790
774 791 self.channelList = None
775 792
776 793 # self.channelIndexList = None
777 794
778 795 self.flagNoData = True
779 796
780 797 self.flagDiscontinuousBlock = False
781 798
782 799 # self.nPairs = 0
783 800
784 801 self.utctime = None
785 802
786 803 self.blocksize = None
787 804
788 805 self.profileIndex = 0
789 806
790 807 self.nCohInt = 1
791 808
792 809 self.nIncohInt = 1
793 810
794 811 def getNormFactor(self):
795 812 pwcode = 1
796 813 if self.flagDecodeData:
797 814 pwcode = numpy.sum(self.code[0]**2)
798 815
799 816 normFactor = self.nIncohInt * self.nCohInt * pwcode
800 817
801 818 return normFactor
802 819
803 820 def getTimeInterval(self):
804 821
805 822 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
806 823
807 824 return timeInterval
808 825
809 826 normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
810 827 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
811 828
812 829
813 830 class Fits(JROData):
814 831
815 832 heightList = None
816 833
817 834 channelList = None
818 835
819 836 flagNoData = True
820 837
821 838 flagDiscontinuousBlock = False
822 839
823 840 useLocalTime = False
824 841
825 842 utctime = None
826 843
827 844 timeZone = None
828 845
829 846 # ippSeconds = None
830 847
831 848 # timeInterval = None
832 849
833 850 nCohInt = None
834 851
835 852 nIncohInt = None
836 853
837 854 noise = None
838 855
839 856 windowOfFilter = 1
840 857
841 858 # Speed of ligth
842 859 C = 3e8
843 860
844 861 frequency = 49.92e6
845 862
846 863 realtime = False
847 864
848 865 def __init__(self):
849 866
850 867 self.type = "Fits"
851 868
852 869 self.nProfiles = None
853 870
854 871 self.heightList = None
855 872
856 873 self.channelList = None
857 874
858 875 # self.channelIndexList = None
859 876
860 877 self.flagNoData = True
861 878
862 879 self.utctime = None
863 880
864 881 self.nCohInt = 1
865 882
866 883 self.nIncohInt = 1
867 884
868 885 self.useLocalTime = True
869 886
870 887 self.profileIndex = 0
871 888
872 889 # self.utctime = None
873 890 # self.timeZone = None
874 891 # self.ltctime = None
875 892 # self.timeInterval = None
876 893 # self.header = None
877 894 # self.data_header = None
878 895 # self.data = None
879 896 # self.datatime = None
880 897 # self.flagNoData = False
881 898 # self.expName = ''
882 899 # self.nChannels = None
883 900 # self.nSamples = None
884 901 # self.dataBlocksPerFile = None
885 902 # self.comments = ''
886 903 #
887 904
888 905 def getltctime(self):
889 906
890 907 if self.useLocalTime:
891 908 return self.utctime - self.timeZone * 60
892 909
893 910 return self.utctime
894 911
895 912 def getDatatime(self):
896 913
897 914 datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
898 915 return datatime
899 916
900 917 def getTimeRange(self):
901 918
902 919 datatime = []
903 920
904 921 datatime.append(self.ltctime)
905 922 datatime.append(self.ltctime + self.timeInterval)
906 923
907 924 datatime = numpy.array(datatime)
908 925
909 926 return datatime
910 927
911 928 def getHeiRange(self):
912 929
913 930 heis = self.heightList
914 931
915 932 return heis
916 933
917 934 def getNHeights(self):
918 935
919 936 return len(self.heightList)
920 937
921 938 def getNChannels(self):
922 939
923 940 return len(self.channelList)
924 941
925 942 def getChannelIndexList(self):
926 943
927 944 return list(range(self.nChannels))
928 945
929 946 def getNoise(self, type=1):
930 947
931 948 #noise = numpy.zeros(self.nChannels)
932 949
933 950 if type == 1:
934 951 noise = self.getNoisebyHildebrand()
935 952
936 953 if type == 2:
937 954 noise = self.getNoisebySort()
938 955
939 956 if type == 3:
940 957 noise = self.getNoisebyWindow()
941 958
942 959 return noise
943 960
944 961 def getTimeInterval(self):
945 962
946 963 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
947 964
948 965 return timeInterval
949 966
950 967 datatime = property(getDatatime, "I'm the 'datatime' property")
951 968 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
952 969 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
953 970 channelIndexList = property(
954 971 getChannelIndexList, "I'm the 'channelIndexList' property.")
955 972 noise = property(getNoise, "I'm the 'nHeights' property.")
956 973
957 974 ltctime = property(getltctime, "I'm the 'ltctime' property")
958 975 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
959 976
960 977
961 978 class Correlation(JROData):
962 979
963 980 noise = None
964 981
965 982 SNR = None
966 983
967 984 #--------------------------------------------------
968 985
969 986 mode = None
970 987
971 988 split = False
972 989
973 990 data_cf = None
974 991
975 992 lags = None
976 993
977 994 lagRange = None
978 995
979 996 pairsList = None
980 997
981 998 normFactor = None
982 999
983 1000 #--------------------------------------------------
984 1001
985 1002 # calculateVelocity = None
986 1003
987 1004 nLags = None
988 1005
989 1006 nPairs = None
990 1007
991 1008 nAvg = None
992 1009
993 1010 def __init__(self):
994 1011 '''
995 1012 Constructor
996 1013 '''
997 1014 self.radarControllerHeaderObj = RadarControllerHeader()
998 1015
999 1016 self.systemHeaderObj = SystemHeader()
1000 1017
1001 1018 self.type = "Correlation"
1002 1019
1003 1020 self.data = None
1004 1021
1005 1022 self.dtype = None
1006 1023
1007 1024 self.nProfiles = None
1008 1025
1009 1026 self.heightList = None
1010 1027
1011 1028 self.channelList = None
1012 1029
1013 1030 self.flagNoData = True
1014 1031
1015 1032 self.flagDiscontinuousBlock = False
1016 1033
1017 1034 self.utctime = None
1018 1035
1019 1036 self.timeZone = None
1020 1037
1021 1038 self.dstFlag = None
1022 1039
1023 1040 self.errorCount = None
1024 1041
1025 1042 self.blocksize = None
1026 1043
1027 1044 self.flagDecodeData = False # asumo q la data no esta decodificada
1028 1045
1029 1046 self.flagDeflipData = False # asumo q la data no esta sin flip
1030 1047
1031 1048 self.pairsList = None
1032 1049
1033 1050 self.nPoints = None
1034 1051
1035 1052 def getPairsList(self):
1036 1053
1037 1054 return self.pairsList
1038 1055
1039 1056 def getNoise(self, mode=2):
1040 1057
1041 1058 indR = numpy.where(self.lagR == 0)[0][0]
1042 1059 indT = numpy.where(self.lagT == 0)[0][0]
1043 1060
1044 1061 jspectra0 = self.data_corr[:, :, indR, :]
1045 1062 jspectra = copy.copy(jspectra0)
1046 1063
1047 1064 num_chan = jspectra.shape[0]
1048 1065 num_hei = jspectra.shape[2]
1049 1066
1050 1067 freq_dc = jspectra.shape[1] / 2
1051 1068 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
1052 1069
1053 1070 if ind_vel[0] < 0:
1054 1071 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
1055 1072
1056 1073 if mode == 1:
1057 1074 jspectra[:, freq_dc, :] = (
1058 1075 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
1059 1076
1060 1077 if mode == 2:
1061 1078
1062 1079 vel = numpy.array([-2, -1, 1, 2])
1063 1080 xx = numpy.zeros([4, 4])
1064 1081
1065 1082 for fil in range(4):
1066 1083 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
1067 1084
1068 1085 xx_inv = numpy.linalg.inv(xx)
1069 1086 xx_aux = xx_inv[0, :]
1070 1087
1071 1088 for ich in range(num_chan):
1072 1089 yy = jspectra[ich, ind_vel, :]
1073 1090 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
1074 1091
1075 1092 junkid = jspectra[ich, freq_dc, :] <= 0
1076 1093 cjunkid = sum(junkid)
1077 1094
1078 1095 if cjunkid.any():
1079 1096 jspectra[ich, freq_dc, junkid.nonzero()] = (
1080 1097 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
1081 1098
1082 1099 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
1083 1100
1084 1101 return noise
1085 1102
1086 1103 def getTimeInterval(self):
1087 1104
1088 1105 timeInterval = self.ippSeconds * self.nCohInt * self.nProfiles
1089 1106
1090 1107 return timeInterval
1091 1108
1092 1109 def splitFunctions(self):
1093 1110
1094 1111 pairsList = self.pairsList
1095 1112 ccf_pairs = []
1096 1113 acf_pairs = []
1097 1114 ccf_ind = []
1098 1115 acf_ind = []
1099 1116 for l in range(len(pairsList)):
1100 1117 chan0 = pairsList[l][0]
1101 1118 chan1 = pairsList[l][1]
1102 1119
1103 1120 # Obteniendo pares de Autocorrelacion
1104 1121 if chan0 == chan1:
1105 1122 acf_pairs.append(chan0)
1106 1123 acf_ind.append(l)
1107 1124 else:
1108 1125 ccf_pairs.append(pairsList[l])
1109 1126 ccf_ind.append(l)
1110 1127
1111 1128 data_acf = self.data_cf[acf_ind]
1112 1129 data_ccf = self.data_cf[ccf_ind]
1113 1130
1114 1131 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
1115 1132
1116 1133 def getNormFactor(self):
1117 1134 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
1118 1135 acf_pairs = numpy.array(acf_pairs)
1119 1136 normFactor = numpy.zeros((self.nPairs, self.nHeights))
1120 1137
1121 1138 for p in range(self.nPairs):
1122 1139 pair = self.pairsList[p]
1123 1140
1124 1141 ch0 = pair[0]
1125 1142 ch1 = pair[1]
1126 1143
1127 1144 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
1128 1145 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
1129 1146 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
1130 1147
1131 1148 return normFactor
1132 1149
1133 1150 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
1134 1151 normFactor = property(getNormFactor, "I'm the 'normFactor property'")
1135 1152
1136 1153
1137 1154 class Parameters(Spectra):
1138 1155
1139 1156 experimentInfo = None # Information about the experiment
1140 1157
1141 1158 # Information from previous data
1142 1159
1143 1160 inputUnit = None # Type of data to be processed
1144 1161
1145 1162 operation = None # Type of operation to parametrize
1146 1163
1147 1164 # normFactor = None #Normalization Factor
1148 1165
1149 1166 groupList = None # List of Pairs, Groups, etc
1150 1167
1151 1168 # Parameters
1152 1169
1153 1170 data_param = None # Parameters obtained
1154 1171
1155 1172 data_pre = None # Data Pre Parametrization
1156 1173
1157 1174 data_SNR = None # Signal to Noise Ratio
1158 1175
1159 1176 # heightRange = None #Heights
1160 1177
1161 1178 abscissaList = None # Abscissa, can be velocities, lags or time
1162 1179
1163 1180 # noise = None #Noise Potency
1164 1181
1165 1182 utctimeInit = None # Initial UTC time
1166 1183
1167 1184 paramInterval = None # Time interval to calculate Parameters in seconds
1168 1185
1169 1186 useLocalTime = True
1170 1187
1171 1188 # Fitting
1172 1189
1173 1190 data_error = None # Error of the estimation
1174 1191
1175 1192 constants = None
1176 1193
1177 1194 library = None
1178 1195
1179 1196 # Output signal
1180 1197
1181 1198 outputInterval = None # Time interval to calculate output signal in seconds
1182 1199
1183 1200 data_output = None # Out signal
1184 1201
1185 1202 nAvg = None
1186 1203
1187 1204 noise_estimation = None
1188 1205
1189 1206 GauSPC = None # Fit gaussian SPC
1190 1207
1191 1208 def __init__(self):
1192 1209 '''
1193 1210 Constructor
1194 1211 '''
1195 1212 self.radarControllerHeaderObj = RadarControllerHeader()
1196 1213
1197 1214 self.systemHeaderObj = SystemHeader()
1198 1215
1199 1216 self.type = "Parameters"
1200 1217
1201 1218 def getTimeRange1(self, interval):
1202 1219
1203 1220 datatime = []
1204 1221
1205 1222 if self.useLocalTime:
1206 1223 time1 = self.utctimeInit - self.timeZone * 60
1207 1224 else:
1208 1225 time1 = self.utctimeInit
1209 1226
1210 1227 datatime.append(time1)
1211 1228 datatime.append(time1 + interval)
1212 1229 datatime = numpy.array(datatime)
1213 1230
1214 1231 return datatime
1215 1232
1216 1233 def getTimeInterval(self):
1217 1234
1218 1235 if hasattr(self, 'timeInterval1'):
1219 1236 return self.timeInterval1
1220 1237 else:
1221 1238 return self.paramInterval
1222 1239
1223 1240 def setValue(self, value):
1224 1241
1225 1242 print("This property should not be initialized")
1226 1243
1227 1244 return
1228 1245
1229 1246 def getNoise(self):
1230 1247
1231 1248 return self.spc_noise
1232 1249
1233 1250 timeInterval = property(getTimeInterval)
1234 1251 noise = property(getNoise, setValue, "I'm the 'Noise' property.") No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,2151 +1,2158
1 1 import os
2 2 import datetime
3 3 import numpy
4 4 import inspect
5 5 from .figure import Figure, isRealtime, isTimeInHourRange
6 6 from .plotting_codes import *
7
7 from schainpy.model.proc.jroproc_base import MPDecorator
8 from schainpy.utils import log
8 9
9 10 class FitGauPlot(Figure):
10 11
11 12 isConfig = None
12 13 __nsubplots = None
13 14
14 15 WIDTHPROF = None
15 16 HEIGHTPROF = None
16 17 PREFIX = 'fitgau'
17 18
18 19 def __init__(self, **kwargs):
19 20 Figure.__init__(self, **kwargs)
20 21 self.isConfig = False
21 22 self.__nsubplots = 1
22 23
23 24 self.WIDTH = 250
24 25 self.HEIGHT = 250
25 26 self.WIDTHPROF = 120
26 27 self.HEIGHTPROF = 0
27 28 self.counter_imagwr = 0
28 29
29 30 self.PLOT_CODE = SPEC_CODE
30 31
31 32 self.FTP_WEI = None
32 33 self.EXP_CODE = None
33 34 self.SUB_EXP_CODE = None
34 35 self.PLOT_POS = None
35 36
36 37 self.__xfilter_ena = False
37 38 self.__yfilter_ena = False
38 39
39 40 def getSubplots(self):
40 41
41 42 ncol = int(numpy.sqrt(self.nplots)+0.9)
42 43 nrow = int(self.nplots*1./ncol + 0.9)
43 44
44 45 return nrow, ncol
45 46
46 47 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
47 48
48 49 self.__showprofile = showprofile
49 50 self.nplots = nplots
50 51
51 52 ncolspan = 1
52 53 colspan = 1
53 54 if showprofile:
54 55 ncolspan = 3
55 56 colspan = 2
56 57 self.__nsubplots = 2
57 58
58 59 self.createFigure(id = id,
59 60 wintitle = wintitle,
60 61 widthplot = self.WIDTH + self.WIDTHPROF,
61 62 heightplot = self.HEIGHT + self.HEIGHTPROF,
62 63 show=show)
63 64
64 65 nrow, ncol = self.getSubplots()
65 66
66 67 counter = 0
67 68 for y in range(nrow):
68 69 for x in range(ncol):
69 70
70 71 if counter >= self.nplots:
71 72 break
72 73
73 74 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
74 75
75 76 if showprofile:
76 77 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
77 78
78 79 counter += 1
79 80
80 81 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
81 82 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
82 83 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
83 84 server=None, folder=None, username=None, password=None,
84 85 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
85 86 xaxis="frequency", colormap='jet', normFactor=None , GauSelector = 1):
86 87
87 88 """
88 89
89 90 Input:
90 91 dataOut :
91 92 id :
92 93 wintitle :
93 94 channelList :
94 95 showProfile :
95 96 xmin : None,
96 97 xmax : None,
97 98 ymin : None,
98 99 ymax : None,
99 100 zmin : None,
100 101 zmax : None
101 102 """
102 103 if realtime:
103 104 if not(isRealtime(utcdatatime = dataOut.utctime)):
104 105 print('Skipping this plot function')
105 106 return
106 107
107 108 if channelList == None:
108 109 channelIndexList = dataOut.channelIndexList
109 110 else:
110 111 channelIndexList = []
111 112 for channel in channelList:
112 113 if channel not in dataOut.channelList:
113 114 raise ValueError("Channel %d is not in dataOut.channelList" %channel)
114 115 channelIndexList.append(dataOut.channelList.index(channel))
115 116
116 117 # if normFactor is None:
117 118 # factor = dataOut.normFactor
118 119 # else:
119 120 # factor = normFactor
120 121 if xaxis == "frequency":
121 122 x = dataOut.spc_range[0]
122 123 xlabel = "Frequency (kHz)"
123 124
124 125 elif xaxis == "time":
125 126 x = dataOut.spc_range[1]
126 127 xlabel = "Time (ms)"
127 128
128 129 else:
129 130 x = dataOut.spc_range[2]
130 131 xlabel = "Velocity (m/s)"
131 132
132 133 ylabel = "Range (Km)"
133 134
134 135 y = dataOut.getHeiRange()
135 136
136 137 z = dataOut.GauSPC[:,GauSelector,:,:] #GauSelector] #dataOut.data_spc/factor
137 138 print('GausSPC', z[0,32,10:40])
138 139 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
139 140 zdB = 10*numpy.log10(z)
140 141
141 142 avg = numpy.average(z, axis=1)
142 143 avgdB = 10*numpy.log10(avg)
143 144
144 145 noise = dataOut.spc_noise
145 146 noisedB = 10*numpy.log10(noise)
146 147
147 148 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
148 149 title = wintitle + " Spectra"
149 150 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
150 151 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
151 152
152 153 if not self.isConfig:
153 154
154 155 nplots = len(channelIndexList)
155 156
156 157 self.setup(id=id,
157 158 nplots=nplots,
158 159 wintitle=wintitle,
159 160 showprofile=showprofile,
160 161 show=show)
161 162
162 163 if xmin == None: xmin = numpy.nanmin(x)
163 164 if xmax == None: xmax = numpy.nanmax(x)
164 165 if ymin == None: ymin = numpy.nanmin(y)
165 166 if ymax == None: ymax = numpy.nanmax(y)
166 167 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
167 168 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
168 169
169 170 self.FTP_WEI = ftp_wei
170 171 self.EXP_CODE = exp_code
171 172 self.SUB_EXP_CODE = sub_exp_code
172 173 self.PLOT_POS = plot_pos
173 174
174 175 self.isConfig = True
175 176
176 177 self.setWinTitle(title)
177 178
178 179 for i in range(self.nplots):
179 180 index = channelIndexList[i]
180 181 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
181 182 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
182 183 if len(dataOut.beam.codeList) != 0:
183 184 title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
184 185
185 186 axes = self.axesList[i*self.__nsubplots]
186 187 axes.pcolor(x, y, zdB[index,:,:],
187 188 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
188 189 xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
189 190 ticksize=9, cblabel='')
190 191
191 192 if self.__showprofile:
192 193 axes = self.axesList[i*self.__nsubplots +1]
193 194 axes.pline(avgdB[index,:], y,
194 195 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
195 196 xlabel='dB', ylabel='', title='',
196 197 ytick_visible=False,
197 198 grid='x')
198 199
199 200 noiseline = numpy.repeat(noisedB[index], len(y))
200 201 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
201 202
202 203 self.draw()
203 204
204 205 if figfile == None:
205 206 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
206 207 name = str_datetime
207 208 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
208 209 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
209 210 figfile = self.getFilename(name)
210 211
211 212 self.save(figpath=figpath,
212 213 figfile=figfile,
213 214 save=save,
214 215 ftp=ftp,
215 216 wr_period=wr_period,
216 217 thisDatetime=thisDatetime)
217 218
218 219
219 220
220 221 class MomentsPlot(Figure):
221 222
222 223 isConfig = None
223 224 __nsubplots = None
224 225
225 226 WIDTHPROF = None
226 227 HEIGHTPROF = None
227 228 PREFIX = 'prm'
228 def __init__(self, **kwargs):
229 Figure.__init__(self, **kwargs)
229 def __init__(self):
230 Figure.__init__(self)
230 231 self.isConfig = False
231 232 self.__nsubplots = 1
232 233
233 234 self.WIDTH = 280
234 235 self.HEIGHT = 250
235 236 self.WIDTHPROF = 120
236 237 self.HEIGHTPROF = 0
237 238 self.counter_imagwr = 0
238 239
239 240 self.PLOT_CODE = MOMENTS_CODE
240 241
241 242 self.FTP_WEI = None
242 243 self.EXP_CODE = None
243 244 self.SUB_EXP_CODE = None
244 245 self.PLOT_POS = None
245 246
246 247 def getSubplots(self):
247 248
248 249 ncol = int(numpy.sqrt(self.nplots)+0.9)
249 250 nrow = int(self.nplots*1./ncol + 0.9)
250 251
251 252 return nrow, ncol
252 253
253 254 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
254 255
255 256 self.__showprofile = showprofile
256 257 self.nplots = nplots
257 258
258 259 ncolspan = 1
259 260 colspan = 1
260 261 if showprofile:
261 262 ncolspan = 3
262 263 colspan = 2
263 264 self.__nsubplots = 2
264 265
265 266 self.createFigure(id = id,
266 267 wintitle = wintitle,
267 268 widthplot = self.WIDTH + self.WIDTHPROF,
268 269 heightplot = self.HEIGHT + self.HEIGHTPROF,
269 270 show=show)
270 271
271 272 nrow, ncol = self.getSubplots()
272 273
273 274 counter = 0
274 275 for y in range(nrow):
275 276 for x in range(ncol):
276 277
277 278 if counter >= self.nplots:
278 279 break
279 280
280 281 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
281 282
282 283 if showprofile:
283 284 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
284 285
285 286 counter += 1
286 287
287 288 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
288 289 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
289 290 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
290 291 server=None, folder=None, username=None, password=None,
291 292 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
292 293
293 294 """
294 295
295 296 Input:
296 297 dataOut :
297 298 id :
298 299 wintitle :
299 300 channelList :
300 301 showProfile :
301 302 xmin : None,
302 303 xmax : None,
303 304 ymin : None,
304 305 ymax : None,
305 306 zmin : None,
306 307 zmax : None
307 308 """
308 309
309 310 if dataOut.flagNoData:
310 311 return None
311 312
312 313 if realtime:
313 314 if not(isRealtime(utcdatatime = dataOut.utctime)):
314 315 print('Skipping this plot function')
315 316 return
316 317
317 318 if channelList == None:
318 319 channelIndexList = dataOut.channelIndexList
319 320 else:
320 321 channelIndexList = []
321 322 for channel in channelList:
322 323 if channel not in dataOut.channelList:
323 324 raise ValueError("Channel %d is not in dataOut.channelList")
324 325 channelIndexList.append(dataOut.channelList.index(channel))
325 326
326 327 factor = dataOut.normFactor
327 328 x = dataOut.abscissaList
328 329 y = dataOut.heightList
329 330
330 331 z = dataOut.data_pre[channelIndexList,:,:]/factor
331 332 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
332 333 avg = numpy.average(z, axis=1)
333 334 noise = dataOut.noise/factor
334 335
335 336 zdB = 10*numpy.log10(z)
336 337 avgdB = 10*numpy.log10(avg)
337 338 noisedB = 10*numpy.log10(noise)
338 339
339 340 #thisDatetime = dataOut.datatime
340 341 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
341 342 title = wintitle + " Parameters"
342 343 xlabel = "Velocity (m/s)"
343 344 ylabel = "Range (Km)"
344 345
345 346 update_figfile = False
346 347
347 348 if not self.isConfig:
348 349
349 350 nplots = len(channelIndexList)
350 351
351 352 self.setup(id=id,
352 353 nplots=nplots,
353 354 wintitle=wintitle,
354 355 showprofile=showprofile,
355 356 show=show)
356 357
357 358 if xmin == None: xmin = numpy.nanmin(x)
358 359 if xmax == None: xmax = numpy.nanmax(x)
359 360 if ymin == None: ymin = numpy.nanmin(y)
360 361 if ymax == None: ymax = numpy.nanmax(y)
361 362 if zmin == None: zmin = numpy.nanmin(avgdB)*0.9
362 363 if zmax == None: zmax = numpy.nanmax(avgdB)*0.9
363 364
364 365 self.FTP_WEI = ftp_wei
365 366 self.EXP_CODE = exp_code
366 367 self.SUB_EXP_CODE = sub_exp_code
367 368 self.PLOT_POS = plot_pos
368 369
369 370 self.isConfig = True
370 371 update_figfile = True
371 372
372 373 self.setWinTitle(title)
373 374
374 375 for i in range(self.nplots):
375 376 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
376 377 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[i], noisedB[i], str_datetime)
377 378 axes = self.axesList[i*self.__nsubplots]
378 379 axes.pcolor(x, y, zdB[i,:,:],
379 380 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
380 381 xlabel=xlabel, ylabel=ylabel, title=title,
381 382 ticksize=9, cblabel='')
382 383 #Mean Line
383 384 mean = dataOut.data_param[i, 1, :]
384 385 axes.addpline(mean, y, idline=0, color="black", linestyle="solid", lw=1)
385 386
386 387 if self.__showprofile:
387 388 axes = self.axesList[i*self.__nsubplots +1]
388 389 axes.pline(avgdB[i], y,
389 390 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
390 391 xlabel='dB', ylabel='', title='',
391 392 ytick_visible=False,
392 393 grid='x')
393 394
394 395 noiseline = numpy.repeat(noisedB[i], len(y))
395 396 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
396 397
397 398 self.draw()
398 399
399 400 self.save(figpath=figpath,
400 401 figfile=figfile,
401 402 save=save,
402 403 ftp=ftp,
403 404 wr_period=wr_period,
404 405 thisDatetime=thisDatetime)
405 406
406 407
407
408 408 class SkyMapPlot(Figure):
409 409
410 410 __isConfig = None
411 411 __nsubplots = None
412 412
413 413 WIDTHPROF = None
414 414 HEIGHTPROF = None
415 415 PREFIX = 'mmap'
416 416
417 417 def __init__(self, **kwargs):
418 418 Figure.__init__(self, **kwargs)
419 419 self.isConfig = False
420 420 self.__nsubplots = 1
421 421
422 422 # self.WIDTH = 280
423 423 # self.HEIGHT = 250
424 424 self.WIDTH = 600
425 425 self.HEIGHT = 600
426 426 self.WIDTHPROF = 120
427 427 self.HEIGHTPROF = 0
428 428 self.counter_imagwr = 0
429 429
430 430 self.PLOT_CODE = MSKYMAP_CODE
431 431
432 432 self.FTP_WEI = None
433 433 self.EXP_CODE = None
434 434 self.SUB_EXP_CODE = None
435 435 self.PLOT_POS = None
436 436
437 437 def getSubplots(self):
438 438
439 439 ncol = int(numpy.sqrt(self.nplots)+0.9)
440 440 nrow = int(self.nplots*1./ncol + 0.9)
441 441
442 442 return nrow, ncol
443 443
444 444 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
445 445
446 446 self.__showprofile = showprofile
447 447 self.nplots = nplots
448 448
449 449 ncolspan = 1
450 450 colspan = 1
451 451
452 452 self.createFigure(id = id,
453 453 wintitle = wintitle,
454 454 widthplot = self.WIDTH, #+ self.WIDTHPROF,
455 455 heightplot = self.HEIGHT,# + self.HEIGHTPROF,
456 456 show=show)
457 457
458 458 nrow, ncol = 1,1
459 459 counter = 0
460 460 x = 0
461 461 y = 0
462 462 self.addAxes(1, 1, 0, 0, 1, 1, True)
463 463
464 464 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
465 465 tmin=0, tmax=24, timerange=None,
466 466 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
467 467 server=None, folder=None, username=None, password=None,
468 468 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
469 469
470 470 """
471 471
472 472 Input:
473 473 dataOut :
474 474 id :
475 475 wintitle :
476 476 channelList :
477 477 showProfile :
478 478 xmin : None,
479 479 xmax : None,
480 480 ymin : None,
481 481 ymax : None,
482 482 zmin : None,
483 483 zmax : None
484 484 """
485 485
486 486 arrayParameters = dataOut.data_param
487 487 error = arrayParameters[:,-1]
488 488 indValid = numpy.where(error == 0)[0]
489 489 finalMeteor = arrayParameters[indValid,:]
490 490 finalAzimuth = finalMeteor[:,3]
491 491 finalZenith = finalMeteor[:,4]
492 492
493 493 x = finalAzimuth*numpy.pi/180
494 494 y = finalZenith
495 495 x1 = [dataOut.ltctime, dataOut.ltctime]
496 496
497 497 #thisDatetime = dataOut.datatime
498 498 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
499 499 title = wintitle + " Parameters"
500 500 xlabel = "Zonal Zenith Angle (deg) "
501 501 ylabel = "Meridional Zenith Angle (deg)"
502 502 update_figfile = False
503 503
504 504 if not self.isConfig:
505 505
506 506 nplots = 1
507 507
508 508 self.setup(id=id,
509 509 nplots=nplots,
510 510 wintitle=wintitle,
511 511 showprofile=showprofile,
512 512 show=show)
513 513
514 514 if self.xmin is None and self.xmax is None:
515 515 self.xmin, self.xmax = self.getTimeLim(x1, tmin, tmax, timerange)
516 516
517 517 if timerange != None:
518 518 self.timerange = timerange
519 519 else:
520 520 self.timerange = self.xmax - self.xmin
521 521
522 522 self.FTP_WEI = ftp_wei
523 523 self.EXP_CODE = exp_code
524 524 self.SUB_EXP_CODE = sub_exp_code
525 525 self.PLOT_POS = plot_pos
526 526 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
527 527 self.firstdate = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
528 528 self.isConfig = True
529 529 update_figfile = True
530 530
531 531 self.setWinTitle(title)
532 532
533 533 i = 0
534 534 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
535 535
536 536 axes = self.axesList[i*self.__nsubplots]
537 537 nevents = axes.x_buffer.shape[0] + x.shape[0]
538 538 title = "Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n" %(self.firstdate,str_datetime,nevents)
539 539 axes.polar(x, y,
540 540 title=title, xlabel=xlabel, ylabel=ylabel,
541 541 ticksize=9, cblabel='')
542 542
543 543 self.draw()
544 544
545 545 self.save(figpath=figpath,
546 546 figfile=figfile,
547 547 save=save,
548 548 ftp=ftp,
549 549 wr_period=wr_period,
550 550 thisDatetime=thisDatetime,
551 551 update_figfile=update_figfile)
552 552
553 553 if dataOut.ltctime >= self.xmax:
554 554 self.isConfigmagwr = wr_period
555 555 self.isConfig = False
556 556 update_figfile = True
557 557 axes.__firsttime = True
558 558 self.xmin += self.timerange
559 559 self.xmax += self.timerange
560 560
561 561
562 562
563 563
564 564 class WindProfilerPlot(Figure):
565 565
566 566 __isConfig = None
567 567 __nsubplots = None
568 568
569 569 WIDTHPROF = None
570 570 HEIGHTPROF = None
571 571 PREFIX = 'wind'
572 572
573 573 def __init__(self, **kwargs):
574 574 Figure.__init__(self, **kwargs)
575 575 self.timerange = None
576 576 self.isConfig = False
577 577 self.__nsubplots = 1
578 578
579 579 self.WIDTH = 800
580 580 self.HEIGHT = 300
581 581 self.WIDTHPROF = 120
582 582 self.HEIGHTPROF = 0
583 583 self.counter_imagwr = 0
584 584
585 585 self.PLOT_CODE = WIND_CODE
586 586
587 587 self.FTP_WEI = None
588 588 self.EXP_CODE = None
589 589 self.SUB_EXP_CODE = None
590 590 self.PLOT_POS = None
591 591 self.tmin = None
592 592 self.tmax = None
593 593
594 594 self.xmin = None
595 595 self.xmax = None
596 596
597 597 self.figfile = None
598 598
599 599 def getSubplots(self):
600 600
601 601 ncol = 1
602 602 nrow = self.nplots
603 603
604 604 return nrow, ncol
605 605
606 606 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
607 607
608 608 self.__showprofile = showprofile
609 609 self.nplots = nplots
610 610
611 611 ncolspan = 1
612 612 colspan = 1
613 613
614 614 self.createFigure(id = id,
615 615 wintitle = wintitle,
616 616 widthplot = self.WIDTH + self.WIDTHPROF,
617 617 heightplot = self.HEIGHT + self.HEIGHTPROF,
618 618 show=show)
619 619
620 620 nrow, ncol = self.getSubplots()
621 621
622 622 counter = 0
623 623 for y in range(nrow):
624 624 if counter >= self.nplots:
625 625 break
626 626
627 627 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
628 628 counter += 1
629 629
630 630 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='False',
631 631 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
632 632 zmax_ver = None, zmin_ver = None, SNRmin = None, SNRmax = None,
633 633 timerange=None, SNRthresh = None,
634 634 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
635 635 server=None, folder=None, username=None, password=None,
636 636 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
637 637 """
638 638
639 639 Input:
640 640 dataOut :
641 641 id :
642 642 wintitle :
643 643 channelList :
644 644 showProfile :
645 645 xmin : None,
646 646 xmax : None,
647 647 ymin : None,
648 648 ymax : None,
649 649 zmin : None,
650 650 zmax : None
651 651 """
652 652
653 653 # if timerange is not None:
654 654 # self.timerange = timerange
655 655 #
656 656 # tmin = None
657 657 # tmax = None
658 658
659 659 x = dataOut.getTimeRange1(dataOut.paramInterval)
660 660 y = dataOut.heightList
661 661 z = dataOut.data_output.copy()
662 662 nplots = z.shape[0] #Number of wind dimensions estimated
663 663 nplotsw = nplots
664 664
665 665
666 666 #If there is a SNR function defined
667 667 if dataOut.data_SNR is not None:
668 668 nplots += 1
669 669 SNR = dataOut.data_SNR
670 670 SNRavg = numpy.average(SNR, axis=0)
671 671
672 672 SNRdB = 10*numpy.log10(SNR)
673 673 SNRavgdB = 10*numpy.log10(SNRavg)
674 674
675 675 if SNRthresh == None: SNRthresh = -5.0
676 676 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
677 677
678 678 for i in range(nplotsw):
679 679 z[i,ind] = numpy.nan
680 680
681 681 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
682 682 #thisDatetime = datetime.datetime.now()
683 683 title = wintitle + "Wind"
684 684 xlabel = ""
685 685 ylabel = "Height (km)"
686 686 update_figfile = False
687 687
688 688 if not self.isConfig:
689 689
690 690 self.setup(id=id,
691 691 nplots=nplots,
692 692 wintitle=wintitle,
693 693 showprofile=showprofile,
694 694 show=show)
695 695
696 696 if timerange is not None:
697 697 self.timerange = timerange
698 698
699 699 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
700 700
701 701 if ymin == None: ymin = numpy.nanmin(y)
702 702 if ymax == None: ymax = numpy.nanmax(y)
703 703
704 704 if zmax == None: zmax = numpy.nanmax(abs(z[list(range(2)),:]))
705 705 #if numpy.isnan(zmax): zmax = 50
706 706 if zmin == None: zmin = -zmax
707 707
708 708 if nplotsw == 3:
709 709 if zmax_ver == None: zmax_ver = numpy.nanmax(abs(z[2,:]))
710 710 if zmin_ver == None: zmin_ver = -zmax_ver
711 711
712 712 if dataOut.data_SNR is not None:
713 713 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
714 714 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
715 715
716 716
717 717 self.FTP_WEI = ftp_wei
718 718 self.EXP_CODE = exp_code
719 719 self.SUB_EXP_CODE = sub_exp_code
720 720 self.PLOT_POS = plot_pos
721 721
722 722 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
723 723 self.isConfig = True
724 724 self.figfile = figfile
725 725 update_figfile = True
726 726
727 727 self.setWinTitle(title)
728 728
729 729 if ((self.xmax - x[1]) < (x[1]-x[0])):
730 730 x[1] = self.xmax
731 731
732 732 strWind = ['Zonal', 'Meridional', 'Vertical']
733 733 strCb = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
734 734 zmaxVector = [zmax, zmax, zmax_ver]
735 735 zminVector = [zmin, zmin, zmin_ver]
736 736 windFactor = [1,1,100]
737 737
738 738 for i in range(nplotsw):
739 739
740 740 title = "%s Wind: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
741 741 axes = self.axesList[i*self.__nsubplots]
742 742
743 743 z1 = z[i,:].reshape((1,-1))*windFactor[i]
744 744 #z1=numpy.ma.masked_where(z1==0.,z1)
745 745
746 746 axes.pcolorbuffer(x, y, z1,
747 747 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
748 748 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
749 749 ticksize=9, cblabel=strCb[i], cbsize="1%", colormap="seismic" )
750 750
751 751 if dataOut.data_SNR is not None:
752 752 i += 1
753 753 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
754 754 axes = self.axesList[i*self.__nsubplots]
755 755 SNRavgdB = SNRavgdB.reshape((1,-1))
756 756 axes.pcolorbuffer(x, y, SNRavgdB,
757 757 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
758 758 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
759 759 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
760 760
761 761 self.draw()
762 762
763 763 self.save(figpath=figpath,
764 764 figfile=figfile,
765 765 save=save,
766 766 ftp=ftp,
767 767 wr_period=wr_period,
768 768 thisDatetime=thisDatetime,
769 769 update_figfile=update_figfile)
770 770
771 771 if dataOut.ltctime + dataOut.paramInterval >= self.xmax:
772 772 self.counter_imagwr = wr_period
773 773 self.isConfig = False
774 774 update_figfile = True
775 775
776
776 @MPDecorator
777 777 class ParametersPlot(Figure):
778 778
779 779 __isConfig = None
780 780 __nsubplots = None
781 781
782 782 WIDTHPROF = None
783 783 HEIGHTPROF = None
784 784 PREFIX = 'param'
785 785
786 786 nplots = None
787 787 nchan = None
788 788
789 def __init__(self, **kwargs):
790 Figure.__init__(self, **kwargs)
789 def __init__(self):#, **kwargs):
790 Figure.__init__(self)#, **kwargs)
791 791 self.timerange = None
792 792 self.isConfig = False
793 793 self.__nsubplots = 1
794 794
795 795 self.WIDTH = 800
796 796 self.HEIGHT = 180
797 797 self.WIDTHPROF = 120
798 798 self.HEIGHTPROF = 0
799 799 self.counter_imagwr = 0
800 800
801 801 self.PLOT_CODE = RTI_CODE
802 802
803 803 self.FTP_WEI = None
804 804 self.EXP_CODE = None
805 805 self.SUB_EXP_CODE = None
806 806 self.PLOT_POS = None
807 807 self.tmin = None
808 808 self.tmax = None
809 809
810 810 self.xmin = None
811 811 self.xmax = None
812 812
813 813 self.figfile = None
814 814
815 815 def getSubplots(self):
816 816
817 817 ncol = 1
818 818 nrow = self.nplots
819 819
820 820 return nrow, ncol
821 821
822 822 def setup(self, id, nplots, wintitle, show=True):
823 823
824 824 self.nplots = nplots
825 825
826 826 ncolspan = 1
827 827 colspan = 1
828 828
829 829 self.createFigure(id = id,
830 830 wintitle = wintitle,
831 831 widthplot = self.WIDTH + self.WIDTHPROF,
832 832 heightplot = self.HEIGHT + self.HEIGHTPROF,
833 833 show=show)
834 834
835 835 nrow, ncol = self.getSubplots()
836 836
837 837 counter = 0
838 838 for y in range(nrow):
839 839 for x in range(ncol):
840 840
841 841 if counter >= self.nplots:
842 842 break
843 843
844 844 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
845 845
846 846 counter += 1
847 847
848 848 def run(self, dataOut, id, wintitle="", channelList=None, paramIndex = 0, colormap="jet",
849 849 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None, timerange=None,
850 850 showSNR=False, SNRthresh = -numpy.inf, SNRmin=None, SNRmax=None,
851 851 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
852 852 server=None, folder=None, username=None, password=None,
853 853 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, HEIGHT=None):
854 854 """
855 855
856 856 Input:
857 857 dataOut :
858 858 id :
859 859 wintitle :
860 860 channelList :
861 861 showProfile :
862 862 xmin : None,
863 863 xmax : None,
864 864 ymin : None,
865 865 ymax : None,
866 866 zmin : None,
867 867 zmax : None
868 868 """
869 if dataOut.flagNoData:
870 return dataOut
871
869 872
870 873 if HEIGHT is not None:
871 874 self.HEIGHT = HEIGHT
872 875
873 876
874 877 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
875 878 return
876 879
877 880 if channelList == None:
878 881 channelIndexList = list(range(dataOut.data_param.shape[0]))
879 882 else:
880 883 channelIndexList = []
881 884 for channel in channelList:
882 885 if channel not in dataOut.channelList:
883 886 raise ValueError("Channel %d is not in dataOut.channelList")
884 887 channelIndexList.append(dataOut.channelList.index(channel))
885 888
886 889 x = dataOut.getTimeRange1(dataOut.paramInterval)
887 890 y = dataOut.getHeiRange()
888 891
889 892 if dataOut.data_param.ndim == 3:
890 893 z = dataOut.data_param[channelIndexList,paramIndex,:]
891 894 else:
892 895 z = dataOut.data_param[channelIndexList,:]
893 896
894 897 if showSNR:
895 898 #SNR data
896 899 SNRarray = dataOut.data_SNR[channelIndexList,:]
897 900 SNRdB = 10*numpy.log10(SNRarray)
898 901 ind = numpy.where(SNRdB < SNRthresh)
899 902 z[ind] = numpy.nan
900 903
901 904 thisDatetime = dataOut.datatime
902 905 # thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
903 906 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
904 907 xlabel = ""
905 908 ylabel = "Range (Km)"
906 909
907 910 update_figfile = False
908 911
909 912 if not self.isConfig:
910 913
911 914 nchan = len(channelIndexList)
912 915 self.nchan = nchan
913 916 self.plotFact = 1
914 917 nplots = nchan
915 918
916 919 if showSNR:
917 920 nplots = nchan*2
918 921 self.plotFact = 2
919 922 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
920 923 if SNRmax == None: SNRmax = numpy.nanmax(SNRdB)
921 924
922 925 self.setup(id=id,
923 926 nplots=nplots,
924 927 wintitle=wintitle,
925 928 show=show)
926 929
927 930 if timerange != None:
928 931 self.timerange = timerange
929 932
930 933 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
931 934
932 935 if ymin == None: ymin = numpy.nanmin(y)
933 936 if ymax == None: ymax = numpy.nanmax(y)
934 937 if zmin == None: zmin = numpy.nanmin(z)
935 938 if zmax == None: zmax = numpy.nanmax(z)
936 939
937 940 self.FTP_WEI = ftp_wei
938 941 self.EXP_CODE = exp_code
939 942 self.SUB_EXP_CODE = sub_exp_code
940 943 self.PLOT_POS = plot_pos
941 944
942 945 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
943 946 self.isConfig = True
944 947 self.figfile = figfile
945 948 update_figfile = True
946 949
947 950 self.setWinTitle(title)
948 951
949 952 for i in range(self.nchan):
950 953 index = channelIndexList[i]
951 954 title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
952 955 axes = self.axesList[i*self.plotFact]
953 956 z1 = z[i,:].reshape((1,-1))
954 957 axes.pcolorbuffer(x, y, z1,
955 958 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
956 959 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
957 960 ticksize=9, cblabel='', cbsize="1%",colormap=colormap)
958 961
959 962 if showSNR:
960 963 title = "Channel %d SNR: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
961 964 axes = self.axesList[i*self.plotFact + 1]
962 965 SNRdB1 = SNRdB[i,:].reshape((1,-1))
963 966 axes.pcolorbuffer(x, y, SNRdB1,
964 967 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
965 968 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
966 969 ticksize=9, cblabel='', cbsize="1%",colormap='jet')
967 970
968 971
969 972 self.draw()
970 973
971 974 if dataOut.ltctime >= self.xmax:
972 975 self.counter_imagwr = wr_period
973 976 self.isConfig = False
974 977 update_figfile = True
975 978
976 979 self.save(figpath=figpath,
977 980 figfile=figfile,
978 981 save=save,
979 982 ftp=ftp,
980 983 wr_period=wr_period,
981 984 thisDatetime=thisDatetime,
982 985 update_figfile=update_figfile)
983 986
984
985
987 return dataOut
988 @MPDecorator
986 989 class Parameters1Plot(Figure):
987 990
988 991 __isConfig = None
989 992 __nsubplots = None
990 993
991 994 WIDTHPROF = None
992 995 HEIGHTPROF = None
993 996 PREFIX = 'prm'
994 997
995 def __init__(self, **kwargs):
996 Figure.__init__(self, **kwargs)
998 def __init__(self):
999 Figure.__init__(self)
997 1000 self.timerange = 2*60*60
998 1001 self.isConfig = False
999 1002 self.__nsubplots = 1
1000 1003
1001 1004 self.WIDTH = 800
1002 1005 self.HEIGHT = 180
1003 1006 self.WIDTHPROF = 120
1004 1007 self.HEIGHTPROF = 0
1005 1008 self.counter_imagwr = 0
1006 1009
1007 1010 self.PLOT_CODE = PARMS_CODE
1008 1011
1009 1012 self.FTP_WEI = None
1010 1013 self.EXP_CODE = None
1011 1014 self.SUB_EXP_CODE = None
1012 1015 self.PLOT_POS = None
1013 1016 self.tmin = None
1014 1017 self.tmax = None
1015 1018
1016 1019 self.xmin = None
1017 1020 self.xmax = None
1018 1021
1019 1022 self.figfile = None
1020 1023
1021 1024 def getSubplots(self):
1022 1025
1023 1026 ncol = 1
1024 1027 nrow = self.nplots
1025 1028
1026 1029 return nrow, ncol
1027 1030
1028 1031 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1029 1032
1030 1033 self.__showprofile = showprofile
1031 1034 self.nplots = nplots
1032 1035
1033 1036 ncolspan = 1
1034 1037 colspan = 1
1035 1038
1036 1039 self.createFigure(id = id,
1037 1040 wintitle = wintitle,
1038 1041 widthplot = self.WIDTH + self.WIDTHPROF,
1039 1042 heightplot = self.HEIGHT + self.HEIGHTPROF,
1040 1043 show=show)
1041 1044
1042 1045 nrow, ncol = self.getSubplots()
1043 1046
1044 1047 counter = 0
1045 1048 for y in range(nrow):
1046 1049 for x in range(ncol):
1047 1050
1048 1051 if counter >= self.nplots:
1049 1052 break
1050 1053
1051 1054 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1052 1055
1053 1056 if showprofile:
1054 1057 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
1055 1058
1056 1059 counter += 1
1057 1060
1058 1061 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
1059 1062 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,timerange=None,
1060 1063 parameterIndex = None, onlyPositive = False,
1061 1064 SNRthresh = -numpy.inf, SNR = True, SNRmin = None, SNRmax = None, onlySNR = False,
1062 1065 DOP = True,
1063 1066 zlabel = "", parameterName = "", parameterObject = "data_param",
1064 1067 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
1065 1068 server=None, folder=None, username=None, password=None,
1066 1069 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1067 1070 #print inspect.getargspec(self.run).args
1068 1071 """
1069 1072
1070 1073 Input:
1071 1074 dataOut :
1072 1075 id :
1073 1076 wintitle :
1074 1077 channelList :
1075 1078 showProfile :
1076 1079 xmin : None,
1077 1080 xmax : None,
1078 1081 ymin : None,
1079 1082 ymax : None,
1080 1083 zmin : None,
1081 1084 zmax : None
1082 1085 """
1086 if dataOut.flagNoData:
1087 return dataOut
1083 1088
1084 1089 data_param = getattr(dataOut, parameterObject)
1085 1090
1086 1091 if channelList == None:
1087 1092 channelIndexList = numpy.arange(data_param.shape[0])
1088 1093 else:
1089 1094 channelIndexList = numpy.array(channelList)
1090 1095
1091 1096 nchan = len(channelIndexList) #Number of channels being plotted
1092 1097
1093 1098 if nchan < 1:
1094 1099 return
1095 1100
1096 1101 nGraphsByChannel = 0
1097 1102
1098 1103 if SNR:
1099 1104 nGraphsByChannel += 1
1100 1105 if DOP:
1101 1106 nGraphsByChannel += 1
1102 1107
1103 1108 if nGraphsByChannel < 1:
1104 1109 return
1105 1110
1106 1111 nplots = nGraphsByChannel*nchan
1107 1112
1108 1113 if timerange is not None:
1109 1114 self.timerange = timerange
1110 1115
1111 1116 #tmin = None
1112 1117 #tmax = None
1113 1118 if parameterIndex == None:
1114 1119 parameterIndex = 1
1115 1120
1116 1121 x = dataOut.getTimeRange1(dataOut.paramInterval)
1117 1122 y = dataOut.heightList
1118 1123
1119 1124 if dataOut.data_param.ndim == 3:
1120 1125 z = dataOut.data_param[channelIndexList,parameterIndex,:]
1121 1126 else:
1122 1127 z = dataOut.data_param[channelIndexList,:]
1123 1128
1124 1129 if dataOut.data_SNR is not None:
1125 1130 if dataOut.data_SNR.ndim == 2:
1126 1131 SNRavg = numpy.average(dataOut.data_SNR, axis=0)
1127 1132 else:
1128 1133 SNRavg = dataOut.data_SNR
1129 1134 SNRdB = 10*numpy.log10(SNRavg)
1130 1135
1131 1136 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1132 1137 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
1133 1138 xlabel = ""
1134 1139 ylabel = "Range (Km)"
1135 1140
1136 1141 if onlyPositive:
1137 1142 colormap = "jet"
1138 1143 zmin = 0
1139 1144 else: colormap = "RdBu_r"
1140 1145
1141 1146 if not self.isConfig:
1142 1147
1143 1148 self.setup(id=id,
1144 1149 nplots=nplots,
1145 1150 wintitle=wintitle,
1146 1151 showprofile=showprofile,
1147 1152 show=show)
1148 1153
1149 1154 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1150 1155
1151 1156 if ymin == None: ymin = numpy.nanmin(y)
1152 1157 if ymax == None: ymax = numpy.nanmax(y)
1153 1158 if zmin == None: zmin = numpy.nanmin(z)
1154 1159 if zmax == None: zmax = numpy.nanmax(z)
1155 1160
1156 1161 if SNR:
1157 1162 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
1158 1163 if SNRmax == None: SNRmax = numpy.nanmax(SNRdB)
1159 1164
1160 1165 self.FTP_WEI = ftp_wei
1161 1166 self.EXP_CODE = exp_code
1162 1167 self.SUB_EXP_CODE = sub_exp_code
1163 1168 self.PLOT_POS = plot_pos
1164 1169
1165 1170 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1166 1171 self.isConfig = True
1167 1172 self.figfile = figfile
1168 1173
1169 1174 self.setWinTitle(title)
1170 1175
1171 1176 if ((self.xmax - x[1]) < (x[1]-x[0])):
1172 1177 x[1] = self.xmax
1173 1178
1174 1179 for i in range(nchan):
1175 1180
1176 1181 if (SNR and not onlySNR): j = 2*i
1177 1182 else: j = i
1178 1183
1179 1184 j = nGraphsByChannel*i
1180 1185
1181 1186 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1182 1187 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
1183 1188
1184 1189 if not onlySNR:
1185 1190 axes = self.axesList[j*self.__nsubplots]
1186 1191 z1 = z[i,:].reshape((1,-1))
1187 1192 axes.pcolorbuffer(x, y, z1,
1188 1193 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
1189 1194 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
1190 1195 ticksize=9, cblabel=zlabel, cbsize="1%")
1191 1196
1192 1197 if DOP:
1193 1198 title = "%s Channel %d: %s" %(parameterName, channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1194 1199
1195 1200 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1196 1201 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
1197 1202 axes = self.axesList[j]
1198 1203 z1 = z[i,:].reshape((1,-1))
1199 1204 axes.pcolorbuffer(x, y, z1,
1200 1205 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
1201 1206 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
1202 1207 ticksize=9, cblabel=zlabel, cbsize="1%")
1203 1208
1204 1209 if SNR:
1205 1210 title = "Channel %d Signal Noise Ratio (SNR): %s" %(channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1206 1211 axes = self.axesList[(j)*self.__nsubplots]
1207 1212 if not onlySNR:
1208 1213 axes = self.axesList[(j + 1)*self.__nsubplots]
1209 1214
1210 1215 axes = self.axesList[(j + nGraphsByChannel-1)]
1211 1216 z1 = SNRdB.reshape((1,-1))
1212 1217 axes.pcolorbuffer(x, y, z1,
1213 1218 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1214 1219 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap="jet",
1215 1220 ticksize=9, cblabel=zlabel, cbsize="1%")
1216 1221
1217 1222
1218 1223
1219 1224 self.draw()
1220 1225
1221 1226 if x[1] >= self.axesList[0].xmax:
1222 1227 self.counter_imagwr = wr_period
1223 1228 self.isConfig = False
1224 1229 self.figfile = None
1225 1230
1226 1231 self.save(figpath=figpath,
1227 1232 figfile=figfile,
1228 1233 save=save,
1229 1234 ftp=ftp,
1230 1235 wr_period=wr_period,
1231 1236 thisDatetime=thisDatetime,
1232 1237 update_figfile=False)
1238 return dataOut
1233 1239
1234 1240 class SpectralFittingPlot(Figure):
1235 1241
1236 1242 __isConfig = None
1237 1243 __nsubplots = None
1238 1244
1239 1245 WIDTHPROF = None
1240 1246 HEIGHTPROF = None
1241 1247 PREFIX = 'prm'
1242 1248
1243 1249
1244 1250 N = None
1245 1251 ippSeconds = None
1246 1252
1247 1253 def __init__(self, **kwargs):
1248 1254 Figure.__init__(self, **kwargs)
1249 1255 self.isConfig = False
1250 1256 self.__nsubplots = 1
1251 1257
1252 1258 self.PLOT_CODE = SPECFIT_CODE
1253 1259
1254 1260 self.WIDTH = 450
1255 1261 self.HEIGHT = 250
1256 1262 self.WIDTHPROF = 0
1257 1263 self.HEIGHTPROF = 0
1258 1264
1259 1265 def getSubplots(self):
1260 1266
1261 1267 ncol = int(numpy.sqrt(self.nplots)+0.9)
1262 1268 nrow = int(self.nplots*1./ncol + 0.9)
1263 1269
1264 1270 return nrow, ncol
1265 1271
1266 1272 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
1267 1273
1268 1274 showprofile = False
1269 1275 self.__showprofile = showprofile
1270 1276 self.nplots = nplots
1271 1277
1272 1278 ncolspan = 5
1273 1279 colspan = 4
1274 1280 if showprofile:
1275 1281 ncolspan = 5
1276 1282 colspan = 4
1277 1283 self.__nsubplots = 2
1278 1284
1279 1285 self.createFigure(id = id,
1280 1286 wintitle = wintitle,
1281 1287 widthplot = self.WIDTH + self.WIDTHPROF,
1282 1288 heightplot = self.HEIGHT + self.HEIGHTPROF,
1283 1289 show=show)
1284 1290
1285 1291 nrow, ncol = self.getSubplots()
1286 1292
1287 1293 counter = 0
1288 1294 for y in range(nrow):
1289 1295 for x in range(ncol):
1290 1296
1291 1297 if counter >= self.nplots:
1292 1298 break
1293 1299
1294 1300 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1295 1301
1296 1302 if showprofile:
1297 1303 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
1298 1304
1299 1305 counter += 1
1300 1306
1301 1307 def run(self, dataOut, id, cutHeight=None, fit=False, wintitle="", channelList=None, showprofile=True,
1302 1308 xmin=None, xmax=None, ymin=None, ymax=None,
1303 1309 save=False, figpath='./', figfile=None, show=True):
1304 1310
1305 1311 """
1306 1312
1307 1313 Input:
1308 1314 dataOut :
1309 1315 id :
1310 1316 wintitle :
1311 1317 channelList :
1312 1318 showProfile :
1313 1319 xmin : None,
1314 1320 xmax : None,
1315 1321 zmin : None,
1316 1322 zmax : None
1317 1323 """
1318 1324
1319 1325 if cutHeight==None:
1320 1326 h=270
1321 1327 heightindex = numpy.abs(cutHeight - dataOut.heightList).argmin()
1322 1328 cutHeight = dataOut.heightList[heightindex]
1323 1329
1324 1330 factor = dataOut.normFactor
1325 1331 x = dataOut.abscissaList[:-1]
1326 1332 #y = dataOut.getHeiRange()
1327 1333
1328 1334 z = dataOut.data_pre[:,:,heightindex]/factor
1329 1335 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1330 1336 avg = numpy.average(z, axis=1)
1331 1337 listChannels = z.shape[0]
1332 1338
1333 1339 #Reconstruct Function
1334 1340 if fit==True:
1335 1341 groupArray = dataOut.groupList
1336 1342 listChannels = groupArray.reshape((groupArray.size))
1337 1343 listChannels.sort()
1338 1344 spcFitLine = numpy.zeros(z.shape)
1339 1345 constants = dataOut.constants
1340 1346
1341 1347 nGroups = groupArray.shape[0]
1342 1348 nChannels = groupArray.shape[1]
1343 1349 nProfiles = z.shape[1]
1344 1350
1345 1351 for f in range(nGroups):
1346 1352 groupChann = groupArray[f,:]
1347 1353 p = dataOut.data_param[f,:,heightindex]
1348 1354 # p = numpy.array([ 89.343967,0.14036615,0.17086219,18.89835291,1.58388365,1.55099167])
1349 1355 fitLineAux = dataOut.library.modelFunction(p, constants)*nProfiles
1350 1356 fitLineAux = fitLineAux.reshape((nChannels,nProfiles))
1351 1357 spcFitLine[groupChann,:] = fitLineAux
1352 1358 # spcFitLine = spcFitLine/factor
1353 1359
1354 1360 z = z[listChannels,:]
1355 1361 spcFitLine = spcFitLine[listChannels,:]
1356 1362 spcFitLinedB = 10*numpy.log10(spcFitLine)
1357 1363
1358 1364 zdB = 10*numpy.log10(z)
1359 1365 #thisDatetime = dataOut.datatime
1360 1366 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1361 1367 title = wintitle + " Doppler Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1362 1368 xlabel = "Velocity (m/s)"
1363 1369 ylabel = "Spectrum"
1364 1370
1365 1371 if not self.isConfig:
1366 1372
1367 1373 nplots = listChannels.size
1368 1374
1369 1375 self.setup(id=id,
1370 1376 nplots=nplots,
1371 1377 wintitle=wintitle,
1372 1378 showprofile=showprofile,
1373 1379 show=show)
1374 1380
1375 1381 if xmin == None: xmin = numpy.nanmin(x)
1376 1382 if xmax == None: xmax = numpy.nanmax(x)
1377 1383 if ymin == None: ymin = numpy.nanmin(zdB)
1378 1384 if ymax == None: ymax = numpy.nanmax(zdB)+2
1379 1385
1380 1386 self.isConfig = True
1381 1387
1382 1388 self.setWinTitle(title)
1383 1389 for i in range(self.nplots):
1384 1390 # title = "Channel %d: %4.2fdB" %(dataOut.channelList[i]+1, noisedB[i])
1385 1391 title = "Height %4.1f km\nChannel %d:" %(cutHeight, listChannels[i])
1386 1392 axes = self.axesList[i*self.__nsubplots]
1387 1393 if fit == False:
1388 1394 axes.pline(x, zdB[i,:],
1389 1395 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1390 1396 xlabel=xlabel, ylabel=ylabel, title=title
1391 1397 )
1392 1398 if fit == True:
1393 1399 fitline=spcFitLinedB[i,:]
1394 1400 y=numpy.vstack([zdB[i,:],fitline] )
1395 1401 legendlabels=['Data','Fitting']
1396 1402 axes.pmultilineyaxis(x, y,
1397 1403 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1398 1404 xlabel=xlabel, ylabel=ylabel, title=title,
1399 1405 legendlabels=legendlabels, marker=None,
1400 1406 linestyle='solid', grid='both')
1401 1407
1402 1408 self.draw()
1403 1409
1404 1410 self.save(figpath=figpath,
1405 1411 figfile=figfile,
1406 1412 save=save,
1407 1413 ftp=ftp,
1408 1414 wr_period=wr_period,
1409 1415 thisDatetime=thisDatetime)
1410 1416
1411 1417
1412 1418 class EWDriftsPlot(Figure):
1413 1419
1414 1420 __isConfig = None
1415 1421 __nsubplots = None
1416 1422
1417 1423 WIDTHPROF = None
1418 1424 HEIGHTPROF = None
1419 1425 PREFIX = 'drift'
1420 1426
1421 1427 def __init__(self, **kwargs):
1422 1428 Figure.__init__(self, **kwargs)
1423 1429 self.timerange = 2*60*60
1424 1430 self.isConfig = False
1425 1431 self.__nsubplots = 1
1426 1432
1427 1433 self.WIDTH = 800
1428 1434 self.HEIGHT = 150
1429 1435 self.WIDTHPROF = 120
1430 1436 self.HEIGHTPROF = 0
1431 1437 self.counter_imagwr = 0
1432 1438
1433 1439 self.PLOT_CODE = EWDRIFT_CODE
1434 1440
1435 1441 self.FTP_WEI = None
1436 1442 self.EXP_CODE = None
1437 1443 self.SUB_EXP_CODE = None
1438 1444 self.PLOT_POS = None
1439 1445 self.tmin = None
1440 1446 self.tmax = None
1441 1447
1442 1448 self.xmin = None
1443 1449 self.xmax = None
1444 1450
1445 1451 self.figfile = None
1446 1452
1447 1453 def getSubplots(self):
1448 1454
1449 1455 ncol = 1
1450 1456 nrow = self.nplots
1451 1457
1452 1458 return nrow, ncol
1453 1459
1454 1460 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1455 1461
1456 1462 self.__showprofile = showprofile
1457 1463 self.nplots = nplots
1458 1464
1459 1465 ncolspan = 1
1460 1466 colspan = 1
1461 1467
1462 1468 self.createFigure(id = id,
1463 1469 wintitle = wintitle,
1464 1470 widthplot = self.WIDTH + self.WIDTHPROF,
1465 1471 heightplot = self.HEIGHT + self.HEIGHTPROF,
1466 1472 show=show)
1467 1473
1468 1474 nrow, ncol = self.getSubplots()
1469 1475
1470 1476 counter = 0
1471 1477 for y in range(nrow):
1472 1478 if counter >= self.nplots:
1473 1479 break
1474 1480
1475 1481 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
1476 1482 counter += 1
1477 1483
1478 1484 def run(self, dataOut, id, wintitle="", channelList=None,
1479 1485 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
1480 1486 zmaxVertical = None, zminVertical = None, zmaxZonal = None, zminZonal = None,
1481 1487 timerange=None, SNRthresh = -numpy.inf, SNRmin = None, SNRmax = None, SNR_1 = False,
1482 1488 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
1483 1489 server=None, folder=None, username=None, password=None,
1484 1490 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1485 1491 """
1486 1492
1487 1493 Input:
1488 1494 dataOut :
1489 1495 id :
1490 1496 wintitle :
1491 1497 channelList :
1492 1498 showProfile :
1493 1499 xmin : None,
1494 1500 xmax : None,
1495 1501 ymin : None,
1496 1502 ymax : None,
1497 1503 zmin : None,
1498 1504 zmax : None
1499 1505 """
1500 1506
1501 1507 if timerange is not None:
1502 1508 self.timerange = timerange
1503 1509
1504 1510 tmin = None
1505 1511 tmax = None
1506 1512
1507 1513 x = dataOut.getTimeRange1(dataOut.outputInterval)
1508 1514 # y = dataOut.heightList
1509 1515 y = dataOut.heightList
1510 1516
1511 1517 z = dataOut.data_output
1512 1518 nplots = z.shape[0] #Number of wind dimensions estimated
1513 1519 nplotsw = nplots
1514 1520
1515 1521 #If there is a SNR function defined
1516 1522 if dataOut.data_SNR is not None:
1517 1523 nplots += 1
1518 1524 SNR = dataOut.data_SNR
1519 1525
1520 1526 if SNR_1:
1521 1527 SNR += 1
1522 1528
1523 1529 SNRavg = numpy.average(SNR, axis=0)
1524 1530
1525 1531 SNRdB = 10*numpy.log10(SNR)
1526 1532 SNRavgdB = 10*numpy.log10(SNRavg)
1527 1533
1528 1534 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
1529 1535
1530 1536 for i in range(nplotsw):
1531 1537 z[i,ind] = numpy.nan
1532 1538
1533 1539
1534 1540 showprofile = False
1535 1541 # thisDatetime = dataOut.datatime
1536 1542 thisDatetime = datetime.datetime.utcfromtimestamp(x[1])
1537 1543 title = wintitle + " EW Drifts"
1538 1544 xlabel = ""
1539 1545 ylabel = "Height (Km)"
1540 1546
1541 1547 if not self.isConfig:
1542 1548
1543 1549 self.setup(id=id,
1544 1550 nplots=nplots,
1545 1551 wintitle=wintitle,
1546 1552 showprofile=showprofile,
1547 1553 show=show)
1548 1554
1549 1555 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1550 1556
1551 1557 if ymin == None: ymin = numpy.nanmin(y)
1552 1558 if ymax == None: ymax = numpy.nanmax(y)
1553 1559
1554 1560 if zmaxZonal == None: zmaxZonal = numpy.nanmax(abs(z[0,:]))
1555 1561 if zminZonal == None: zminZonal = -zmaxZonal
1556 1562 if zmaxVertical == None: zmaxVertical = numpy.nanmax(abs(z[1,:]))
1557 1563 if zminVertical == None: zminVertical = -zmaxVertical
1558 1564
1559 1565 if dataOut.data_SNR is not None:
1560 1566 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
1561 1567 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
1562 1568
1563 1569 self.FTP_WEI = ftp_wei
1564 1570 self.EXP_CODE = exp_code
1565 1571 self.SUB_EXP_CODE = sub_exp_code
1566 1572 self.PLOT_POS = plot_pos
1567 1573
1568 1574 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1569 1575 self.isConfig = True
1570 1576
1571 1577
1572 1578 self.setWinTitle(title)
1573 1579
1574 1580 if ((self.xmax - x[1]) < (x[1]-x[0])):
1575 1581 x[1] = self.xmax
1576 1582
1577 1583 strWind = ['Zonal','Vertical']
1578 1584 strCb = 'Velocity (m/s)'
1579 1585 zmaxVector = [zmaxZonal, zmaxVertical]
1580 1586 zminVector = [zminZonal, zminVertical]
1581 1587
1582 1588 for i in range(nplotsw):
1583 1589
1584 1590 title = "%s Drifts: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1585 1591 axes = self.axesList[i*self.__nsubplots]
1586 1592
1587 1593 z1 = z[i,:].reshape((1,-1))
1588 1594
1589 1595 axes.pcolorbuffer(x, y, z1,
1590 1596 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
1591 1597 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1592 1598 ticksize=9, cblabel=strCb, cbsize="1%", colormap="RdBu_r")
1593 1599
1594 1600 if dataOut.data_SNR is not None:
1595 1601 i += 1
1596 1602 if SNR_1:
1597 1603 title = "Signal Noise Ratio + 1 (SNR+1): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1598 1604 else:
1599 1605 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1600 1606 axes = self.axesList[i*self.__nsubplots]
1601 1607 SNRavgdB = SNRavgdB.reshape((1,-1))
1602 1608
1603 1609 axes.pcolorbuffer(x, y, SNRavgdB,
1604 1610 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1605 1611 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1606 1612 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
1607 1613
1608 1614 self.draw()
1609 1615
1610 1616 if x[1] >= self.axesList[0].xmax:
1611 1617 self.counter_imagwr = wr_period
1612 1618 self.isConfig = False
1613 1619 self.figfile = None
1614 1620
1615 1621
1616 1622
1617 1623
1618 1624 class PhasePlot(Figure):
1619 1625
1620 1626 __isConfig = None
1621 1627 __nsubplots = None
1622 1628
1623 1629 PREFIX = 'mphase'
1624 1630
1625 1631
1626 1632 def __init__(self, **kwargs):
1627 1633 Figure.__init__(self, **kwargs)
1628 1634 self.timerange = 24*60*60
1629 1635 self.isConfig = False
1630 1636 self.__nsubplots = 1
1631 1637 self.counter_imagwr = 0
1632 1638 self.WIDTH = 600
1633 1639 self.HEIGHT = 300
1634 1640 self.WIDTHPROF = 120
1635 1641 self.HEIGHTPROF = 0
1636 1642 self.xdata = None
1637 1643 self.ydata = None
1638 1644
1639 1645 self.PLOT_CODE = MPHASE_CODE
1640 1646
1641 1647 self.FTP_WEI = None
1642 1648 self.EXP_CODE = None
1643 1649 self.SUB_EXP_CODE = None
1644 1650 self.PLOT_POS = None
1645 1651
1646 1652
1647 1653 self.filename_phase = None
1648 1654
1649 1655 self.figfile = None
1650 1656
1651 1657 def getSubplots(self):
1652 1658
1653 1659 ncol = 1
1654 1660 nrow = 1
1655 1661
1656 1662 return nrow, ncol
1657 1663
1658 1664 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1659 1665
1660 1666 self.__showprofile = showprofile
1661 1667 self.nplots = nplots
1662 1668
1663 1669 ncolspan = 7
1664 1670 colspan = 6
1665 1671 self.__nsubplots = 2
1666 1672
1667 1673 self.createFigure(id = id,
1668 1674 wintitle = wintitle,
1669 1675 widthplot = self.WIDTH+self.WIDTHPROF,
1670 1676 heightplot = self.HEIGHT+self.HEIGHTPROF,
1671 1677 show=show)
1672 1678
1673 1679 nrow, ncol = self.getSubplots()
1674 1680
1675 1681 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1676 1682
1677 1683
1678 1684 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1679 1685 xmin=None, xmax=None, ymin=None, ymax=None,
1680 1686 timerange=None,
1681 1687 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1682 1688 server=None, folder=None, username=None, password=None,
1683 1689 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1684 1690
1685 1691
1686 1692 tmin = None
1687 1693 tmax = None
1688 1694 x = dataOut.getTimeRange1(dataOut.outputInterval)
1689 1695 y = dataOut.getHeiRange()
1690 1696
1691 1697
1692 1698 #thisDatetime = dataOut.datatime
1693 1699 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
1694 1700 title = wintitle + " Phase of Beacon Signal" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1695 1701 xlabel = "Local Time"
1696 1702 ylabel = "Phase"
1697 1703
1698 1704
1699 1705 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1700 1706 phase_beacon = dataOut.data_output
1701 1707 update_figfile = False
1702 1708
1703 1709 if not self.isConfig:
1704 1710
1705 1711 self.nplots = phase_beacon.size
1706 1712
1707 1713 self.setup(id=id,
1708 1714 nplots=self.nplots,
1709 1715 wintitle=wintitle,
1710 1716 showprofile=showprofile,
1711 1717 show=show)
1712 1718
1713 1719 if timerange is not None:
1714 1720 self.timerange = timerange
1715 1721
1716 1722 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1717 1723
1718 1724 if ymin == None: ymin = numpy.nanmin(phase_beacon) - 10.0
1719 1725 if ymax == None: ymax = numpy.nanmax(phase_beacon) + 10.0
1720 1726
1721 1727 self.FTP_WEI = ftp_wei
1722 1728 self.EXP_CODE = exp_code
1723 1729 self.SUB_EXP_CODE = sub_exp_code
1724 1730 self.PLOT_POS = plot_pos
1725 1731
1726 1732 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1727 1733 self.isConfig = True
1728 1734 self.figfile = figfile
1729 1735 self.xdata = numpy.array([])
1730 1736 self.ydata = numpy.array([])
1731 1737
1732 1738 #open file beacon phase
1733 1739 path = '%s%03d' %(self.PREFIX, self.id)
1734 1740 beacon_file = os.path.join(path,'%s.txt'%self.name)
1735 1741 self.filename_phase = os.path.join(figpath,beacon_file)
1736 1742 update_figfile = True
1737 1743
1738 1744
1739 1745 #store data beacon phase
1740 1746 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1741 1747
1742 1748 self.setWinTitle(title)
1743 1749
1744 1750
1745 1751 title = "Phase Offset %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1746 1752
1747 1753 legendlabels = ["phase %d"%(chan) for chan in numpy.arange(self.nplots)]
1748 1754
1749 1755 axes = self.axesList[0]
1750 1756
1751 1757 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1752 1758
1753 1759 if len(self.ydata)==0:
1754 1760 self.ydata = phase_beacon.reshape(-1,1)
1755 1761 else:
1756 1762 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1757 1763
1758 1764
1759 1765 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1760 1766 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1761 1767 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1762 1768 XAxisAsTime=True, grid='both'
1763 1769 )
1764 1770
1765 1771 self.draw()
1766 1772
1767 1773 self.save(figpath=figpath,
1768 1774 figfile=figfile,
1769 1775 save=save,
1770 1776 ftp=ftp,
1771 1777 wr_period=wr_period,
1772 1778 thisDatetime=thisDatetime,
1773 1779 update_figfile=update_figfile)
1774 1780
1775 1781 if dataOut.ltctime + dataOut.outputInterval >= self.xmax:
1776 1782 self.counter_imagwr = wr_period
1777 1783 self.isConfig = False
1778 1784 update_figfile = True
1779 1785
1780 1786
1781 1787
1782 1788 class NSMeteorDetection1Plot(Figure):
1783 1789
1784 1790 isConfig = None
1785 1791 __nsubplots = None
1786 1792
1787 1793 WIDTHPROF = None
1788 1794 HEIGHTPROF = None
1789 1795 PREFIX = 'nsm'
1790 1796
1791 1797 zminList = None
1792 1798 zmaxList = None
1793 1799 cmapList = None
1794 1800 titleList = None
1795 1801 nPairs = None
1796 1802 nChannels = None
1797 1803 nParam = None
1798 1804
1799 1805 def __init__(self, **kwargs):
1800 1806 Figure.__init__(self, **kwargs)
1801 1807 self.isConfig = False
1802 1808 self.__nsubplots = 1
1803 1809
1804 1810 self.WIDTH = 750
1805 1811 self.HEIGHT = 250
1806 1812 self.WIDTHPROF = 120
1807 1813 self.HEIGHTPROF = 0
1808 1814 self.counter_imagwr = 0
1809 1815
1810 1816 self.PLOT_CODE = SPEC_CODE
1811 1817
1812 1818 self.FTP_WEI = None
1813 1819 self.EXP_CODE = None
1814 1820 self.SUB_EXP_CODE = None
1815 1821 self.PLOT_POS = None
1816 1822
1817 1823 self.__xfilter_ena = False
1818 1824 self.__yfilter_ena = False
1819 1825
1820 1826 def getSubplots(self):
1821 1827
1822 1828 ncol = 3
1823 1829 nrow = int(numpy.ceil(self.nplots/3.0))
1824 1830
1825 1831 return nrow, ncol
1826 1832
1827 1833 def setup(self, id, nplots, wintitle, show=True):
1828 1834
1829 1835 self.nplots = nplots
1830 1836
1831 1837 ncolspan = 1
1832 1838 colspan = 1
1833 1839
1834 1840 self.createFigure(id = id,
1835 1841 wintitle = wintitle,
1836 1842 widthplot = self.WIDTH + self.WIDTHPROF,
1837 1843 heightplot = self.HEIGHT + self.HEIGHTPROF,
1838 1844 show=show)
1839 1845
1840 1846 nrow, ncol = self.getSubplots()
1841 1847
1842 1848 counter = 0
1843 1849 for y in range(nrow):
1844 1850 for x in range(ncol):
1845 1851
1846 1852 if counter >= self.nplots:
1847 1853 break
1848 1854
1849 1855 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1850 1856
1851 1857 counter += 1
1852 1858
1853 1859 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
1854 1860 xmin=None, xmax=None, ymin=None, ymax=None, SNRmin=None, SNRmax=None,
1855 1861 vmin=None, vmax=None, wmin=None, wmax=None, mode = 'SA',
1856 1862 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1857 1863 server=None, folder=None, username=None, password=None,
1858 1864 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
1859 1865 xaxis="frequency"):
1860 1866
1861 1867 """
1862 1868
1863 1869 Input:
1864 1870 dataOut :
1865 1871 id :
1866 1872 wintitle :
1867 1873 channelList :
1868 1874 showProfile :
1869 1875 xmin : None,
1870 1876 xmax : None,
1871 1877 ymin : None,
1872 1878 ymax : None,
1873 1879 zmin : None,
1874 1880 zmax : None
1875 1881 """
1876 1882 #SEPARAR EN DOS PLOTS
1877 1883 nParam = dataOut.data_param.shape[1] - 3
1878 1884
1879 1885 utctime = dataOut.data_param[0,0]
1880 1886 tmet = dataOut.data_param[:,1].astype(int)
1881 1887 hmet = dataOut.data_param[:,2].astype(int)
1882 1888
1883 1889 x = dataOut.abscissaList
1884 1890 y = dataOut.heightList
1885 1891
1886 1892 z = numpy.zeros((nParam, y.size, x.size - 1))
1887 1893 z[:,:] = numpy.nan
1888 1894 z[:,hmet,tmet] = dataOut.data_param[:,3:].T
1889 1895 z[0,:,:] = 10*numpy.log10(z[0,:,:])
1890 1896
1891 1897 xlabel = "Time (s)"
1892 1898 ylabel = "Range (km)"
1893 1899
1894 1900 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
1895 1901
1896 1902 if not self.isConfig:
1897 1903
1898 1904 nplots = nParam
1899 1905
1900 1906 self.setup(id=id,
1901 1907 nplots=nplots,
1902 1908 wintitle=wintitle,
1903 1909 show=show)
1904 1910
1905 1911 if xmin is None: xmin = numpy.nanmin(x)
1906 1912 if xmax is None: xmax = numpy.nanmax(x)
1907 1913 if ymin is None: ymin = numpy.nanmin(y)
1908 1914 if ymax is None: ymax = numpy.nanmax(y)
1909 1915 if SNRmin is None: SNRmin = numpy.nanmin(z[0,:])
1910 1916 if SNRmax is None: SNRmax = numpy.nanmax(z[0,:])
1911 1917 if vmax is None: vmax = numpy.nanmax(numpy.abs(z[1,:]))
1912 1918 if vmin is None: vmin = -vmax
1913 1919 if wmin is None: wmin = 0
1914 1920 if wmax is None: wmax = 50
1915 1921
1916 1922 pairsList = dataOut.groupList
1917 1923 self.nPairs = len(dataOut.groupList)
1918 1924
1919 1925 zminList = [SNRmin, vmin, cmin] + [pmin]*self.nPairs
1920 1926 zmaxList = [SNRmax, vmax, cmax] + [pmax]*self.nPairs
1921 1927 titleList = ["SNR","Radial Velocity","Coherence"]
1922 1928 cmapList = ["jet","RdBu_r","jet"]
1923 1929
1924 1930 for i in range(self.nPairs):
1925 1931 strAux1 = "Phase Difference "+ str(pairsList[i][0]) + str(pairsList[i][1])
1926 1932 titleList = titleList + [strAux1]
1927 1933 cmapList = cmapList + ["RdBu_r"]
1928 1934
1929 1935 self.zminList = zminList
1930 1936 self.zmaxList = zmaxList
1931 1937 self.cmapList = cmapList
1932 1938 self.titleList = titleList
1933 1939
1934 1940 self.FTP_WEI = ftp_wei
1935 1941 self.EXP_CODE = exp_code
1936 1942 self.SUB_EXP_CODE = sub_exp_code
1937 1943 self.PLOT_POS = plot_pos
1938 1944
1939 1945 self.isConfig = True
1940 1946
1941 1947 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
1942 1948
1943 1949 for i in range(nParam):
1944 1950 title = self.titleList[i] + ": " +str_datetime
1945 1951 axes = self.axesList[i]
1946 1952 axes.pcolor(x, y, z[i,:].T,
1947 1953 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=self.zminList[i], zmax=self.zmaxList[i],
1948 1954 xlabel=xlabel, ylabel=ylabel, title=title, colormap=self.cmapList[i],ticksize=9, cblabel='')
1949 1955 self.draw()
1950 1956
1951 1957 if figfile == None:
1952 1958 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
1953 1959 name = str_datetime
1954 1960 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1955 1961 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
1956 1962 figfile = self.getFilename(name)
1957 1963
1958 1964 self.save(figpath=figpath,
1959 1965 figfile=figfile,
1960 1966 save=save,
1961 1967 ftp=ftp,
1962 1968 wr_period=wr_period,
1963 1969 thisDatetime=thisDatetime)
1964 1970
1965 1971
1966 1972 class NSMeteorDetection2Plot(Figure):
1967 1973
1968 1974 isConfig = None
1969 1975 __nsubplots = None
1970 1976
1971 1977 WIDTHPROF = None
1972 1978 HEIGHTPROF = None
1973 1979 PREFIX = 'nsm'
1974 1980
1975 1981 zminList = None
1976 1982 zmaxList = None
1977 1983 cmapList = None
1978 1984 titleList = None
1979 1985 nPairs = None
1980 1986 nChannels = None
1981 1987 nParam = None
1982 1988
1983 1989 def __init__(self, **kwargs):
1984 1990 Figure.__init__(self, **kwargs)
1985 1991 self.isConfig = False
1986 1992 self.__nsubplots = 1
1987 1993
1988 1994 self.WIDTH = 750
1989 1995 self.HEIGHT = 250
1990 1996 self.WIDTHPROF = 120
1991 1997 self.HEIGHTPROF = 0
1992 1998 self.counter_imagwr = 0
1993 1999
1994 2000 self.PLOT_CODE = SPEC_CODE
1995 2001
1996 2002 self.FTP_WEI = None
1997 2003 self.EXP_CODE = None
1998 2004 self.SUB_EXP_CODE = None
1999 2005 self.PLOT_POS = None
2000 2006
2001 2007 self.__xfilter_ena = False
2002 2008 self.__yfilter_ena = False
2003 2009
2004 2010 def getSubplots(self):
2005 2011
2006 2012 ncol = 3
2007 2013 nrow = int(numpy.ceil(self.nplots/3.0))
2008 2014
2009 2015 return nrow, ncol
2010 2016
2011 2017 def setup(self, id, nplots, wintitle, show=True):
2012 2018
2013 2019 self.nplots = nplots
2014 2020
2015 2021 ncolspan = 1
2016 2022 colspan = 1
2017 2023
2018 2024 self.createFigure(id = id,
2019 2025 wintitle = wintitle,
2020 2026 widthplot = self.WIDTH + self.WIDTHPROF,
2021 2027 heightplot = self.HEIGHT + self.HEIGHTPROF,
2022 2028 show=show)
2023 2029
2024 2030 nrow, ncol = self.getSubplots()
2025 2031
2026 2032 counter = 0
2027 2033 for y in range(nrow):
2028 2034 for x in range(ncol):
2029 2035
2030 2036 if counter >= self.nplots:
2031 2037 break
2032 2038
2033 2039 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
2034 2040
2035 2041 counter += 1
2036 2042
2037 2043 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
2038 2044 xmin=None, xmax=None, ymin=None, ymax=None, SNRmin=None, SNRmax=None,
2039 2045 vmin=None, vmax=None, wmin=None, wmax=None, mode = 'SA',
2040 2046 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
2041 2047 server=None, folder=None, username=None, password=None,
2042 2048 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
2043 2049 xaxis="frequency"):
2044 2050
2045 2051 """
2046 2052
2047 2053 Input:
2048 2054 dataOut :
2049 2055 id :
2050 2056 wintitle :
2051 2057 channelList :
2052 2058 showProfile :
2053 2059 xmin : None,
2054 2060 xmax : None,
2055 2061 ymin : None,
2056 2062 ymax : None,
2057 2063 zmin : None,
2058 2064 zmax : None
2059 2065 """
2060 2066 #Rebuild matrix
2061 2067 utctime = dataOut.data_param[0,0]
2062 2068 cmet = dataOut.data_param[:,1].astype(int)
2063 2069 tmet = dataOut.data_param[:,2].astype(int)
2064 2070 hmet = dataOut.data_param[:,3].astype(int)
2065 2071
2066 2072 nParam = 3
2067 2073 nChan = len(dataOut.groupList)
2068 2074 x = dataOut.abscissaList
2069 2075 y = dataOut.heightList
2070 2076
2071 2077 z = numpy.full((nChan, nParam, y.size, x.size - 1),numpy.nan)
2072 2078 z[cmet,:,hmet,tmet] = dataOut.data_param[:,4:]
2073 2079 z[:,0,:,:] = 10*numpy.log10(z[:,0,:,:]) #logarithmic scale
2074 2080 z = numpy.reshape(z, (nChan*nParam, y.size, x.size-1))
2075 2081
2076 2082 xlabel = "Time (s)"
2077 2083 ylabel = "Range (km)"
2078 2084
2079 2085 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
2080 2086
2081 2087 if not self.isConfig:
2082 2088
2083 2089 nplots = nParam*nChan
2084 2090
2085 2091 self.setup(id=id,
2086 2092 nplots=nplots,
2087 2093 wintitle=wintitle,
2088 2094 show=show)
2089 2095
2090 2096 if xmin is None: xmin = numpy.nanmin(x)
2091 2097 if xmax is None: xmax = numpy.nanmax(x)
2092 2098 if ymin is None: ymin = numpy.nanmin(y)
2093 2099 if ymax is None: ymax = numpy.nanmax(y)
2094 2100 if SNRmin is None: SNRmin = numpy.nanmin(z[0,:])
2095 2101 if SNRmax is None: SNRmax = numpy.nanmax(z[0,:])
2096 2102 if vmax is None: vmax = numpy.nanmax(numpy.abs(z[1,:]))
2097 2103 if vmin is None: vmin = -vmax
2098 2104 if wmin is None: wmin = 0
2099 2105 if wmax is None: wmax = 50
2100 2106
2101 2107 self.nChannels = nChan
2102 2108
2103 2109 zminList = []
2104 2110 zmaxList = []
2105 2111 titleList = []
2106 2112 cmapList = []
2107 2113 for i in range(self.nChannels):
2108 2114 strAux1 = "SNR Channel "+ str(i)
2109 2115 strAux2 = "Radial Velocity Channel "+ str(i)
2110 2116 strAux3 = "Spectral Width Channel "+ str(i)
2111 2117
2112 2118 titleList = titleList + [strAux1,strAux2,strAux3]
2113 2119 cmapList = cmapList + ["jet","RdBu_r","jet"]
2114 2120 zminList = zminList + [SNRmin,vmin,wmin]
2115 2121 zmaxList = zmaxList + [SNRmax,vmax,wmax]
2116 2122
2117 2123 self.zminList = zminList
2118 2124 self.zmaxList = zmaxList
2119 2125 self.cmapList = cmapList
2120 2126 self.titleList = titleList
2121 2127
2122 2128 self.FTP_WEI = ftp_wei
2123 2129 self.EXP_CODE = exp_code
2124 2130 self.SUB_EXP_CODE = sub_exp_code
2125 2131 self.PLOT_POS = plot_pos
2126 2132
2127 2133 self.isConfig = True
2128 2134
2129 2135 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
2130 2136
2131 2137 for i in range(self.nplots):
2132 2138 title = self.titleList[i] + ": " +str_datetime
2133 2139 axes = self.axesList[i]
2134 2140 axes.pcolor(x, y, z[i,:].T,
2135 2141 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=self.zminList[i], zmax=self.zmaxList[i],
2136 2142 xlabel=xlabel, ylabel=ylabel, title=title, colormap=self.cmapList[i],ticksize=9, cblabel='')
2137 2143 self.draw()
2138 2144
2139 2145 if figfile == None:
2140 2146 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
2141 2147 name = str_datetime
2142 2148 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
2143 2149 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
2144 2150 figfile = self.getFilename(name)
2145 2151
2146 2152 self.save(figpath=figpath,
2147 2153 figfile=figfile,
2148 2154 save=save,
2149 2155 ftp=ftp,
2150 2156 wr_period=wr_period,
2151 thisDatetime=thisDatetime) No newline at end of file
2157 thisDatetime=thisDatetime)
2158 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1587 +1,1587
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9
10 10 from .figure import Figure, isRealtime, isTimeInHourRange
11 11 from .plotting_codes import *
12 12 from schainpy.model.proc.jroproc_base import MPDecorator
13 13
14 14 from schainpy.utils import log
15 15
16 16 @MPDecorator
17 17 class SpectraPlot(Figure):
18 18
19 19 isConfig = None
20 20 __nsubplots = None
21 21
22 22 WIDTHPROF = None
23 23 HEIGHTPROF = None
24 24 PREFIX = 'spc'
25 25
26 def __init__(self):#, **kwargs):
27 Figure.__init__(self)#, **kwargs)
26 def __init__(self):
27 Figure.__init__(self)
28 28 self.isConfig = False
29 29 self.__nsubplots = 1
30 30 self.WIDTH = 250
31 31 self.HEIGHT = 250
32 32 self.WIDTHPROF = 120
33 33 self.HEIGHTPROF = 0
34 34 self.counter_imagwr = 0
35 35
36 36 self.PLOT_CODE = SPEC_CODE
37 37
38 38 self.FTP_WEI = None
39 39 self.EXP_CODE = None
40 40 self.SUB_EXP_CODE = None
41 41 self.PLOT_POS = None
42 42
43 43 self.__xfilter_ena = False
44 44 self.__yfilter_ena = False
45 45
46 46 def getSubplots(self):
47 47
48 48 ncol = int(numpy.sqrt(self.nplots)+0.9)
49 49 nrow = int(self.nplots*1./ncol + 0.9)
50 50
51 51 return nrow, ncol
52 52
53 53 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
54 54
55 55 self.__showprofile = showprofile
56 56 self.nplots = nplots
57 57
58 58 ncolspan = 1
59 59 colspan = 1
60 60 if showprofile:
61 61 ncolspan = 3
62 62 colspan = 2
63 63 self.__nsubplots = 2
64 64
65 65 self.createFigure(id = id,
66 66 wintitle = wintitle,
67 67 widthplot = self.WIDTH + self.WIDTHPROF,
68 68 heightplot = self.HEIGHT + self.HEIGHTPROF,
69 69 show=show)
70 70
71 71 nrow, ncol = self.getSubplots()
72 72
73 73 counter = 0
74 74 for y in range(nrow):
75 75 for x in range(ncol):
76 76
77 77 if counter >= self.nplots:
78 78 break
79 79
80 80 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
81 81
82 82 if showprofile:
83 83 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
84 84
85 85 counter += 1
86 86
87 87 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
88 88 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
89 89 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
90 90 server=None, folder=None, username=None, password=None,
91 91 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
92 92 xaxis="frequency", colormap='jet', normFactor=None):
93 93
94 94 """
95 95
96 96 Input:
97 97 dataOut :
98 98 id :
99 99 wintitle :
100 100 channelList :
101 101 showProfile :
102 102 xmin : None,
103 103 xmax : None,
104 104 ymin : None,
105 105 ymax : None,
106 106 zmin : None,
107 107 zmax : None
108 108 """
109 109 if dataOut.flagNoData:
110 110 return dataOut
111 111
112 112 if realtime:
113 113 if not(isRealtime(utcdatatime = dataOut.utctime)):
114 114 print('Skipping this plot function')
115 115 return
116 116
117 117 if channelList == None:
118 118 channelIndexList = dataOut.channelIndexList
119 119 else:
120 120 channelIndexList = []
121 121 for channel in channelList:
122 122 if channel not in dataOut.channelList:
123 123 raise ValueError("Channel %d is not in dataOut.channelList" %channel)
124 124 channelIndexList.append(dataOut.channelList.index(channel))
125 125
126 126 if normFactor is None:
127 127 factor = dataOut.normFactor
128 128 else:
129 129 factor = normFactor
130 130 if xaxis == "frequency":
131 131 x = dataOut.getFreqRange(1)/1000.
132 132 xlabel = "Frequency (kHz)"
133 133
134 134 elif xaxis == "time":
135 135 x = dataOut.getAcfRange(1)
136 136 xlabel = "Time (ms)"
137 137
138 138 else:
139 139 x = dataOut.getVelRange(1)
140 140 xlabel = "Velocity (m/s)"
141 141
142 142 ylabel = "Range (Km)"
143 143
144 144 y = dataOut.getHeiRange()
145 145
146 146 z = dataOut.data_spc/factor
147 147 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
148 148 zdB = 10*numpy.log10(z)
149 149
150 150 avg = numpy.average(z, axis=1)
151 151 avgdB = 10*numpy.log10(avg)
152 152
153 153 noise = dataOut.getNoise()/factor
154 154 noisedB = 10*numpy.log10(noise)
155 155
156 156 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
157 157 title = wintitle + " Spectra"
158 158 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
159 159 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
160 160
161 161 if not self.isConfig:
162 162
163 163 nplots = len(channelIndexList)
164 164
165 165 self.setup(id=id,
166 166 nplots=nplots,
167 167 wintitle=wintitle,
168 168 showprofile=showprofile,
169 169 show=show)
170 170
171 171 if xmin == None: xmin = numpy.nanmin(x)
172 172 if xmax == None: xmax = numpy.nanmax(x)
173 173 if ymin == None: ymin = numpy.nanmin(y)
174 174 if ymax == None: ymax = numpy.nanmax(y)
175 175 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
176 176 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
177 177
178 178 self.FTP_WEI = ftp_wei
179 179 self.EXP_CODE = exp_code
180 180 self.SUB_EXP_CODE = sub_exp_code
181 181 self.PLOT_POS = plot_pos
182 182
183 183 self.isConfig = True
184 184
185 185 self.setWinTitle(title)
186 186
187 187 for i in range(self.nplots):
188 188 index = channelIndexList[i]
189 189 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
190 190 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
191 191 if len(dataOut.beam.codeList) != 0:
192 192 title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
193 193
194 194 axes = self.axesList[i*self.__nsubplots]
195 195 axes.pcolor(x, y, zdB[index,:,:],
196 196 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
197 197 xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
198 198 ticksize=9, cblabel='')
199 199
200 200 if self.__showprofile:
201 201 axes = self.axesList[i*self.__nsubplots +1]
202 202 axes.pline(avgdB[index,:], y,
203 203 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
204 204 xlabel='dB', ylabel='', title='',
205 205 ytick_visible=False,
206 206 grid='x')
207 207
208 208 noiseline = numpy.repeat(noisedB[index], len(y))
209 209 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
210 210
211 211 self.draw()
212 212
213 213 if figfile == None:
214 214 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
215 215 name = str_datetime
216 216 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
217 217 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
218 218 figfile = self.getFilename(name)
219 219
220 220 self.save(figpath=figpath,
221 221 figfile=figfile,
222 222 save=save,
223 223 ftp=ftp,
224 224 wr_period=wr_period,
225 225 thisDatetime=thisDatetime)
226 226
227 227 return dataOut
228 228 @MPDecorator
229 229 class CrossSpectraPlot(Figure):
230 230
231 231 isConfig = None
232 232 __nsubplots = None
233 233
234 234 WIDTH = None
235 235 HEIGHT = None
236 236 WIDTHPROF = None
237 237 HEIGHTPROF = None
238 238 PREFIX = 'cspc'
239 239
240 def __init__(self):#, **kwargs):
241 Figure.__init__(self)#, **kwargs)
240 def __init__(self):
241 Figure.__init__(self)
242 242 self.isConfig = False
243 243 self.__nsubplots = 4
244 244 self.counter_imagwr = 0
245 245 self.WIDTH = 250
246 246 self.HEIGHT = 250
247 247 self.WIDTHPROF = 0
248 248 self.HEIGHTPROF = 0
249 249
250 250 self.PLOT_CODE = CROSS_CODE
251 251 self.FTP_WEI = None
252 252 self.EXP_CODE = None
253 253 self.SUB_EXP_CODE = None
254 254 self.PLOT_POS = None
255 255
256 256 def getSubplots(self):
257 257
258 258 ncol = 4
259 259 nrow = self.nplots
260 260
261 261 return nrow, ncol
262 262
263 263 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
264 264
265 265 self.__showprofile = showprofile
266 266 self.nplots = nplots
267 267
268 268 ncolspan = 1
269 269 colspan = 1
270 270
271 271 self.createFigure(id = id,
272 272 wintitle = wintitle,
273 273 widthplot = self.WIDTH + self.WIDTHPROF,
274 274 heightplot = self.HEIGHT + self.HEIGHTPROF,
275 275 show=True)
276 276
277 277 nrow, ncol = self.getSubplots()
278 278
279 279 counter = 0
280 280 for y in range(nrow):
281 281 for x in range(ncol):
282 282 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
283 283
284 284 counter += 1
285 285
286 286 def run(self, dataOut, id, wintitle="", pairsList=None,
287 287 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
288 288 coh_min=None, coh_max=None, phase_min=None, phase_max=None,
289 289 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
290 290 power_cmap='jet', coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
291 291 server=None, folder=None, username=None, password=None,
292 292 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None,
293 293 xaxis='frequency'):
294 294
295 295 """
296 296
297 297 Input:
298 298 dataOut :
299 299 id :
300 300 wintitle :
301 301 channelList :
302 302 showProfile :
303 303 xmin : None,
304 304 xmax : None,
305 305 ymin : None,
306 306 ymax : None,
307 307 zmin : None,
308 308 zmax : None
309 309 """
310 310
311 311 if dataOut.flagNoData:
312 312 return dataOut
313 313
314 314 if pairsList == None:
315 315 pairsIndexList = dataOut.pairsIndexList
316 316 else:
317 317 pairsIndexList = []
318 318 for pair in pairsList:
319 319 if pair not in dataOut.pairsList:
320 320 raise ValueError("Pair %s is not in dataOut.pairsList" %str(pair))
321 321 pairsIndexList.append(dataOut.pairsList.index(pair))
322 322
323 323 if not pairsIndexList:
324 324 return
325 325
326 326 if len(pairsIndexList) > 4:
327 327 pairsIndexList = pairsIndexList[0:4]
328 328
329 329 if normFactor is None:
330 330 factor = dataOut.normFactor
331 331 else:
332 332 factor = normFactor
333 333 x = dataOut.getVelRange(1)
334 334 y = dataOut.getHeiRange()
335 335 z = dataOut.data_spc[:,:,:]/factor
336 336 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
337 337
338 338 noise = dataOut.noise/factor
339 339
340 340 zdB = 10*numpy.log10(z)
341 341 noisedB = 10*numpy.log10(noise)
342 342
343 343 if coh_min == None:
344 344 coh_min = 0.0
345 345 if coh_max == None:
346 346 coh_max = 1.0
347 347
348 348 if phase_min == None:
349 349 phase_min = -180
350 350 if phase_max == None:
351 351 phase_max = 180
352 352
353 353 #thisDatetime = dataOut.datatime
354 354 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
355 355 title = wintitle + " Cross-Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
356 356 # xlabel = "Velocity (m/s)"
357 357 ylabel = "Range (Km)"
358 358
359 359 if xaxis == "frequency":
360 360 x = dataOut.getFreqRange(1)/1000.
361 361 xlabel = "Frequency (kHz)"
362 362
363 363 elif xaxis == "time":
364 364 x = dataOut.getAcfRange(1)
365 365 xlabel = "Time (ms)"
366 366
367 367 else:
368 368 x = dataOut.getVelRange(1)
369 369 xlabel = "Velocity (m/s)"
370 370
371 371 if not self.isConfig:
372 372
373 373 nplots = len(pairsIndexList)
374 374
375 375 self.setup(id=id,
376 376 nplots=nplots,
377 377 wintitle=wintitle,
378 378 showprofile=False,
379 379 show=show)
380 380
381 381 avg = numpy.abs(numpy.average(z, axis=1))
382 382 avgdB = 10*numpy.log10(avg)
383 383
384 384 if xmin == None: xmin = numpy.nanmin(x)
385 385 if xmax == None: xmax = numpy.nanmax(x)
386 386 if ymin == None: ymin = numpy.nanmin(y)
387 387 if ymax == None: ymax = numpy.nanmax(y)
388 388 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
389 389 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
390 390
391 391 self.FTP_WEI = ftp_wei
392 392 self.EXP_CODE = exp_code
393 393 self.SUB_EXP_CODE = sub_exp_code
394 394 self.PLOT_POS = plot_pos
395 395
396 396 self.isConfig = True
397 397
398 398 self.setWinTitle(title)
399 399
400 400 for i in range(self.nplots):
401 401 pair = dataOut.pairsList[pairsIndexList[i]]
402 402
403 403 chan_index0 = dataOut.channelList.index(pair[0])
404 404 chan_index1 = dataOut.channelList.index(pair[1])
405 405
406 406 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
407 407 title = "Ch%d: %4.2fdB: %s" %(pair[0], noisedB[chan_index0], str_datetime)
408 408 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index0,:,:]/factor)
409 409 axes0 = self.axesList[i*self.__nsubplots]
410 410 axes0.pcolor(x, y, zdB,
411 411 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
412 412 xlabel=xlabel, ylabel=ylabel, title=title,
413 413 ticksize=9, colormap=power_cmap, cblabel='')
414 414
415 415 title = "Ch%d: %4.2fdB: %s" %(pair[1], noisedB[chan_index1], str_datetime)
416 416 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index1,:,:]/factor)
417 417 axes0 = self.axesList[i*self.__nsubplots+1]
418 418 axes0.pcolor(x, y, zdB,
419 419 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
420 420 xlabel=xlabel, ylabel=ylabel, title=title,
421 421 ticksize=9, colormap=power_cmap, cblabel='')
422 422
423 423 coherenceComplex = dataOut.data_cspc[pairsIndexList[i],:,:]/numpy.sqrt(dataOut.data_spc[chan_index0,:,:]*dataOut.data_spc[chan_index1,:,:])
424 424 coherence = numpy.abs(coherenceComplex)
425 425 # phase = numpy.arctan(-1*coherenceComplex.imag/coherenceComplex.real)*180/numpy.pi
426 426 phase = numpy.arctan2(coherenceComplex.imag, coherenceComplex.real)*180/numpy.pi
427 427
428 428 title = "Coherence Ch%d * Ch%d" %(pair[0], pair[1])
429 429 axes0 = self.axesList[i*self.__nsubplots+2]
430 430 axes0.pcolor(x, y, coherence,
431 431 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=coh_min, zmax=coh_max,
432 432 xlabel=xlabel, ylabel=ylabel, title=title,
433 433 ticksize=9, colormap=coherence_cmap, cblabel='')
434 434
435 435 title = "Phase Ch%d * Ch%d" %(pair[0], pair[1])
436 436 axes0 = self.axesList[i*self.__nsubplots+3]
437 437 axes0.pcolor(x, y, phase,
438 438 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
439 439 xlabel=xlabel, ylabel=ylabel, title=title,
440 440 ticksize=9, colormap=phase_cmap, cblabel='')
441 441
442 442
443 443
444 444 self.draw()
445 445
446 446 self.save(figpath=figpath,
447 447 figfile=figfile,
448 448 save=save,
449 449 ftp=ftp,
450 450 wr_period=wr_period,
451 451 thisDatetime=thisDatetime)
452 452
453 453 return dataOut
454 454
455 455 @MPDecorator
456 456 class RTIPlot(Figure):
457 457
458 458 __isConfig = None
459 459 __nsubplots = None
460 460
461 461 WIDTHPROF = None
462 462 HEIGHTPROF = None
463 463 PREFIX = 'rti'
464 464
465 def __init__(self):#, **kwargs):
465 def __init__(self):
466 466
467 Figure.__init__(self)#, **kwargs)
467 Figure.__init__(self)
468 468 self.timerange = None
469 469 self.isConfig = False
470 470 self.__nsubplots = 1
471 471
472 472 self.WIDTH = 800
473 473 self.HEIGHT = 180
474 474 self.WIDTHPROF = 120
475 475 self.HEIGHTPROF = 0
476 476 self.counter_imagwr = 0
477 477
478 478 self.PLOT_CODE = RTI_CODE
479 479
480 480 self.FTP_WEI = None
481 481 self.EXP_CODE = None
482 482 self.SUB_EXP_CODE = None
483 483 self.PLOT_POS = None
484 484 self.tmin = None
485 485 self.tmax = None
486 486
487 487 self.xmin = None
488 488 self.xmax = None
489 489
490 490 self.figfile = None
491 491
492 492 def getSubplots(self):
493 493
494 494 ncol = 1
495 495 nrow = self.nplots
496 496
497 497 return nrow, ncol
498 498
499 499 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
500 500
501 501 self.__showprofile = showprofile
502 502 self.nplots = nplots
503 503
504 504 ncolspan = 1
505 505 colspan = 1
506 506 if showprofile:
507 507 ncolspan = 7
508 508 colspan = 6
509 509 self.__nsubplots = 2
510 510
511 511 self.createFigure(id = id,
512 512 wintitle = wintitle,
513 513 widthplot = self.WIDTH + self.WIDTHPROF,
514 514 heightplot = self.HEIGHT + self.HEIGHTPROF,
515 515 show=show)
516 516
517 517 nrow, ncol = self.getSubplots()
518 518
519 519 counter = 0
520 520 for y in range(nrow):
521 521 for x in range(ncol):
522 522
523 523 if counter >= self.nplots:
524 524 break
525 525
526 526 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
527 527
528 528 if showprofile:
529 529 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
530 530
531 531 counter += 1
532 532
533 533 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
534 534 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
535 535 timerange=None, colormap='jet',
536 536 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
537 537 server=None, folder=None, username=None, password=None,
538 538 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None, HEIGHT=None):
539 539
540 540 """
541 541
542 542 Input:
543 543 dataOut :
544 544 id :
545 545 wintitle :
546 546 channelList :
547 547 showProfile :
548 548 xmin : None,
549 549 xmax : None,
550 550 ymin : None,
551 551 ymax : None,
552 552 zmin : None,
553 553 zmax : None
554 554 """
555 555 if dataOut.flagNoData:
556 556 return dataOut
557 557
558 558 #colormap = kwargs.get('colormap', 'jet')
559 559 if HEIGHT is not None:
560 560 self.HEIGHT = HEIGHT
561 561
562 562 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
563 563 return
564 564
565 565 if channelList == None:
566 566 channelIndexList = dataOut.channelIndexList
567 567 else:
568 568 channelIndexList = []
569 569 for channel in channelList:
570 570 if channel not in dataOut.channelList:
571 571 raise ValueError("Channel %d is not in dataOut.channelList")
572 572 channelIndexList.append(dataOut.channelList.index(channel))
573 573
574 574 if normFactor is None:
575 575 factor = dataOut.normFactor
576 576 else:
577 577 factor = normFactor
578 578
579 # factor = dataOut.normFactor
579 #factor = dataOut.normFactor
580 580 x = dataOut.getTimeRange()
581 581 y = dataOut.getHeiRange()
582 582
583 583 z = dataOut.data_spc/factor
584 584 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
585 585 avg = numpy.average(z, axis=1)
586 586 avgdB = 10.*numpy.log10(avg)
587 587 # avgdB = dataOut.getPower()
588 588
589 589
590 590 thisDatetime = dataOut.datatime
591 # thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
591 #thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
592 592 title = wintitle + " RTI" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
593 593 xlabel = ""
594 594 ylabel = "Range (Km)"
595 595
596 596 update_figfile = False
597 597
598 598 if self.xmax is not None and dataOut.ltctime >= self.xmax: #yong
599 599 self.counter_imagwr = wr_period
600 600 self.isConfig = False
601 601 update_figfile = True
602 602
603 603 if not self.isConfig:
604 604
605 605 nplots = len(channelIndexList)
606 606
607 607 self.setup(id=id,
608 608 nplots=nplots,
609 609 wintitle=wintitle,
610 610 showprofile=showprofile,
611 611 show=show)
612 612
613 613 if timerange != None:
614 614 self.timerange = timerange
615 615
616 616 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
617 617
618 618 noise = dataOut.noise/factor
619 619 noisedB = 10*numpy.log10(noise)
620 620
621 621 if ymin == None: ymin = numpy.nanmin(y)
622 622 if ymax == None: ymax = numpy.nanmax(y)
623 623 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
624 624 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
625 625
626 626 self.FTP_WEI = ftp_wei
627 627 self.EXP_CODE = exp_code
628 628 self.SUB_EXP_CODE = sub_exp_code
629 629 self.PLOT_POS = plot_pos
630 630
631 631 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
632 632 self.isConfig = True
633 633 self.figfile = figfile
634 634 update_figfile = True
635 635
636 636 self.setWinTitle(title)
637 637
638 638 for i in range(self.nplots):
639 639 index = channelIndexList[i]
640 640 title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
641 641 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
642 642 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
643 643 axes = self.axesList[i*self.__nsubplots]
644 644 zdB = avgdB[index].reshape((1,-1))
645 645 axes.pcolorbuffer(x, y, zdB,
646 646 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
647 647 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
648 648 ticksize=9, cblabel='', cbsize="1%", colormap=colormap)
649 649
650 650 if self.__showprofile:
651 651 axes = self.axesList[i*self.__nsubplots +1]
652 652 axes.pline(avgdB[index], y,
653 653 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
654 654 xlabel='dB', ylabel='', title='',
655 655 ytick_visible=False,
656 656 grid='x')
657 657
658 658 self.draw()
659 659
660 660 self.save(figpath=figpath,
661 661 figfile=figfile,
662 662 save=save,
663 663 ftp=ftp,
664 664 wr_period=wr_period,
665 665 thisDatetime=thisDatetime,
666 666 update_figfile=update_figfile)
667 667 return dataOut
668 668
669 669 @MPDecorator
670 670 class CoherenceMap(Figure):
671 671 isConfig = None
672 672 __nsubplots = None
673 673
674 674 WIDTHPROF = None
675 675 HEIGHTPROF = None
676 676 PREFIX = 'cmap'
677 677
678 def __init__(self):#, **kwargs):
679 Figure.__init__(self)#, **kwargs)
678 def __init__(self):
679 Figure.__init__(self)
680 680 self.timerange = 2*60*60
681 681 self.isConfig = False
682 682 self.__nsubplots = 1
683 683
684 684 self.WIDTH = 800
685 685 self.HEIGHT = 180
686 686 self.WIDTHPROF = 120
687 687 self.HEIGHTPROF = 0
688 688 self.counter_imagwr = 0
689 689
690 690 self.PLOT_CODE = COH_CODE
691 691
692 692 self.FTP_WEI = None
693 693 self.EXP_CODE = None
694 694 self.SUB_EXP_CODE = None
695 695 self.PLOT_POS = None
696 696 self.counter_imagwr = 0
697 697
698 698 self.xmin = None
699 699 self.xmax = None
700 700
701 701 def getSubplots(self):
702 702 ncol = 1
703 703 nrow = self.nplots*2
704 704
705 705 return nrow, ncol
706 706
707 707 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
708 708 self.__showprofile = showprofile
709 709 self.nplots = nplots
710 710
711 711 ncolspan = 1
712 712 colspan = 1
713 713 if showprofile:
714 714 ncolspan = 7
715 715 colspan = 6
716 716 self.__nsubplots = 2
717 717
718 718 self.createFigure(id = id,
719 719 wintitle = wintitle,
720 720 widthplot = self.WIDTH + self.WIDTHPROF,
721 721 heightplot = self.HEIGHT + self.HEIGHTPROF,
722 722 show=True)
723 723
724 724 nrow, ncol = self.getSubplots()
725 725
726 726 for y in range(nrow):
727 727 for x in range(ncol):
728 728
729 729 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
730 730
731 731 if showprofile:
732 732 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
733 733
734 734 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
735 735 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
736 736 timerange=None, phase_min=None, phase_max=None,
737 737 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
738 738 coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
739 739 server=None, folder=None, username=None, password=None,
740 740 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
741 741
742 742
743 743 if dataOut.flagNoData:
744 744 return dataOut
745 745
746 746 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
747 747 return
748 748
749 749 if pairsList == None:
750 750 pairsIndexList = dataOut.pairsIndexList
751 751 else:
752 752 pairsIndexList = []
753 753 for pair in pairsList:
754 754 if pair not in dataOut.pairsList:
755 755 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
756 756 pairsIndexList.append(dataOut.pairsList.index(pair))
757 757
758 758 if pairsIndexList == []:
759 759 return
760 760
761 761 if len(pairsIndexList) > 4:
762 762 pairsIndexList = pairsIndexList[0:4]
763 763
764 764 if phase_min == None:
765 765 phase_min = -180
766 766 if phase_max == None:
767 767 phase_max = 180
768 768
769 769 x = dataOut.getTimeRange()
770 770 y = dataOut.getHeiRange()
771 771
772 772 thisDatetime = dataOut.datatime
773 773
774 774 title = wintitle + " CoherenceMap" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
775 775 xlabel = ""
776 776 ylabel = "Range (Km)"
777 777 update_figfile = False
778 778
779 779 if not self.isConfig:
780 780 nplots = len(pairsIndexList)
781 781 self.setup(id=id,
782 782 nplots=nplots,
783 783 wintitle=wintitle,
784 784 showprofile=showprofile,
785 785 show=show)
786 786
787 787 if timerange != None:
788 788 self.timerange = timerange
789 789
790 790 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
791 791
792 792 if ymin == None: ymin = numpy.nanmin(y)
793 793 if ymax == None: ymax = numpy.nanmax(y)
794 794 if zmin == None: zmin = 0.
795 795 if zmax == None: zmax = 1.
796 796
797 797 self.FTP_WEI = ftp_wei
798 798 self.EXP_CODE = exp_code
799 799 self.SUB_EXP_CODE = sub_exp_code
800 800 self.PLOT_POS = plot_pos
801 801
802 802 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
803 803
804 804 self.isConfig = True
805 805 update_figfile = True
806 806
807 807 self.setWinTitle(title)
808 808
809 809 for i in range(self.nplots):
810 810
811 811 pair = dataOut.pairsList[pairsIndexList[i]]
812 812
813 813 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i],:,:],axis=0)
814 814 powa = numpy.average(dataOut.data_spc[pair[0],:,:],axis=0)
815 815 powb = numpy.average(dataOut.data_spc[pair[1],:,:],axis=0)
816 816
817 817
818 818 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
819 819 coherence = numpy.abs(avgcoherenceComplex)
820 820
821 821 z = coherence.reshape((1,-1))
822 822
823 823 counter = 0
824 824
825 825 title = "Coherence Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
826 826 axes = self.axesList[i*self.__nsubplots*2]
827 827 axes.pcolorbuffer(x, y, z,
828 828 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
829 829 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
830 830 ticksize=9, cblabel='', colormap=coherence_cmap, cbsize="1%")
831 831
832 832 if self.__showprofile:
833 833 counter += 1
834 834 axes = self.axesList[i*self.__nsubplots*2 + counter]
835 835 axes.pline(coherence, y,
836 836 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
837 837 xlabel='', ylabel='', title='', ticksize=7,
838 838 ytick_visible=False, nxticks=5,
839 839 grid='x')
840 840
841 841 counter += 1
842 842
843 843 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
844 844
845 845 z = phase.reshape((1,-1))
846 846
847 847 title = "Phase Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
848 848 axes = self.axesList[i*self.__nsubplots*2 + counter]
849 849 axes.pcolorbuffer(x, y, z,
850 850 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
851 851 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
852 852 ticksize=9, cblabel='', colormap=phase_cmap, cbsize="1%")
853 853
854 854 if self.__showprofile:
855 855 counter += 1
856 856 axes = self.axesList[i*self.__nsubplots*2 + counter]
857 857 axes.pline(phase, y,
858 858 xmin=phase_min, xmax=phase_max, ymin=ymin, ymax=ymax,
859 859 xlabel='', ylabel='', title='', ticksize=7,
860 860 ytick_visible=False, nxticks=4,
861 861 grid='x')
862 862
863 863 self.draw()
864 864
865 865 if dataOut.ltctime >= self.xmax:
866 866 self.counter_imagwr = wr_period
867 867 self.isConfig = False
868 868 update_figfile = True
869 869
870 870 self.save(figpath=figpath,
871 871 figfile=figfile,
872 872 save=save,
873 873 ftp=ftp,
874 874 wr_period=wr_period,
875 875 thisDatetime=thisDatetime,
876 876 update_figfile=update_figfile)
877 877
878 878 return dataOut
879 879
880 880 @MPDecorator
881 881 class PowerProfilePlot(Figure):
882 882
883 883 isConfig = None
884 884 __nsubplots = None
885 885
886 886 WIDTHPROF = None
887 887 HEIGHTPROF = None
888 888 PREFIX = 'spcprofile'
889 889
890 def __init__(self):#, **kwargs):
891 Figure.__init__(self)#, **kwargs)
890 def __init__(self):
891 Figure.__init__(self)
892 892 self.isConfig = False
893 893 self.__nsubplots = 1
894 894
895 895 self.PLOT_CODE = POWER_CODE
896 896
897 897 self.WIDTH = 300
898 898 self.HEIGHT = 500
899 899 self.counter_imagwr = 0
900 900
901 901 def getSubplots(self):
902 902 ncol = 1
903 903 nrow = 1
904 904
905 905 return nrow, ncol
906 906
907 907 def setup(self, id, nplots, wintitle, show):
908 908
909 909 self.nplots = nplots
910 910
911 911 ncolspan = 1
912 912 colspan = 1
913 913
914 914 self.createFigure(id = id,
915 915 wintitle = wintitle,
916 916 widthplot = self.WIDTH,
917 917 heightplot = self.HEIGHT,
918 918 show=show)
919 919
920 920 nrow, ncol = self.getSubplots()
921 921
922 922 counter = 0
923 923 for y in range(nrow):
924 924 for x in range(ncol):
925 925 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
926 926
927 927 def run(self, dataOut, id, wintitle="", channelList=None,
928 928 xmin=None, xmax=None, ymin=None, ymax=None,
929 929 save=False, figpath='./', figfile=None, show=True,
930 930 ftp=False, wr_period=1, server=None,
931 931 folder=None, username=None, password=None):
932 932
933 933 if dataOut.flagNoData:
934 934 return dataOut
935 935
936 936
937 937 if channelList == None:
938 938 channelIndexList = dataOut.channelIndexList
939 939 channelList = dataOut.channelList
940 940 else:
941 941 channelIndexList = []
942 942 for channel in channelList:
943 943 if channel not in dataOut.channelList:
944 944 raise ValueError("Channel %d is not in dataOut.channelList")
945 945 channelIndexList.append(dataOut.channelList.index(channel))
946 946
947 947 factor = dataOut.normFactor
948 948
949 949 y = dataOut.getHeiRange()
950 950
951 951 #for voltage
952 952 if dataOut.type == 'Voltage':
953 953 x = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
954 954 x = x.real
955 955 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
956 956
957 957 #for spectra
958 958 if dataOut.type == 'Spectra':
959 959 x = dataOut.data_spc[channelIndexList,:,:]/factor
960 960 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
961 961 x = numpy.average(x, axis=1)
962 962
963 963
964 964 xdB = 10*numpy.log10(x)
965 965
966 966 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
967 967 title = wintitle + " Power Profile %s" %(thisDatetime.strftime("%d-%b-%Y"))
968 968 xlabel = "dB"
969 969 ylabel = "Range (Km)"
970 970
971 971 if not self.isConfig:
972 972
973 973 nplots = 1
974 974
975 975 self.setup(id=id,
976 976 nplots=nplots,
977 977 wintitle=wintitle,
978 978 show=show)
979 979
980 980 if ymin == None: ymin = numpy.nanmin(y)
981 981 if ymax == None: ymax = numpy.nanmax(y)
982 982 if xmin == None: xmin = numpy.nanmin(xdB)*0.9
983 983 if xmax == None: xmax = numpy.nanmax(xdB)*1.1
984 984
985 985 self.isConfig = True
986 986
987 987 self.setWinTitle(title)
988 988
989 989 title = "Power Profile: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
990 990 axes = self.axesList[0]
991 991
992 992 legendlabels = ["channel %d"%x for x in channelList]
993 993 axes.pmultiline(xdB, y,
994 994 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
995 995 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
996 996 ytick_visible=True, nxticks=5,
997 997 grid='x')
998 998
999 999 self.draw()
1000 1000
1001 1001 self.save(figpath=figpath,
1002 1002 figfile=figfile,
1003 1003 save=save,
1004 1004 ftp=ftp,
1005 1005 wr_period=wr_period,
1006 1006 thisDatetime=thisDatetime)
1007 1007
1008 1008 return dataOut
1009 1009
1010 1010 @MPDecorator
1011 1011 class SpectraCutPlot(Figure):
1012 1012
1013 1013 isConfig = None
1014 1014 __nsubplots = None
1015 1015
1016 1016 WIDTHPROF = None
1017 1017 HEIGHTPROF = None
1018 1018 PREFIX = 'spc_cut'
1019 1019
1020 def __init__(self):#, **kwargs):
1021 Figure.__init__(self)#, **kwargs)
1020 def __init__(self):
1021 Figure.__init__(self)
1022 1022 self.isConfig = False
1023 1023 self.__nsubplots = 1
1024 1024
1025 1025 self.PLOT_CODE = POWER_CODE
1026 1026
1027 1027 self.WIDTH = 700
1028 1028 self.HEIGHT = 500
1029 1029 self.counter_imagwr = 0
1030 1030
1031 1031 def getSubplots(self):
1032 1032 ncol = 1
1033 1033 nrow = 1
1034 1034
1035 1035 return nrow, ncol
1036 1036
1037 1037 def setup(self, id, nplots, wintitle, show):
1038 1038
1039 1039 self.nplots = nplots
1040 1040
1041 1041 ncolspan = 1
1042 1042 colspan = 1
1043 1043
1044 1044 self.createFigure(id = id,
1045 1045 wintitle = wintitle,
1046 1046 widthplot = self.WIDTH,
1047 1047 heightplot = self.HEIGHT,
1048 1048 show=show)
1049 1049
1050 1050 nrow, ncol = self.getSubplots()
1051 1051
1052 1052 counter = 0
1053 1053 for y in range(nrow):
1054 1054 for x in range(ncol):
1055 1055 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1056 1056
1057 1057 def run(self, dataOut, id, wintitle="", channelList=None,
1058 1058 xmin=None, xmax=None, ymin=None, ymax=None,
1059 1059 save=False, figpath='./', figfile=None, show=True,
1060 1060 ftp=False, wr_period=1, server=None,
1061 1061 folder=None, username=None, password=None,
1062 1062 xaxis="frequency"):
1063 1063
1064 1064 if dataOut.flagNoData:
1065 1065 return dataOut
1066 1066
1067 1067 if channelList == None:
1068 1068 channelIndexList = dataOut.channelIndexList
1069 1069 channelList = dataOut.channelList
1070 1070 else:
1071 1071 channelIndexList = []
1072 1072 for channel in channelList:
1073 1073 if channel not in dataOut.channelList:
1074 1074 raise ValueError("Channel %d is not in dataOut.channelList")
1075 1075 channelIndexList.append(dataOut.channelList.index(channel))
1076 1076
1077 1077 factor = dataOut.normFactor
1078 1078
1079 1079 y = dataOut.getHeiRange()
1080 1080
1081 1081 z = dataOut.data_spc/factor
1082 1082 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1083 1083
1084 1084 hei_index = numpy.arange(25)*3 + 20
1085 1085
1086 1086 if xaxis == "frequency":
1087 1087 x = dataOut.getFreqRange()/1000.
1088 1088 zdB = 10*numpy.log10(z[0,:,hei_index])
1089 1089 xlabel = "Frequency (kHz)"
1090 1090 ylabel = "Power (dB)"
1091 1091
1092 1092 elif xaxis == "time":
1093 1093 x = dataOut.getAcfRange()
1094 1094 zdB = z[0,:,hei_index]
1095 1095 xlabel = "Time (ms)"
1096 1096 ylabel = "ACF"
1097 1097
1098 1098 else:
1099 1099 x = dataOut.getVelRange()
1100 1100 zdB = 10*numpy.log10(z[0,:,hei_index])
1101 1101 xlabel = "Velocity (m/s)"
1102 1102 ylabel = "Power (dB)"
1103 1103
1104 1104 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1105 1105 title = wintitle + " Range Cuts %s" %(thisDatetime.strftime("%d-%b-%Y"))
1106 1106
1107 1107 if not self.isConfig:
1108 1108
1109 1109 nplots = 1
1110 1110
1111 1111 self.setup(id=id,
1112 1112 nplots=nplots,
1113 1113 wintitle=wintitle,
1114 1114 show=show)
1115 1115
1116 1116 if xmin == None: xmin = numpy.nanmin(x)*0.9
1117 1117 if xmax == None: xmax = numpy.nanmax(x)*1.1
1118 1118 if ymin == None: ymin = numpy.nanmin(zdB)
1119 1119 if ymax == None: ymax = numpy.nanmax(zdB)
1120 1120
1121 1121 self.isConfig = True
1122 1122
1123 1123 self.setWinTitle(title)
1124 1124
1125 1125 title = "Spectra Cuts: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1126 1126 axes = self.axesList[0]
1127 1127
1128 1128 legendlabels = ["Range = %dKm" %y[i] for i in hei_index]
1129 1129
1130 1130 axes.pmultilineyaxis( x, zdB,
1131 1131 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1132 1132 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
1133 1133 ytick_visible=True, nxticks=5,
1134 1134 grid='x')
1135 1135
1136 1136 self.draw()
1137 1137
1138 1138 self.save(figpath=figpath,
1139 1139 figfile=figfile,
1140 1140 save=save,
1141 1141 ftp=ftp,
1142 1142 wr_period=wr_period,
1143 1143 thisDatetime=thisDatetime)
1144 1144
1145 1145 return dataOut
1146 1146
1147 1147 @MPDecorator
1148 1148 class Noise(Figure):
1149 1149
1150 1150 isConfig = None
1151 1151 __nsubplots = None
1152 1152
1153 1153 PREFIX = 'noise'
1154 1154
1155 1155
1156 def __init__(self):#, **kwargs):
1157 Figure.__init__(self)#, **kwargs)
1156 def __init__(self):
1157 Figure.__init__(self)
1158 1158 self.timerange = 24*60*60
1159 1159 self.isConfig = False
1160 1160 self.__nsubplots = 1
1161 1161 self.counter_imagwr = 0
1162 1162 self.WIDTH = 800
1163 1163 self.HEIGHT = 400
1164 1164 self.WIDTHPROF = 120
1165 1165 self.HEIGHTPROF = 0
1166 1166 self.xdata = None
1167 1167 self.ydata = None
1168 1168
1169 1169 self.PLOT_CODE = NOISE_CODE
1170 1170
1171 1171 self.FTP_WEI = None
1172 1172 self.EXP_CODE = None
1173 1173 self.SUB_EXP_CODE = None
1174 1174 self.PLOT_POS = None
1175 1175 self.figfile = None
1176 1176
1177 1177 self.xmin = None
1178 1178 self.xmax = None
1179 1179
1180 1180 def getSubplots(self):
1181 1181
1182 1182 ncol = 1
1183 1183 nrow = 1
1184 1184
1185 1185 return nrow, ncol
1186 1186
1187 1187 def openfile(self, filename):
1188 1188 dirname = os.path.dirname(filename)
1189 1189
1190 1190 if not os.path.exists(dirname):
1191 1191 os.mkdir(dirname)
1192 1192
1193 1193 f = open(filename,'w+')
1194 1194 f.write('\n\n')
1195 1195 f.write('JICAMARCA RADIO OBSERVATORY - Noise \n')
1196 1196 f.write('DD MM YYYY HH MM SS Channel0 Channel1 Channel2 Channel3\n\n' )
1197 1197 f.close()
1198 1198
1199 1199 def save_data(self, filename_phase, data, data_datetime):
1200 1200
1201 1201 f=open(filename_phase,'a')
1202 1202
1203 1203 timetuple_data = data_datetime.timetuple()
1204 1204 day = str(timetuple_data.tm_mday)
1205 1205 month = str(timetuple_data.tm_mon)
1206 1206 year = str(timetuple_data.tm_year)
1207 1207 hour = str(timetuple_data.tm_hour)
1208 1208 minute = str(timetuple_data.tm_min)
1209 1209 second = str(timetuple_data.tm_sec)
1210 1210
1211 1211 data_msg = ''
1212 1212 for i in range(len(data)):
1213 1213 data_msg += str(data[i]) + ' '
1214 1214
1215 1215 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' ' + data_msg + '\n')
1216 1216 f.close()
1217 1217
1218 1218
1219 1219 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1220 1220
1221 1221 self.__showprofile = showprofile
1222 1222 self.nplots = nplots
1223 1223
1224 1224 ncolspan = 7
1225 1225 colspan = 6
1226 1226 self.__nsubplots = 2
1227 1227
1228 1228 self.createFigure(id = id,
1229 1229 wintitle = wintitle,
1230 1230 widthplot = self.WIDTH+self.WIDTHPROF,
1231 1231 heightplot = self.HEIGHT+self.HEIGHTPROF,
1232 1232 show=show)
1233 1233
1234 1234 nrow, ncol = self.getSubplots()
1235 1235
1236 1236 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1237 1237
1238 1238
1239 1239 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
1240 1240 xmin=None, xmax=None, ymin=None, ymax=None,
1241 1241 timerange=None,
1242 1242 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1243 1243 server=None, folder=None, username=None, password=None,
1244 1244 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1245 1245
1246 1246 if dataOut.flagNoData:
1247 1247 return dataOut
1248 1248
1249 1249 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1250 1250 return
1251 1251
1252 1252 if channelList == None:
1253 1253 channelIndexList = dataOut.channelIndexList
1254 1254 channelList = dataOut.channelList
1255 1255 else:
1256 1256 channelIndexList = []
1257 1257 for channel in channelList:
1258 1258 if channel not in dataOut.channelList:
1259 1259 raise ValueError("Channel %d is not in dataOut.channelList")
1260 1260 channelIndexList.append(dataOut.channelList.index(channel))
1261 1261
1262 1262 x = dataOut.getTimeRange()
1263 1263 #y = dataOut.getHeiRange()
1264 1264 factor = dataOut.normFactor
1265 1265 noise = dataOut.noise[channelIndexList]/factor
1266 1266 noisedB = 10*numpy.log10(noise)
1267 1267
1268 1268 thisDatetime = dataOut.datatime
1269 1269
1270 1270 title = wintitle + " Noise" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1271 1271 xlabel = ""
1272 1272 ylabel = "Intensity (dB)"
1273 1273 update_figfile = False
1274 1274
1275 1275 if not self.isConfig:
1276 1276
1277 1277 nplots = 1
1278 1278
1279 1279 self.setup(id=id,
1280 1280 nplots=nplots,
1281 1281 wintitle=wintitle,
1282 1282 showprofile=showprofile,
1283 1283 show=show)
1284 1284
1285 1285 if timerange != None:
1286 1286 self.timerange = timerange
1287 1287
1288 1288 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1289 1289
1290 1290 if ymin == None: ymin = numpy.floor(numpy.nanmin(noisedB)) - 10.0
1291 1291 if ymax == None: ymax = numpy.nanmax(noisedB) + 10.0
1292 1292
1293 1293 self.FTP_WEI = ftp_wei
1294 1294 self.EXP_CODE = exp_code
1295 1295 self.SUB_EXP_CODE = sub_exp_code
1296 1296 self.PLOT_POS = plot_pos
1297 1297
1298 1298
1299 1299 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1300 1300 self.isConfig = True
1301 1301 self.figfile = figfile
1302 1302 self.xdata = numpy.array([])
1303 1303 self.ydata = numpy.array([])
1304 1304
1305 1305 update_figfile = True
1306 1306
1307 1307 #open file beacon phase
1308 1308 path = '%s%03d' %(self.PREFIX, self.id)
1309 1309 noise_file = os.path.join(path,'%s.txt'%self.name)
1310 1310 self.filename_noise = os.path.join(figpath,noise_file)
1311 1311
1312 1312 self.setWinTitle(title)
1313 1313
1314 1314 title = "Noise %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1315 1315
1316 1316 legendlabels = ["channel %d"%(idchannel) for idchannel in channelList]
1317 1317 axes = self.axesList[0]
1318 1318
1319 1319 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1320 1320
1321 1321 if len(self.ydata)==0:
1322 1322 self.ydata = noisedB.reshape(-1,1)
1323 1323 else:
1324 1324 self.ydata = numpy.hstack((self.ydata, noisedB.reshape(-1,1)))
1325 1325
1326 1326
1327 1327 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1328 1328 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1329 1329 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1330 1330 XAxisAsTime=True, grid='both'
1331 1331 )
1332 1332
1333 1333 self.draw()
1334 1334
1335 1335 if dataOut.ltctime >= self.xmax:
1336 1336 self.counter_imagwr = wr_period
1337 1337 self.isConfig = False
1338 1338 update_figfile = True
1339 1339
1340 1340 self.save(figpath=figpath,
1341 1341 figfile=figfile,
1342 1342 save=save,
1343 1343 ftp=ftp,
1344 1344 wr_period=wr_period,
1345 1345 thisDatetime=thisDatetime,
1346 1346 update_figfile=update_figfile)
1347 1347
1348 1348 #store data beacon phase
1349 1349 if save:
1350 1350 self.save_data(self.filename_noise, noisedB, thisDatetime)
1351 1351
1352 1352 return dataOut
1353 1353
1354 1354 @MPDecorator
1355 1355 class BeaconPhase(Figure):
1356 1356
1357 1357 __isConfig = None
1358 1358 __nsubplots = None
1359 1359
1360 1360 PREFIX = 'beacon_phase'
1361 1361
1362 def __init__(self):#, **kwargs):
1363 Figure.__init__(self)#, **kwargs)
1362 def __init__(self):
1363 Figure.__init__(self)
1364 1364 self.timerange = 24*60*60
1365 1365 self.isConfig = False
1366 1366 self.__nsubplots = 1
1367 1367 self.counter_imagwr = 0
1368 1368 self.WIDTH = 800
1369 1369 self.HEIGHT = 400
1370 1370 self.WIDTHPROF = 120
1371 1371 self.HEIGHTPROF = 0
1372 1372 self.xdata = None
1373 1373 self.ydata = None
1374 1374
1375 1375 self.PLOT_CODE = BEACON_CODE
1376 1376
1377 1377 self.FTP_WEI = None
1378 1378 self.EXP_CODE = None
1379 1379 self.SUB_EXP_CODE = None
1380 1380 self.PLOT_POS = None
1381 1381
1382 1382 self.filename_phase = None
1383 1383
1384 1384 self.figfile = None
1385 1385
1386 1386 self.xmin = None
1387 1387 self.xmax = None
1388 1388
1389 1389 def getSubplots(self):
1390 1390
1391 1391 ncol = 1
1392 1392 nrow = 1
1393 1393
1394 1394 return nrow, ncol
1395 1395
1396 1396 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1397 1397
1398 1398 self.__showprofile = showprofile
1399 1399 self.nplots = nplots
1400 1400
1401 1401 ncolspan = 7
1402 1402 colspan = 6
1403 1403 self.__nsubplots = 2
1404 1404
1405 1405 self.createFigure(id = id,
1406 1406 wintitle = wintitle,
1407 1407 widthplot = self.WIDTH+self.WIDTHPROF,
1408 1408 heightplot = self.HEIGHT+self.HEIGHTPROF,
1409 1409 show=show)
1410 1410
1411 1411 nrow, ncol = self.getSubplots()
1412 1412
1413 1413 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1414 1414
1415 1415 def save_phase(self, filename_phase):
1416 1416 f = open(filename_phase,'w+')
1417 1417 f.write('\n\n')
1418 1418 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
1419 1419 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
1420 1420 f.close()
1421 1421
1422 1422 def save_data(self, filename_phase, data, data_datetime):
1423 1423 f=open(filename_phase,'a')
1424 1424 timetuple_data = data_datetime.timetuple()
1425 1425 day = str(timetuple_data.tm_mday)
1426 1426 month = str(timetuple_data.tm_mon)
1427 1427 year = str(timetuple_data.tm_year)
1428 1428 hour = str(timetuple_data.tm_hour)
1429 1429 minute = str(timetuple_data.tm_min)
1430 1430 second = str(timetuple_data.tm_sec)
1431 1431 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
1432 1432 f.close()
1433 1433
1434 1434
1435 1435 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1436 1436 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
1437 1437 timerange=None,
1438 1438 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1439 1439 server=None, folder=None, username=None, password=None,
1440 1440 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1441 1441
1442 1442 if dataOut.flagNoData:
1443 1443 return dataOut
1444 1444
1445 1445 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1446 1446 return
1447 1447
1448 1448 if pairsList == None:
1449 1449 pairsIndexList = dataOut.pairsIndexList[:10]
1450 1450 else:
1451 1451 pairsIndexList = []
1452 1452 for pair in pairsList:
1453 1453 if pair not in dataOut.pairsList:
1454 1454 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
1455 1455 pairsIndexList.append(dataOut.pairsList.index(pair))
1456 1456
1457 1457 if pairsIndexList == []:
1458 1458 return
1459 1459
1460 1460 # if len(pairsIndexList) > 4:
1461 1461 # pairsIndexList = pairsIndexList[0:4]
1462 1462
1463 1463 hmin_index = None
1464 1464 hmax_index = None
1465 1465
1466 1466 if hmin != None and hmax != None:
1467 1467 indexes = numpy.arange(dataOut.nHeights)
1468 1468 hmin_list = indexes[dataOut.heightList >= hmin]
1469 1469 hmax_list = indexes[dataOut.heightList <= hmax]
1470 1470
1471 1471 if hmin_list.any():
1472 1472 hmin_index = hmin_list[0]
1473 1473
1474 1474 if hmax_list.any():
1475 1475 hmax_index = hmax_list[-1]+1
1476 1476
1477 1477 x = dataOut.getTimeRange()
1478 1478 #y = dataOut.getHeiRange()
1479 1479
1480 1480
1481 1481 thisDatetime = dataOut.datatime
1482 1482
1483 1483 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1484 1484 xlabel = "Local Time"
1485 1485 ylabel = "Phase (degrees)"
1486 1486
1487 1487 update_figfile = False
1488 1488
1489 1489 nplots = len(pairsIndexList)
1490 1490 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1491 1491 phase_beacon = numpy.zeros(len(pairsIndexList))
1492 1492 for i in range(nplots):
1493 1493 pair = dataOut.pairsList[pairsIndexList[i]]
1494 1494 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
1495 1495 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
1496 1496 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
1497 1497 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
1498 1498 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
1499 1499
1500 1500 #print "Phase %d%d" %(pair[0], pair[1])
1501 1501 #print phase[dataOut.beacon_heiIndexList]
1502 1502
1503 1503 if dataOut.beacon_heiIndexList:
1504 1504 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
1505 1505 else:
1506 1506 phase_beacon[i] = numpy.average(phase)
1507 1507
1508 1508 if not self.isConfig:
1509 1509
1510 1510 nplots = len(pairsIndexList)
1511 1511
1512 1512 self.setup(id=id,
1513 1513 nplots=nplots,
1514 1514 wintitle=wintitle,
1515 1515 showprofile=showprofile,
1516 1516 show=show)
1517 1517
1518 1518 if timerange != None:
1519 1519 self.timerange = timerange
1520 1520
1521 1521 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1522 1522
1523 1523 if ymin == None: ymin = 0
1524 1524 if ymax == None: ymax = 360
1525 1525
1526 1526 self.FTP_WEI = ftp_wei
1527 1527 self.EXP_CODE = exp_code
1528 1528 self.SUB_EXP_CODE = sub_exp_code
1529 1529 self.PLOT_POS = plot_pos
1530 1530
1531 1531 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1532 1532 self.isConfig = True
1533 1533 self.figfile = figfile
1534 1534 self.xdata = numpy.array([])
1535 1535 self.ydata = numpy.array([])
1536 1536
1537 1537 update_figfile = True
1538 1538
1539 1539 #open file beacon phase
1540 1540 path = '%s%03d' %(self.PREFIX, self.id)
1541 1541 beacon_file = os.path.join(path,'%s.txt'%self.name)
1542 1542 self.filename_phase = os.path.join(figpath,beacon_file)
1543 1543 #self.save_phase(self.filename_phase)
1544 1544
1545 1545
1546 1546 #store data beacon phase
1547 1547 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1548 1548
1549 1549 self.setWinTitle(title)
1550 1550
1551 1551
1552 1552 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1553 1553
1554 1554 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
1555 1555
1556 1556 axes = self.axesList[0]
1557 1557
1558 1558 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1559 1559
1560 1560 if len(self.ydata)==0:
1561 1561 self.ydata = phase_beacon.reshape(-1,1)
1562 1562 else:
1563 1563 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1564 1564
1565 1565
1566 1566 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1567 1567 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1568 1568 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1569 1569 XAxisAsTime=True, grid='both'
1570 1570 )
1571 1571
1572 1572 self.draw()
1573 1573
1574 1574 if dataOut.ltctime >= self.xmax:
1575 1575 self.counter_imagwr = wr_period
1576 1576 self.isConfig = False
1577 1577 update_figfile = True
1578 1578
1579 1579 self.save(figpath=figpath,
1580 1580 figfile=figfile,
1581 1581 save=save,
1582 1582 ftp=ftp,
1583 1583 wr_period=wr_period,
1584 1584 thisDatetime=thisDatetime,
1585 1585 update_figfile=update_figfile)
1586 1586
1587 return dataOut #Yong No newline at end of file
1587 return dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,232 +1,232
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator #YONG
10 10 from schainpy.utils import log
11 11 from .figure import Figure
12 12
13 13
14 14 @MPDecorator
15 15 class Scope(Figure):
16 16
17 17 isConfig = None
18 18
19 19 def __init__(self):#, **kwargs): #YONG
20 20 Figure.__init__(self)#, **kwargs)
21 21 self.isConfig = False
22 22 self.WIDTH = 300
23 23 self.HEIGHT = 200
24 24 self.counter_imagwr = 0
25 25
26 26 def getSubplots(self):
27 27
28 28 nrow = self.nplots
29 29 ncol = 3
30 30 return nrow, ncol
31 31
32 32 def setup(self, id, nplots, wintitle, show):
33 33
34 34 self.nplots = nplots
35 35
36 36 self.createFigure(id=id,
37 37 wintitle=wintitle,
38 38 show=show)
39 39
40 40 nrow,ncol = self.getSubplots()
41 41 colspan = 3
42 42 rowspan = 1
43 43
44 44 for i in range(nplots):
45 45 self.addAxes(nrow, ncol, i, 0, colspan, rowspan)
46 46
47 47 def plot_iq(self, x, y, id, channelIndexList, thisDatetime, wintitle, show, xmin, xmax, ymin, ymax):
48 48 yreal = y[channelIndexList,:].real
49 49 yimag = y[channelIndexList,:].imag
50 50
51 51 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
52 52 xlabel = "Range (Km)"
53 53 ylabel = "Intensity - IQ"
54 54
55 55 if not self.isConfig:
56 56 nplots = len(channelIndexList)
57 57
58 58 self.setup(id=id,
59 59 nplots=nplots,
60 60 wintitle='',
61 61 show=show)
62 62
63 63 if xmin == None: xmin = numpy.nanmin(x)
64 64 if xmax == None: xmax = numpy.nanmax(x)
65 65 if ymin == None: ymin = min(numpy.nanmin(yreal),numpy.nanmin(yimag))
66 66 if ymax == None: ymax = max(numpy.nanmax(yreal),numpy.nanmax(yimag))
67 67
68 68 self.isConfig = True
69 69
70 70 self.setWinTitle(title)
71 71
72 72 for i in range(len(self.axesList)):
73 73 title = "Channel %d" %(i)
74 74 axes = self.axesList[i]
75 75
76 76 axes.pline(x, yreal[i,:],
77 77 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
78 78 xlabel=xlabel, ylabel=ylabel, title=title)
79 79
80 80 axes.addpline(x, yimag[i,:], idline=1, color="red", linestyle="solid", lw=2)
81 81
82 82 def plot_power(self, x, y, id, channelIndexList, thisDatetime, wintitle, show, xmin, xmax, ymin, ymax):
83 83 y = y[channelIndexList,:] * numpy.conjugate(y[channelIndexList,:])
84 84 yreal = y.real
85 85
86 86 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
87 87 xlabel = "Range (Km)"
88 88 ylabel = "Intensity"
89 89
90 90 if not self.isConfig:
91 91 nplots = len(channelIndexList)
92 92
93 93 self.setup(id=id,
94 94 nplots=nplots,
95 95 wintitle='',
96 96 show=show)
97 97
98 98 if xmin == None: xmin = numpy.nanmin(x)
99 99 if xmax == None: xmax = numpy.nanmax(x)
100 100 if ymin == None: ymin = numpy.nanmin(yreal)
101 101 if ymax == None: ymax = numpy.nanmax(yreal)
102 102
103 103 self.isConfig = True
104 104
105 105 self.setWinTitle(title)
106 106
107 107 for i in range(len(self.axesList)):
108 108 title = "Channel %d" %(i)
109 109 axes = self.axesList[i]
110 110 ychannel = yreal[i,:]
111 111 axes.pline(x, ychannel,
112 112 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
113 113 xlabel=xlabel, ylabel=ylabel, title=title)
114 114
115 115
116 116 def run(self, dataOut, id, wintitle="", channelList=None,
117 117 xmin=None, xmax=None, ymin=None, ymax=None, save=False,
118 118 figpath='./', figfile=None, show=True, wr_period=1,
119 119 ftp=False, server=None, folder=None, username=None, password=None, type='power', **kwargs):
120 120
121 121 """
122 122
123 123 Input:
124 124 dataOut :
125 125 id :
126 126 wintitle :
127 127 channelList :
128 128 xmin : None,
129 129 xmax : None,
130 130 ymin : None,
131 131 ymax : None,
132 132 """
133 133 if dataOut.flagNoData:
134 134 return dataOut
135 135
136 136 if channelList == None:
137 137 channelIndexList = dataOut.channelIndexList
138 138 else:
139 139 channelIndexList = []
140 140 for channel in channelList:
141 141 if channel not in dataOut.channelList:
142 142 raise ValueError("Channel %d is not in dataOut.channelList")
143 143 channelIndexList.append(dataOut.channelList.index(channel))
144 144
145 145 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
146 146
147 147 if dataOut.flagDataAsBlock:
148 148
149 149 for i in range(dataOut.nProfiles):
150 150
151 151 wintitle1 = wintitle + " [Profile = %d] " %i
152 152
153 153 if type == "power":
154 154 self.plot_power(dataOut.heightList,
155 155 dataOut.data[:,i,:],
156 156 id,
157 157 channelIndexList,
158 158 thisDatetime,
159 159 wintitle1,
160 160 show,
161 161 xmin,
162 162 xmax,
163 163 ymin,
164 164 ymax)
165 165
166 166 if type == "iq":
167 167 self.plot_iq(dataOut.heightList,
168 168 dataOut.data[:,i,:],
169 169 id,
170 170 channelIndexList,
171 171 thisDatetime,
172 172 wintitle1,
173 173 show,
174 174 xmin,
175 175 xmax,
176 176 ymin,
177 177 ymax)
178 178
179 179 self.draw()
180 180
181 181 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
182 182 figfile = self.getFilename(name = str_datetime) + "_" + str(i)
183 183
184 184 self.save(figpath=figpath,
185 185 figfile=figfile,
186 186 save=save,
187 187 ftp=ftp,
188 188 wr_period=wr_period,
189 189 thisDatetime=thisDatetime)
190 190
191 191 else:
192 192 wintitle += " [Profile = %d] " %dataOut.profileIndex
193 193
194 194 if type == "power":
195 195 self.plot_power(dataOut.heightList,
196 196 dataOut.data,
197 197 id,
198 198 channelIndexList,
199 199 thisDatetime,
200 200 wintitle,
201 201 show,
202 202 xmin,
203 203 xmax,
204 204 ymin,
205 205 ymax)
206 206
207 207 if type == "iq":
208 208 self.plot_iq(dataOut.heightList,
209 209 dataOut.data,
210 210 id,
211 211 channelIndexList,
212 212 thisDatetime,
213 213 wintitle,
214 214 show,
215 215 xmin,
216 216 xmax,
217 217 ymin,
218 218 ymax)
219 219
220 220 self.draw()
221 221
222 222 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") + "_" + str(dataOut.profileIndex)
223 223 figfile = self.getFilename(name = str_datetime)
224 224
225 225 self.save(figpath=figpath,
226 226 figfile=figfile,
227 227 save=save,
228 228 ftp=ftp,
229 229 wr_period=wr_period,
230 230 thisDatetime=thisDatetime)
231 231
232 return dataOut No newline at end of file
232 return dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1826 +1,1828
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time
14 14 import datetime
15 15 import traceback
16 16 import zmq
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25 25 from schainpy.utils import log
26 26 import schainpy.admin
27 27
28 28 LOCALTIME = True
29 29
30 30
31 31 def isNumber(cad):
32 32 """
33 33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
34 34
35 35 Excepciones:
36 36 Si un determinado string no puede ser convertido a numero
37 37 Input:
38 38 str, string al cual se le analiza para determinar si convertible a un numero o no
39 39
40 40 Return:
41 41 True : si el string es uno numerico
42 42 False : no es un string numerico
43 43 """
44 44 try:
45 45 float(cad)
46 46 return True
47 47 except:
48 48 return False
49 49
50 50
51 51 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
52 52 """
53 53 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
54 54
55 55 Inputs:
56 56 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
57 57
58 58 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
59 59 segundos contados desde 01/01/1970.
60 60 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
61 61 segundos contados desde 01/01/1970.
62 62
63 63 Return:
64 64 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
65 65 fecha especificado, de lo contrario retorna False.
66 66
67 67 Excepciones:
68 68 Si el archivo no existe o no puede ser abierto
69 69 Si la cabecera no puede ser leida.
70 70
71 71 """
72 72 basicHeaderObj = BasicHeader(LOCALTIME)
73 73
74 74 try:
75 75 fp = open(filename, 'rb')
76 76 except IOError:
77 77 print("The file %s can't be opened" % (filename))
78 78 return 0
79 79
80 80 sts = basicHeaderObj.read(fp)
81 81 fp.close()
82 82
83 83 if not(sts):
84 84 print("Skipping the file %s because it has not a valid header" % (filename))
85 85 return 0
86 86
87 87 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
88 88 return 0
89 89
90 90 return 1
91 91
92 92
93 93 def isTimeInRange(thisTime, startTime, endTime):
94 94 if endTime >= startTime:
95 95 if (thisTime < startTime) or (thisTime > endTime):
96 96 return 0
97 97 return 1
98 98 else:
99 99 if (thisTime < startTime) and (thisTime > endTime):
100 100 return 0
101 101 return 1
102 102
103 103
104 104 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
105 105 """
106 106 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
107 107
108 108 Inputs:
109 109 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
110 110
111 111 startDate : fecha inicial del rango seleccionado en formato datetime.date
112 112
113 113 endDate : fecha final del rango seleccionado en formato datetime.date
114 114
115 115 startTime : tiempo inicial del rango seleccionado en formato datetime.time
116 116
117 117 endTime : tiempo final del rango seleccionado en formato datetime.time
118 118
119 119 Return:
120 120 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
121 121 fecha especificado, de lo contrario retorna False.
122 122
123 123 Excepciones:
124 124 Si el archivo no existe o no puede ser abierto
125 125 Si la cabecera no puede ser leida.
126 126
127 127 """
128 128
129 129 try:
130 130 fp = open(filename, 'rb')
131 131 except IOError:
132 132 print("The file %s can't be opened" % (filename))
133 133 return None
134 134
135 135 firstBasicHeaderObj = BasicHeader(LOCALTIME)
136 136 systemHeaderObj = SystemHeader()
137 137 radarControllerHeaderObj = RadarControllerHeader()
138 138 processingHeaderObj = ProcessingHeader()
139 139
140 140 lastBasicHeaderObj = BasicHeader(LOCALTIME)
141 141
142 142 sts = firstBasicHeaderObj.read(fp)
143 143
144 144 if not(sts):
145 145 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
146 146 return None
147 147
148 148 if not systemHeaderObj.read(fp):
149 149 return None
150 150
151 151 if not radarControllerHeaderObj.read(fp):
152 152 return None
153 153
154 154 if not processingHeaderObj.read(fp):
155 155 return None
156 156
157 157 filesize = os.path.getsize(filename)
158 158
159 159 offset = processingHeaderObj.blockSize + 24 # header size
160 160
161 161 if filesize <= offset:
162 162 print("[Reading] %s: This file has not enough data" % filename)
163 163 return None
164 164
165 165 fp.seek(-offset, 2)
166 166
167 167 sts = lastBasicHeaderObj.read(fp)
168 168
169 169 fp.close()
170 170
171 171 thisDatetime = lastBasicHeaderObj.datatime
172 172 thisTime_last_block = thisDatetime.time()
173 173
174 174 thisDatetime = firstBasicHeaderObj.datatime
175 175 thisDate = thisDatetime.date()
176 176 thisTime_first_block = thisDatetime.time()
177 177
178 178 # General case
179 179 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
180 180 #-----------o----------------------------o-----------
181 181 # startTime endTime
182 182
183 183 if endTime >= startTime:
184 184 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
185 185 return None
186 186
187 187 return thisDatetime
188 188
189 189 # If endTime < startTime then endTime belongs to the next day
190 190
191 191 #<<<<<<<<<<<o o>>>>>>>>>>>
192 192 #-----------o----------------------------o-----------
193 193 # endTime startTime
194 194
195 195 if (thisDate == startDate) and (thisTime_last_block < startTime):
196 196 return None
197 197
198 198 if (thisDate == endDate) and (thisTime_first_block > endTime):
199 199 return None
200 200
201 201 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
202 202 return None
203 203
204 204 return thisDatetime
205 205
206 206
207 207 def isFolderInDateRange(folder, startDate=None, endDate=None):
208 208 """
209 209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
210 210
211 211 Inputs:
212 212 folder : nombre completo del directorio.
213 213 Su formato deberia ser "/path_root/?YYYYDDD"
214 214
215 215 siendo:
216 216 YYYY : Anio (ejemplo 2015)
217 217 DDD : Dia del anio (ejemplo 305)
218 218
219 219 startDate : fecha inicial del rango seleccionado en formato datetime.date
220 220
221 221 endDate : fecha final del rango seleccionado en formato datetime.date
222 222
223 223 Return:
224 224 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
225 225 fecha especificado, de lo contrario retorna False.
226 226 Excepciones:
227 227 Si el directorio no tiene el formato adecuado
228 228 """
229 229
230 230 basename = os.path.basename(folder)
231 231
232 232 if not isRadarFolder(basename):
233 233 print("The folder %s has not the rigth format" % folder)
234 234 return 0
235 235
236 236 if startDate and endDate:
237 237 thisDate = getDateFromRadarFolder(basename)
238 238
239 239 if thisDate < startDate:
240 240 return 0
241 241
242 242 if thisDate > endDate:
243 243 return 0
244 244
245 245 return 1
246 246
247 247
248 248 def isFileInDateRange(filename, startDate=None, endDate=None):
249 249 """
250 250 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
251 251
252 252 Inputs:
253 253 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
254 254
255 255 Su formato deberia ser "?YYYYDDDsss"
256 256
257 257 siendo:
258 258 YYYY : Anio (ejemplo 2015)
259 259 DDD : Dia del anio (ejemplo 305)
260 260 sss : set
261 261
262 262 startDate : fecha inicial del rango seleccionado en formato datetime.date
263 263
264 264 endDate : fecha final del rango seleccionado en formato datetime.date
265 265
266 266 Return:
267 267 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
268 268 fecha especificado, de lo contrario retorna False.
269 269 Excepciones:
270 270 Si el archivo no tiene el formato adecuado
271 271 """
272 272
273 273 basename = os.path.basename(filename)
274 274
275 275 if not isRadarFile(basename):
276 276 print("The filename %s has not the rigth format" % filename)
277 277 return 0
278 278
279 279 if startDate and endDate:
280 280 thisDate = getDateFromRadarFile(basename)
281 281
282 282 if thisDate < startDate:
283 283 return 0
284 284
285 285 if thisDate > endDate:
286 286 return 0
287 287
288 288 return 1
289 289
290 290
291 291 def getFileFromSet(path, ext, set):
292 292 validFilelist = []
293 293 fileList = os.listdir(path)
294 294
295 295 # 0 1234 567 89A BCDE
296 296 # H YYYY DDD SSS .ext
297 297
298 298 for thisFile in fileList:
299 299 try:
300 300 year = int(thisFile[1:5])
301 301 doy = int(thisFile[5:8])
302 302 except:
303 303 continue
304 304
305 305 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
306 306 continue
307 307
308 308 validFilelist.append(thisFile)
309 309
310 310 myfile = fnmatch.filter(
311 311 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
312 312
313 313 if len(myfile) != 0:
314 314 return myfile[0]
315 315 else:
316 316 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
317 317 print('the filename %s does not exist' % filename)
318 318 print('...going to the last file: ')
319 319
320 320 if validFilelist:
321 321 validFilelist = sorted(validFilelist, key=str.lower)
322 322 return validFilelist[-1]
323 323
324 324 return None
325 325
326 326
327 327 def getlastFileFromPath(path, ext):
328 328 """
329 329 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
330 330 al final de la depuracion devuelve el ultimo file de la lista que quedo.
331 331
332 332 Input:
333 333 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
334 334 ext : extension de los files contenidos en una carpeta
335 335
336 336 Return:
337 337 El ultimo file de una determinada carpeta, no se considera el path.
338 338 """
339 339 validFilelist = []
340 340 fileList = os.listdir(path)
341 341
342 342 # 0 1234 567 89A BCDE
343 343 # H YYYY DDD SSS .ext
344 344
345 345 for thisFile in fileList:
346 346
347 347 year = thisFile[1:5]
348 348 if not isNumber(year):
349 349 continue
350 350
351 351 doy = thisFile[5:8]
352 352 if not isNumber(doy):
353 353 continue
354 354
355 355 year = int(year)
356 356 doy = int(doy)
357 357
358 358 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
359 359 continue
360 360
361 361 validFilelist.append(thisFile)
362 362
363 363 if validFilelist:
364 364 validFilelist = sorted(validFilelist, key=str.lower)
365 365 return validFilelist[-1]
366 366
367 367 return None
368 368
369 369
370 370 def checkForRealPath(path, foldercounter, year, doy, set, ext):
371 371 """
372 372 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
373 373 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
374 374 el path exacto de un determinado file.
375 375
376 376 Example :
377 377 nombre correcto del file es .../.../D2009307/P2009307367.ext
378 378
379 379 Entonces la funcion prueba con las siguientes combinaciones
380 380 .../.../y2009307367.ext
381 381 .../.../Y2009307367.ext
382 382 .../.../x2009307/y2009307367.ext
383 383 .../.../x2009307/Y2009307367.ext
384 384 .../.../X2009307/y2009307367.ext
385 385 .../.../X2009307/Y2009307367.ext
386 386 siendo para este caso, la ultima combinacion de letras, identica al file buscado
387 387
388 388 Return:
389 389 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
390 390 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
391 391 para el filename
392 392 """
393 393 fullfilename = None
394 394 find_flag = False
395 395 filename = None
396 396
397 397 prefixDirList = [None, 'd', 'D']
398 398 if ext.lower() == ".r": # voltage
399 399 prefixFileList = ['d', 'D']
400 400 elif ext.lower() == ".pdata": # spectra
401 401 prefixFileList = ['p', 'P']
402 402 else:
403 403 return None, filename
404 404
405 405 # barrido por las combinaciones posibles
406 406 for prefixDir in prefixDirList:
407 407 thispath = path
408 408 if prefixDir != None:
409 409 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
410 410 if foldercounter == 0:
411 411 thispath = os.path.join(path, "%s%04d%03d" %
412 412 (prefixDir, year, doy))
413 413 else:
414 414 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
415 415 prefixDir, year, doy, foldercounter))
416 416 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
417 417 # formo el nombre del file xYYYYDDDSSS.ext
418 418 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
419 419 fullfilename = os.path.join(
420 420 thispath, filename) # formo el path completo
421 421
422 422 if os.path.exists(fullfilename): # verifico que exista
423 423 find_flag = True
424 424 break
425 425 if find_flag:
426 426 break
427 427
428 428 if not(find_flag):
429 429 return None, filename
430 430
431 431 return fullfilename, filename
432 432
433 433
434 434 def isRadarFolder(folder):
435 435 try:
436 436 year = int(folder[1:5])
437 437 doy = int(folder[5:8])
438 438 except:
439 439 return 0
440 440
441 441 return 1
442 442
443 443
444 444 def isRadarFile(file):
445 445 try:
446 446 year = int(file[1:5])
447 447 doy = int(file[5:8])
448 448 set = int(file[8:11])
449 449 except:
450 450 return 0
451 451
452 452 return 1
453 453
454 454
455 455 def getDateFromRadarFile(file):
456 456 try:
457 457 year = int(file[1:5])
458 458 doy = int(file[5:8])
459 459 set = int(file[8:11])
460 460 except:
461 461 return None
462 462
463 463 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
464 464 return thisDate
465 465
466 466
467 467 def getDateFromRadarFolder(folder):
468 468 try:
469 469 year = int(folder[1:5])
470 470 doy = int(folder[5:8])
471 471 except:
472 472 return None
473 473
474 474 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
475 475 return thisDate
476 476
477 477
478 478 class JRODataIO:
479 479
480 480 c = 3E8
481 481
482 482 isConfig = False
483 483
484 484 basicHeaderObj = None
485 485
486 486 systemHeaderObj = None
487 487
488 488 radarControllerHeaderObj = None
489 489
490 490 processingHeaderObj = None
491 491
492 492 dtype = None
493 493
494 494 pathList = []
495 495
496 496 filenameList = []
497 497
498 498 filename = None
499 499
500 500 ext = None
501 501
502 502 flagIsNewFile = 1
503 503
504 504 flagDiscontinuousBlock = 0
505 505
506 506 flagIsNewBlock = 0
507 507
508 508 fp = None
509 509
510 510 firstHeaderSize = 0
511 511
512 512 basicHeaderSize = 24
513 513
514 514 versionFile = 1103
515 515
516 516 fileSize = None
517 517
518 518 # ippSeconds = None
519 519
520 520 fileSizeByHeader = None
521 521
522 522 fileIndex = None
523 523
524 524 profileIndex = None
525 525
526 526 blockIndex = None
527 527
528 528 nTotalBlocks = None
529 529
530 530 maxTimeStep = 30
531 531
532 532 lastUTTime = None
533 533
534 534 datablock = None
535 535
536 536 dataOut = None
537 537
538 538 blocksize = None
539 539
540 540 getByBlock = False
541 541
542 542 def __init__(self):
543 543
544 544 raise NotImplementedError
545 545
546 546 def run(self):
547 547
548 548 raise NotImplementedError
549 549
550 550 def getDtypeWidth(self):
551 551
552 552 dtype_index = get_dtype_index(self.dtype)
553 553 dtype_width = get_dtype_width(dtype_index)
554 554
555 555 return dtype_width
556 556
557 557 def getAllowedArgs(self):
558 558 if hasattr(self, '__attrs__'):
559 559 return self.__attrs__
560 560 else:
561 561 return inspect.getargspec(self.run).args
562 562
563 563
564 564 class JRODataReader(JRODataIO):
565 565
566 566 online = 0
567 567
568 568 realtime = 0
569 569
570 570 nReadBlocks = 0
571 571
572 572 delay = 10 # number of seconds waiting a new file
573 573
574 574 nTries = 3 # quantity tries
575 575
576 576 nFiles = 3 # number of files for searching
577 577
578 578 path = None
579 579
580 580 foldercounter = 0
581 581
582 582 flagNoMoreFiles = 0
583 583
584 584 datetimeList = []
585 585
586 586 __isFirstTimeOnline = 1
587 587
588 588 __printInfo = True
589 589
590 590 profileIndex = None
591 591
592 592 nTxs = 1
593 593
594 594 txIndex = None
595 595
596 596 # Added--------------------
597 597
598 598 selBlocksize = None
599 599
600 600 selBlocktime = None
601 601
602 602 def __init__(self):
603 603 """
604 604 This class is used to find data files
605 605
606 606 Example:
607 607 reader = JRODataReader()
608 608 fileList = reader.findDataFiles()
609 609
610 610 """
611 611 pass
612 612
613 613 def createObjByDefault(self):
614 614 """
615 615
616 616 """
617 617 raise NotImplementedError
618 618
619 619 def getBlockDimension(self):
620 620
621 621 raise NotImplementedError
622 622
623 623 def searchFilesOffLine(self,
624 624 path,
625 625 startDate=None,
626 626 endDate=None,
627 627 startTime=datetime.time(0, 0, 0),
628 628 endTime=datetime.time(23, 59, 59),
629 629 set=None,
630 630 expLabel='',
631 631 ext='.r',
632 632 cursor=None,
633 633 skip=None,
634 634 walk=True):
635 635
636 636 self.filenameList = []
637 637 self.datetimeList = []
638 638
639 639 pathList = []
640 640
641 641 dateList, pathList = self.findDatafiles(
642 642 path, startDate, endDate, expLabel, ext, walk, include_path=True)
643 643
644 644 if dateList == []:
645 645 return [], []
646 646
647 647 if len(dateList) > 1:
648 648 print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
649 649 else:
650 650 print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
651 651
652 652 filenameList = []
653 653 datetimeList = []
654 654
655 655 for thisPath in pathList:
656 656
657 657 fileList = glob.glob1(thisPath, "*%s" % ext)
658 658 fileList.sort()
659 659
660 660 for file in fileList:
661 661
662 662 filename = os.path.join(thisPath, file)
663 663
664 664 if not isFileInDateRange(filename, startDate, endDate):
665 665 continue
666 666
667 667 thisDatetime = isFileInTimeRange(
668 668 filename, startDate, endDate, startTime, endTime)
669 669
670 670 if not(thisDatetime):
671 671 continue
672 672
673 673 filenameList.append(filename)
674 674 datetimeList.append(thisDatetime)
675 675
676 676 if cursor is not None and skip is not None:
677 677 filenameList = filenameList[cursor * skip:cursor * skip + skip]
678 678 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
679 679
680 680 if not(filenameList):
681 681 print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
682 682 return [], []
683 683
684 684 print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
685 685
686 686 # for i in range(len(filenameList)):
687 687 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
688 688
689 689 self.filenameList = filenameList
690 690 self.datetimeList = datetimeList
691 691
692 692 return pathList, filenameList
693 693
694 694 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
695 695 """
696 696 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
697 697 devuelve el archivo encontrado ademas de otros datos.
698 698
699 699 Input:
700 700 path : carpeta donde estan contenidos los files que contiene data
701 701
702 702 expLabel : Nombre del subexperimento (subfolder)
703 703
704 704 ext : extension de los files
705 705
706 706 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
707 707
708 708 Return:
709 709 directory : eL directorio donde esta el file encontrado
710 710 filename : el ultimo file de una determinada carpeta
711 711 year : el anho
712 712 doy : el numero de dia del anho
713 713 set : el set del archivo
714 714
715 715
716 716 """
717 717 if not os.path.isdir(path):
718 718 return None, None, None, None, None, None
719 719
720 720 dirList = []
721 721
722 722 if not walk:
723 723 fullpath = path
724 724 foldercounter = 0
725 725 else:
726 726 # Filtra solo los directorios
727 727 for thisPath in os.listdir(path):
728 728 if not os.path.isdir(os.path.join(path, thisPath)):
729 729 continue
730 730 if not isRadarFolder(thisPath):
731 731 continue
732 732
733 733 dirList.append(thisPath)
734 734
735 735 if not(dirList):
736 736 return None, None, None, None, None, None
737 737
738 738 dirList = sorted(dirList, key=str.lower)
739 739
740 740 doypath = dirList[-1]
741 741 foldercounter = int(doypath.split('_')[1]) if len(
742 742 doypath.split('_')) > 1 else 0
743 743 fullpath = os.path.join(path, doypath, expLabel)
744 744
745 745 print("[Reading] %s folder was found: " % (fullpath))
746 746
747 747 if set == None:
748 748 filename = getlastFileFromPath(fullpath, ext)
749 749 else:
750 750 filename = getFileFromSet(fullpath, ext, set)
751 751
752 752 if not(filename):
753 753 return None, None, None, None, None, None
754 754
755 755 print("[Reading] %s file was found" % (filename))
756 756
757 757 if not(self.__verifyFile(os.path.join(fullpath, filename))):
758 758 return None, None, None, None, None, None
759 759
760 760 year = int(filename[1:5])
761 761 doy = int(filename[5:8])
762 762 set = int(filename[8:11])
763 763
764 764 return fullpath, foldercounter, filename, year, doy, set
765 765
766 766 def __setNextFileOffline(self):
767 767
768 768 idFile = self.fileIndex
769 769
770 770 while (True):
771 771 idFile += 1
772 772 if not(idFile < len(self.filenameList)):
773 773 self.flagNoMoreFiles = 1
774 774 # print "[Reading] No more Files"
775 775 return 0
776 776
777 777 filename = self.filenameList[idFile]
778 778
779 779 if not(self.__verifyFile(filename)):
780 780 continue
781 781
782 782 fileSize = os.path.getsize(filename)
783 783 fp = open(filename, 'rb')
784 784 break
785 785
786 786 self.flagIsNewFile = 1
787 787 self.fileIndex = idFile
788 788 self.filename = filename
789 789 self.fileSize = fileSize
790 790 self.fp = fp
791 791
792 792 # print "[Reading] Setting the file: %s"%self.filename
793 793
794 794 return 1
795 795
796 796 def __setNextFileOnline(self):
797 797 """
798 798 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
799 799 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
800 800 siguientes.
801 801
802 802 Affected:
803 803 self.flagIsNewFile
804 804 self.filename
805 805 self.fileSize
806 806 self.fp
807 807 self.set
808 808 self.flagNoMoreFiles
809 809
810 810 Return:
811 811 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
812 812 1 : si el file fue abierto con exito y esta listo a ser leido
813 813
814 814 Excepciones:
815 815 Si un determinado file no puede ser abierto
816 816 """
817 817 nFiles = 0
818 818 fileOk_flag = False
819 819 firstTime_flag = True
820 820
821 821 self.set += 1
822 822
823 823 if self.set > 999:
824 824 self.set = 0
825 825 self.foldercounter += 1
826 826
827 827 # busca el 1er file disponible
828 828 fullfilename, filename = checkForRealPath(
829 829 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
830 830 if fullfilename:
831 831 if self.__verifyFile(fullfilename, False):
832 832 fileOk_flag = True
833 833
834 834 # si no encuentra un file entonces espera y vuelve a buscar
835 835 if not(fileOk_flag):
836 836 # busco en los siguientes self.nFiles+1 files posibles
837 837 for nFiles in range(self.nFiles + 1):
838 838
839 839 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
840 840 tries = self.nTries
841 841 else:
842 842 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
843 843
844 844 for nTries in range(tries):
845 845 if firstTime_flag:
846 846 print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1))
847 847 sleep(self.delay)
848 848 else:
849 849 print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext))
850 850
851 851 fullfilename, filename = checkForRealPath(
852 852 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
853 853 if fullfilename:
854 854 if self.__verifyFile(fullfilename):
855 855 fileOk_flag = True
856 856 break
857 857
858 858 if fileOk_flag:
859 859 break
860 860
861 861 firstTime_flag = False
862 862
863 863 log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
864 864 self.set += 1
865 865
866 866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
867 867 if nFiles == (self.nFiles - 1):
868 868 self.set = 0
869 869 self.doy += 1
870 870 self.foldercounter = 0
871 871
872 872 if fileOk_flag:
873 873 self.fileSize = os.path.getsize(fullfilename)
874 874 self.filename = fullfilename
875 875 self.flagIsNewFile = 1
876 876 if self.fp != None:
877 877 self.fp.close()
878 878 self.fp = open(fullfilename, 'rb')
879 879 self.flagNoMoreFiles = 0
880 880 # print '[Reading] Setting the file: %s' % fullfilename
881 881 else:
882 882 self.fileSize = 0
883 883 self.filename = None
884 884 self.flagIsNewFile = 0
885 885 self.fp = None
886 886 self.flagNoMoreFiles = 1
887 887 # print '[Reading] No more files to read'
888 888
889 889 return fileOk_flag
890 890
891 891 def setNextFile(self):
892 892 if self.fp != None:
893 893 self.fp.close()
894 894
895 895 if self.online:
896 896 newFile = self.__setNextFileOnline()
897 897 else:
898 898 newFile = self.__setNextFileOffline()
899 899
900 900 if not(newFile):
901 901 self.dataOut.error = (-1, 'No more files to read')
902 902 return 0
903 903
904 904 if self.verbose:
905 905 print('[Reading] Setting the file: %s' % self.filename)
906 906
907 907 self.__readFirstHeader()
908 908 self.nReadBlocks = 0
909 909 return 1
910 910
911 911 def __waitNewBlock(self):
912 912 """
913 913 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
914 914
915 915 Si el modo de lectura es OffLine siempre retorn 0
916 916 """
917 917 if not self.online:
918 918 return 0
919 919
920 920 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
921 921 return 0
922 922
923 923 currentPointer = self.fp.tell()
924 924
925 925 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
926 926
927 927 for nTries in range(self.nTries):
928 928
929 929 self.fp.close()
930 930 self.fp = open(self.filename, 'rb')
931 931 self.fp.seek(currentPointer)
932 932
933 933 self.fileSize = os.path.getsize(self.filename)
934 934 currentSize = self.fileSize - currentPointer
935 935
936 936 if (currentSize >= neededSize):
937 937 self.basicHeaderObj.read(self.fp)
938 938 return 1
939 939
940 940 if self.fileSize == self.fileSizeByHeader:
941 941 # self.flagEoF = True
942 942 return 0
943 943
944 944 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
945 945 sleep(self.delay)
946 946
947 947 return 0
948 948
949 949 def waitDataBlock(self, pointer_location):
950 950
951 951 currentPointer = pointer_location
952 952
953 953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
954 954
955 955 for nTries in range(self.nTries):
956 956 self.fp.close()
957 957 self.fp = open(self.filename, 'rb')
958 958 self.fp.seek(currentPointer)
959 959
960 960 self.fileSize = os.path.getsize(self.filename)
961 961 currentSize = self.fileSize - currentPointer
962 962
963 963 if (currentSize >= neededSize):
964 964 return 1
965 965
966 966 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
967 967 sleep(self.delay)
968 968
969 969 return 0
970 970
971 971 def __jumpToLastBlock(self):
972 972
973 973 if not(self.__isFirstTimeOnline):
974 974 return
975 975
976 976 csize = self.fileSize - self.fp.tell()
977 977 blocksize = self.processingHeaderObj.blockSize
978 978
979 979 # salta el primer bloque de datos
980 980 if csize > self.processingHeaderObj.blockSize:
981 981 self.fp.seek(self.fp.tell() + blocksize)
982 982 else:
983 983 return
984 984
985 985 csize = self.fileSize - self.fp.tell()
986 986 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
987 987 while True:
988 988
989 989 if self.fp.tell() < self.fileSize:
990 990 self.fp.seek(self.fp.tell() + neededsize)
991 991 else:
992 992 self.fp.seek(self.fp.tell() - neededsize)
993 993 break
994 994
995 995 # csize = self.fileSize - self.fp.tell()
996 996 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
997 997 # factor = int(csize/neededsize)
998 998 # if factor > 0:
999 999 # self.fp.seek(self.fp.tell() + factor*neededsize)
1000 1000
1001 1001 self.flagIsNewFile = 0
1002 1002 self.__isFirstTimeOnline = 0
1003 1003
1004 1004 def __setNewBlock(self):
1005 1005 # if self.server is None:
1006 1006 if self.fp == None:
1007 1007 return 0
1008 1008
1009 1009 # if self.online:
1010 1010 # self.__jumpToLastBlock()
1011 1011
1012 1012 if self.flagIsNewFile:
1013 1013 self.lastUTTime = self.basicHeaderObj.utc
1014 1014 return 1
1015 1015
1016 1016 if self.realtime:
1017 1017 self.flagDiscontinuousBlock = 1
1018 1018 if not(self.setNextFile()):
1019 1019 return 0
1020 1020 else:
1021 1021 return 1
1022 1022 # if self.server is None:
1023 1023 currentSize = self.fileSize - self.fp.tell()
1024 1024 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1025 1025 if (currentSize >= neededSize):
1026 1026 self.basicHeaderObj.read(self.fp)
1027 1027 self.lastUTTime = self.basicHeaderObj.utc
1028 1028 return 1
1029 1029 # else:
1030 1030 # self.basicHeaderObj.read(self.zHeader)
1031 1031 # self.lastUTTime = self.basicHeaderObj.utc
1032 1032 # return 1
1033 1033 if self.__waitNewBlock():
1034 1034 self.lastUTTime = self.basicHeaderObj.utc
1035 1035 return 1
1036 1036 # if self.server is None:
1037 1037 if not(self.setNextFile()):
1038 1038 return 0
1039 1039
1040 1040 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1041 1041 self.lastUTTime = self.basicHeaderObj.utc
1042 1042
1043 1043 self.flagDiscontinuousBlock = 0
1044 1044
1045 1045 if deltaTime > self.maxTimeStep:
1046 1046 self.flagDiscontinuousBlock = 1
1047 1047
1048 1048 return 1
1049 1049
1050 1050 def readNextBlock(self):
1051 1051
1052 1052 # Skip block out of startTime and endTime
1053 1053 while True:
1054 1054 if not(self.__setNewBlock()):
1055 1055 self.dataOut.error = (-1, 'No more files to read')
1056 1056 return 0
1057 1057
1058 1058 if not(self.readBlock()):
1059 1059 return 0
1060 1060
1061 1061 self.getBasicHeader()
1062 1062 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1063 1063 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1064 1064 self.processingHeaderObj.dataBlocksPerFile,
1065 1065 self.dataOut.datatime.ctime()))
1066 1066 continue
1067 1067
1068 1068 break
1069 1069
1070 1070 if self.verbose:
1071 1071 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1072 1072 self.processingHeaderObj.dataBlocksPerFile,
1073 1073 self.dataOut.datatime.ctime()))
1074 1074 return 1
1075 1075
1076 1076 def __readFirstHeader(self):
1077 1077
1078 1078 self.basicHeaderObj.read(self.fp)
1079 1079 self.systemHeaderObj.read(self.fp)
1080 1080 self.radarControllerHeaderObj.read(self.fp)
1081 1081 self.processingHeaderObj.read(self.fp)
1082 1082
1083 1083 self.firstHeaderSize = self.basicHeaderObj.size
1084 1084
1085 1085 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1086 1086 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1087 1087 if datatype == 0:
1088 1088 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1089 1089 elif datatype == 1:
1090 1090 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1091 1091 elif datatype == 2:
1092 1092 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1093 1093 elif datatype == 3:
1094 1094 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1095 1095 elif datatype == 4:
1096 1096 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1097 1097 elif datatype == 5:
1098 1098 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1099 1099 else:
1100 1100 raise ValueError('Data type was not defined')
1101 1101
1102 1102 self.dtype = datatype_str
1103 1103 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1104 1104 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1105 1105 self.firstHeaderSize + self.basicHeaderSize * \
1106 1106 (self.processingHeaderObj.dataBlocksPerFile - 1)
1107 1107 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1108 1108 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1109 1109 self.getBlockDimension()
1110 1110
1111 1111 def __verifyFile(self, filename, msgFlag=True):
1112 1112
1113 1113 msg = None
1114 1114
1115 1115 try:
1116 1116 fp = open(filename, 'rb')
1117 1117 except IOError:
1118 1118
1119 1119 if msgFlag:
1120 1120 print("[Reading] File %s can't be opened" % (filename))
1121 1121
1122 1122 return False
1123 1123
1124 1124 currentPosition = fp.tell()
1125 1125 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1126 1126
1127 1127 if neededSize == 0:
1128 1128 basicHeaderObj = BasicHeader(LOCALTIME)
1129 1129 systemHeaderObj = SystemHeader()
1130 1130 radarControllerHeaderObj = RadarControllerHeader()
1131 1131 processingHeaderObj = ProcessingHeader()
1132 1132
1133 1133 if not(basicHeaderObj.read(fp)):
1134 1134 fp.close()
1135 1135 return False
1136 1136
1137 1137 if not(systemHeaderObj.read(fp)):
1138 1138 fp.close()
1139 1139 return False
1140 1140
1141 1141 if not(radarControllerHeaderObj.read(fp)):
1142 1142 fp.close()
1143 1143 return False
1144 1144
1145 1145 if not(processingHeaderObj.read(fp)):
1146 1146 fp.close()
1147 1147 return False
1148 1148
1149 1149 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1150 1150 else:
1151 1151 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1152 1152
1153 1153 fp.close()
1154 1154
1155 1155 fileSize = os.path.getsize(filename)
1156 1156 currentSize = fileSize - currentPosition
1157 1157
1158 1158 if currentSize < neededSize:
1159 1159 if msgFlag and (msg != None):
1160 1160 print(msg)
1161 1161 return False
1162 1162
1163 1163 return True
1164 1164
1165 1165 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1166 1166
1167 1167 path_empty = True
1168 1168
1169 1169 dateList = []
1170 1170 pathList = []
1171 1171
1172 1172 multi_path = path.split(',')
1173 1173
1174 1174 if not walk:
1175 1175
1176 1176 for single_path in multi_path:
1177 1177
1178 1178 if not os.path.isdir(single_path):
1179 1179 continue
1180 1180
1181 1181 fileList = glob.glob1(single_path, "*" + ext)
1182 1182
1183 1183 if not fileList:
1184 1184 continue
1185 1185
1186 1186 path_empty = False
1187 1187
1188 1188 fileList.sort()
1189 1189
1190 1190 for thisFile in fileList:
1191 1191
1192 1192 if not os.path.isfile(os.path.join(single_path, thisFile)):
1193 1193 continue
1194 1194
1195 1195 if not isRadarFile(thisFile):
1196 1196 continue
1197 1197
1198 1198 if not isFileInDateRange(thisFile, startDate, endDate):
1199 1199 continue
1200 1200
1201 1201 thisDate = getDateFromRadarFile(thisFile)
1202 1202
1203 1203 if thisDate in dateList:
1204 1204 continue
1205 1205
1206 1206 dateList.append(thisDate)
1207 1207 pathList.append(single_path)
1208 1208
1209 1209 else:
1210 1210 for single_path in multi_path:
1211 1211
1212 1212 if not os.path.isdir(single_path):
1213 1213 continue
1214 1214
1215 1215 dirList = []
1216 1216
1217 1217 for thisPath in os.listdir(single_path):
1218 1218
1219 1219 if not os.path.isdir(os.path.join(single_path, thisPath)):
1220 1220 continue
1221 1221
1222 1222 if not isRadarFolder(thisPath):
1223 1223 continue
1224 1224
1225 1225 if not isFolderInDateRange(thisPath, startDate, endDate):
1226 1226 continue
1227 1227
1228 1228 dirList.append(thisPath)
1229 1229
1230 1230 if not dirList:
1231 1231 continue
1232 1232
1233 1233 dirList.sort()
1234 1234
1235 1235 for thisDir in dirList:
1236 1236
1237 1237 datapath = os.path.join(single_path, thisDir, expLabel)
1238 1238 fileList = glob.glob1(datapath, "*" + ext)
1239 1239
1240 1240 if not fileList:
1241 1241 continue
1242 1242
1243 1243 path_empty = False
1244 1244
1245 1245 thisDate = getDateFromRadarFolder(thisDir)
1246 1246
1247 1247 pathList.append(datapath)
1248 1248 dateList.append(thisDate)
1249 1249
1250 1250 dateList.sort()
1251 1251
1252 1252 if walk:
1253 1253 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1254 1254 else:
1255 1255 pattern_path = multi_path[0]
1256 1256
1257 1257 if path_empty:
1258 1258 print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1259 1259 else:
1260 1260 if not dateList:
1261 1261 print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1262 1262
1263 1263 if include_path:
1264 1264 return dateList, pathList
1265 1265
1266 1266 return dateList
1267 1267
1268 1268 def setup(self,
1269 1269 path=None,
1270 1270 startDate=None,
1271 1271 endDate=None,
1272 1272 startTime=datetime.time(0, 0, 0),
1273 1273 endTime=datetime.time(23, 59, 59),
1274 1274 set=None,
1275 1275 expLabel="",
1276 1276 ext=None,
1277 1277 online=False,
1278 1278 delay=60,
1279 1279 walk=True,
1280 1280 getblock=False,
1281 1281 nTxs=1,
1282 1282 realtime=False,
1283 1283 blocksize=None,
1284 1284 blocktime=None,
1285 1285 skip=None,
1286 1286 cursor=None,
1287 1287 warnings=True,
1288 1288 verbose=True,
1289 1289 server=None,
1290 1290 format=None,
1291 1291 oneDDict=None,
1292 1292 twoDDict=None,
1293 1293 ind2DList=None):
1294 1294 if server is not None:
1295 1295 if 'tcp://' in server:
1296 1296 address = server
1297 1297 else:
1298 1298 address = 'ipc:///tmp/%s' % server
1299 1299 self.server = address
1300 1300 self.context = zmq.Context()
1301 1301 self.receiver = self.context.socket(zmq.PULL)
1302 1302 self.receiver.connect(self.server)
1303 1303 time.sleep(0.5)
1304 1304 print('[Starting] ReceiverData from {}'.format(self.server))
1305 1305 else:
1306 1306 self.server = None
1307 1307 if path == None:
1308 1308 raise ValueError("[Reading] The path is not valid")
1309 1309
1310 1310 if ext == None:
1311 1311 ext = self.ext
1312 1312
1313 1313 if online:
1314 1314 print("[Reading] Searching files in online mode...")
1315 1315
1316 1316 for nTries in range(self.nTries):
1317 1317 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1318 1318 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1319 1319
1320 1320 if fullpath:
1321 1321 break
1322 1322
1323 1323 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
1324 1324 sleep(self.delay)
1325 1325
1326 1326 if not(fullpath):
1327 1327 self.dataOut.error = (-1, 'There isn\'t any valid file in {}'.format(path))
1328 1328 return
1329 1329
1330 1330 self.year = year
1331 1331 self.doy = doy
1332 1332 self.set = set - 1
1333 1333 self.path = path
1334 1334 self.foldercounter = foldercounter
1335 1335 last_set = None
1336 1336 else:
1337 1337 print("[Reading] Searching files in offline mode ...")
1338 1338 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1339 1339 startTime=startTime, endTime=endTime,
1340 1340 set=set, expLabel=expLabel, ext=ext,
1341 1341 walk=walk, cursor=cursor,
1342 1342 skip=skip)
1343 1343
1344 1344 if not(pathList):
1345 1345 self.fileIndex = -1
1346 1346 self.pathList = []
1347 1347 self.filenameList = []
1348 1348 return
1349 1349
1350 1350 self.fileIndex = -1
1351 1351 self.pathList = pathList
1352 1352 self.filenameList = filenameList
1353 1353 file_name = os.path.basename(filenameList[-1])
1354 1354 basename, ext = os.path.splitext(file_name)
1355 1355 last_set = int(basename[-3:])
1356 1356
1357 1357 self.online = online
1358 1358 self.realtime = realtime
1359 1359 self.delay = delay
1360 1360 ext = ext.lower()
1361 1361 self.ext = ext
1362 1362 self.getByBlock = getblock
1363 1363 self.nTxs = nTxs
1364 1364 self.startTime = startTime
1365 1365 self.endTime = endTime
1366 1366 self.endDate = endDate
1367 1367 self.startDate = startDate
1368 1368 # Added-----------------
1369 1369 self.selBlocksize = blocksize
1370 1370 self.selBlocktime = blocktime
1371 1371
1372 1372 # Verbose-----------
1373 1373 self.verbose = verbose
1374 1374 self.warnings = warnings
1375 1375
1376 1376 if not(self.setNextFile()):
1377 1377 if (startDate != None) and (endDate != None):
1378 1378 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
1379 1379 elif startDate != None:
1380 1380 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
1381 1381 else:
1382 1382 print("[Reading] No files")
1383 1383
1384 1384 self.fileIndex = -1
1385 1385 self.pathList = []
1386 1386 self.filenameList = []
1387 1387 return
1388 1388
1389 1389 # self.getBasicHeader()
1390 1390
1391 1391 if last_set != None:
1392 1392 self.dataOut.last_block = last_set * \
1393 1393 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1394 1394 return
1395 1395
1396 1396 def getBasicHeader(self):
1397 1397
1398 1398 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1399 1399 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1400 1400
1401 1401 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1402 1402
1403 1403 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1404 1404
1405 1405 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1406 1406
1407 1407 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1408 1408
1409 1409 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1410 1410
1411 1411 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1412 1412
1413 1413 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1414 1414
1415 1415 def getFirstHeader(self):
1416 1416
1417 1417 raise NotImplementedError
1418 1418
1419 1419 def getData(self):
1420 1420
1421 1421 raise NotImplementedError
1422 1422
1423 1423 def hasNotDataInBuffer(self):
1424 1424
1425 1425 raise NotImplementedError
1426 1426
1427 1427 def readBlock(self):
1428 1428
1429 1429 raise NotImplementedError
1430 1430
1431 1431 def isEndProcess(self):
1432 1432
1433 1433 return self.flagNoMoreFiles
1434 1434
1435 1435 def printReadBlocks(self):
1436 1436
1437 1437 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1438 1438
1439 1439 def printTotalBlocks(self):
1440 1440
1441 1441 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1442 1442
1443 1443 def printNumberOfBlock(self):
1444 1444 'SPAM!'
1445 1445
1446 1446 # if self.flagIsNewBlock:
1447 1447 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1448 1448 # self.processingHeaderObj.dataBlocksPerFile,
1449 1449 # self.dataOut.datatime.ctime())
1450 1450
1451 1451 def printInfo(self):
1452 1452
1453 1453 if self.__printInfo == False:
1454 1454 return
1455 1455
1456 1456 self.basicHeaderObj.printInfo()
1457 1457 self.systemHeaderObj.printInfo()
1458 1458 self.radarControllerHeaderObj.printInfo()
1459 1459 self.processingHeaderObj.printInfo()
1460 1460
1461 1461 self.__printInfo = False
1462 1462
1463 1463 def run(self,
1464 1464 path=None,
1465 1465 startDate=None,
1466 1466 endDate=None,
1467 1467 startTime=datetime.time(0, 0, 0),
1468 1468 endTime=datetime.time(23, 59, 59),
1469 1469 set=None,
1470 1470 expLabel="",
1471 1471 ext=None,
1472 1472 online=False,
1473 1473 delay=60,
1474 1474 walk=True,
1475 1475 getblock=False,
1476 1476 nTxs=1,
1477 1477 realtime=False,
1478 1478 blocksize=None,
1479 1479 blocktime=None,
1480 1480 skip=None,
1481 1481 cursor=None,
1482 1482 warnings=True,
1483 1483 server=None,
1484 1484 verbose=True,
1485 1485 format=None,
1486 1486 oneDDict=None,
1487 1487 twoDDict=None,
1488 1488 ind2DList=None, **kwargs):
1489 1489
1490 1490 if not(self.isConfig):
1491 1491 self.setup(path=path,
1492 1492 startDate=startDate,
1493 1493 endDate=endDate,
1494 1494 startTime=startTime,
1495 1495 endTime=endTime,
1496 1496 set=set,
1497 1497 expLabel=expLabel,
1498 1498 ext=ext,
1499 1499 online=online,
1500 1500 delay=delay,
1501 1501 walk=walk,
1502 1502 getblock=getblock,
1503 1503 nTxs=nTxs,
1504 1504 realtime=realtime,
1505 1505 blocksize=blocksize,
1506 1506 blocktime=blocktime,
1507 1507 skip=skip,
1508 1508 cursor=cursor,
1509 1509 warnings=warnings,
1510 1510 server=server,
1511 1511 verbose=verbose,
1512 1512 format=format,
1513 1513 oneDDict=oneDDict,
1514 1514 twoDDict=twoDDict,
1515 1515 ind2DList=ind2DList)
1516 1516 self.isConfig = True
1517 1517 if server is None:
1518 1518 self.getData()
1519 1519 else:
1520 1520 self.getFromServer()
1521 1521
1522 1522
1523 1523 class JRODataWriter(JRODataIO):
1524 1524
1525 1525 """
1526 1526 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1527 1527 de los datos siempre se realiza por bloques.
1528 1528 """
1529 1529
1530 1530 blockIndex = 0
1531 1531
1532 1532 path = None
1533 1533
1534 1534 setFile = None
1535 1535
1536 1536 profilesPerBlock = None
1537 1537
1538 1538 blocksPerFile = None
1539 1539
1540 1540 nWriteBlocks = 0
1541 1541
1542 1542 fileDate = None
1543 1543
1544 1544 def __init__(self, dataOut=None):
1545 1545 raise NotImplementedError
1546 1546
1547 1547 def hasAllDataInBuffer(self):
1548 1548 raise NotImplementedError
1549 1549
1550 1550 def setBlockDimension(self):
1551 1551 raise NotImplementedError
1552 1552
1553 1553 def writeBlock(self):
1554 1554 raise NotImplementedError
1555 1555
1556 1556 def putData(self):
1557 1557 raise NotImplementedError
1558 1558
1559 1559 def getProcessFlags(self):
1560 1560
1561 1561 processFlags = 0
1562 1562
1563 1563 dtype_index = get_dtype_index(self.dtype)
1564 1564 procflag_dtype = get_procflag_dtype(dtype_index)
1565 1565
1566 1566 processFlags += procflag_dtype
1567 1567
1568 1568 if self.dataOut.flagDecodeData:
1569 1569 processFlags += PROCFLAG.DECODE_DATA
1570 1570
1571 1571 if self.dataOut.flagDeflipData:
1572 1572 processFlags += PROCFLAG.DEFLIP_DATA
1573 1573
1574 1574 if self.dataOut.code is not None:
1575 1575 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1576 1576
1577 1577 if self.dataOut.nCohInt > 1:
1578 1578 processFlags += PROCFLAG.COHERENT_INTEGRATION
1579 1579
1580 1580 if self.dataOut.type == "Spectra":
1581 1581 if self.dataOut.nIncohInt > 1:
1582 1582 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1583 1583
1584 1584 if self.dataOut.data_dc is not None:
1585 1585 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1586 1586
1587 1587 if self.dataOut.flagShiftFFT:
1588 1588 processFlags += PROCFLAG.SHIFT_FFT_DATA
1589 1589
1590 1590 return processFlags
1591 1591
1592 1592 def setBasicHeader(self):
1593 1593
1594 1594 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1595 1595 self.basicHeaderObj.version = self.versionFile
1596 1596 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1597 1597
1598 1598 utc = numpy.floor(self.dataOut.utctime)
1599 1599 milisecond = (self.dataOut.utctime - utc) * 1000.0
1600 1600
1601 1601 self.basicHeaderObj.utc = utc
1602 1602 self.basicHeaderObj.miliSecond = milisecond
1603 1603 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1604 1604 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1605 1605 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1606 1606
1607 1607 def setFirstHeader(self):
1608 1608 """
1609 1609 Obtiene una copia del First Header
1610 1610
1611 1611 Affected:
1612 1612
1613 1613 self.basicHeaderObj
1614 1614 self.systemHeaderObj
1615 1615 self.radarControllerHeaderObj
1616 1616 self.processingHeaderObj self.
1617 1617
1618 1618 Return:
1619 1619 None
1620 1620 """
1621 1621
1622 1622 raise NotImplementedError
1623 1623
1624 1624 def __writeFirstHeader(self):
1625 1625 """
1626 1626 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1627 1627
1628 1628 Affected:
1629 1629 __dataType
1630 1630
1631 1631 Return:
1632 1632 None
1633 1633 """
1634 1634
1635 1635 # CALCULAR PARAMETROS
1636 1636
1637 1637 sizeLongHeader = self.systemHeaderObj.size + \
1638 1638 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1639 1639 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1640 1640
1641 1641 self.basicHeaderObj.write(self.fp)
1642 1642 self.systemHeaderObj.write(self.fp)
1643 1643 self.radarControllerHeaderObj.write(self.fp)
1644 1644 self.processingHeaderObj.write(self.fp)
1645 1645
1646 1646 def __setNewBlock(self):
1647 1647 """
1648 1648 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1649 1649
1650 1650 Return:
1651 1651 0 : si no pudo escribir nada
1652 1652 1 : Si escribio el Basic el First Header
1653 1653 """
1654 1654 if self.fp == None:
1655 1655 self.setNextFile()
1656 1656
1657 1657 if self.flagIsNewFile:
1658 1658 return 1
1659 1659
1660 1660 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1661 1661 self.basicHeaderObj.write(self.fp)
1662 1662 return 1
1663 1663
1664 1664 if not(self.setNextFile()):
1665 1665 return 0
1666 1666
1667 1667 return 1
1668 1668
1669 1669 def writeNextBlock(self):
1670 1670 """
1671 1671 Selecciona el bloque siguiente de datos y los escribe en un file
1672 1672
1673 1673 Return:
1674 1674 0 : Si no hizo pudo escribir el bloque de datos
1675 1675 1 : Si no pudo escribir el bloque de datos
1676 1676 """
1677 1677 if not(self.__setNewBlock()):
1678 1678 return 0
1679 1679
1680 1680 self.writeBlock()
1681 1681
1682 1682 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1683 1683 self.processingHeaderObj.dataBlocksPerFile))
1684 1684
1685 1685 return 1
1686 1686
1687 1687 def setNextFile(self):
1688 1688 """
1689 1689 Determina el siguiente file que sera escrito
1690 1690
1691 1691 Affected:
1692 1692 self.filename
1693 1693 self.subfolder
1694 1694 self.fp
1695 1695 self.setFile
1696 1696 self.flagIsNewFile
1697 1697
1698 1698 Return:
1699 1699 0 : Si el archivo no puede ser escrito
1700 1700 1 : Si el archivo esta listo para ser escrito
1701 1701 """
1702 1702 ext = self.ext
1703 1703 path = self.path
1704 1704
1705 1705 if self.fp != None:
1706 1706 self.fp.close()
1707 1707
1708 1708 timeTuple = time.localtime(self.dataOut.utctime)
1709 1709 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1710 1710
1711 1711 fullpath = os.path.join(path, subfolder)
1712 1712 setFile = self.setFile
1713 1713
1714 1714 if not(os.path.exists(fullpath)):
1715 1715 os.mkdir(fullpath)
1716 1716 setFile = -1 # inicializo mi contador de seteo
1717 1717 else:
1718 1718 filesList = os.listdir(fullpath)
1719 1719 if len(filesList) > 0:
1720 1720 filesList = sorted(filesList, key=str.lower)
1721 1721 filen = filesList[-1]
1722 1722 # el filename debera tener el siguiente formato
1723 1723 # 0 1234 567 89A BCDE (hex)
1724 1724 # x YYYY DDD SSS .ext
1725 1725 if isNumber(filen[8:11]):
1726 1726 # inicializo mi contador de seteo al seteo del ultimo file
1727 1727 setFile = int(filen[8:11])
1728 1728 else:
1729 1729 setFile = -1
1730 1730 else:
1731 1731 setFile = -1 # inicializo mi contador de seteo
1732 1732
1733 1733 setFile += 1
1734 1734
1735 1735 # If this is a new day it resets some values
1736 1736 if self.dataOut.datatime.date() > self.fileDate:
1737 1737 setFile = 0
1738 1738 self.nTotalBlocks = 0
1739 1739
1740 1740 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1741 1741 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1742 1742
1743 1743 filename = os.path.join(path, subfolder, filen)
1744 1744
1745 1745 fp = open(filename, 'wb')
1746 1746
1747 1747 self.blockIndex = 0
1748 1748
1749 1749 # guardando atributos
1750 1750 self.filename = filename
1751 1751 self.subfolder = subfolder
1752 1752 self.fp = fp
1753 1753 self.setFile = setFile
1754 1754 self.flagIsNewFile = 1
1755 1755 self.fileDate = self.dataOut.datatime.date()
1756 1756
1757 1757 self.setFirstHeader()
1758 1758
1759 1759 print('[Writing] Opening file: %s' % self.filename)
1760 1760
1761 1761 self.__writeFirstHeader()
1762 1762
1763 1763 return 1
1764 1764
1765 1765 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1766 1766 """
1767 1767 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1768 1768
1769 1769 Inputs:
1770 1770 path : directory where data will be saved
1771 1771 profilesPerBlock : number of profiles per block
1772 1772 set : initial file set
1773 1773 datatype : An integer number that defines data type:
1774 1774 0 : int8 (1 byte)
1775 1775 1 : int16 (2 bytes)
1776 1776 2 : int32 (4 bytes)
1777 1777 3 : int64 (8 bytes)
1778 1778 4 : float32 (4 bytes)
1779 1779 5 : double64 (8 bytes)
1780 1780
1781 1781 Return:
1782 1782 0 : Si no realizo un buen seteo
1783 1783 1 : Si realizo un buen seteo
1784 1784 """
1785 1785
1786 1786 if ext == None:
1787 1787 ext = self.ext
1788 1788
1789 1789 self.ext = ext.lower()
1790 1790
1791 1791 self.path = path
1792 1792
1793 1793 if set is None:
1794 1794 self.setFile = -1
1795 1795 else:
1796 1796 self.setFile = set - 1
1797 1797
1798 1798 self.blocksPerFile = blocksPerFile
1799 1799
1800 1800 self.profilesPerBlock = profilesPerBlock
1801 1801
1802 1802 self.dataOut = dataOut
1803 1803 self.fileDate = self.dataOut.datatime.date()
1804 1804 # By default
1805 1805 self.dtype = self.dataOut.dtype
1806 1806
1807 1807 if datatype is not None:
1808 1808 self.dtype = get_numpy_dtype(datatype)
1809 1809
1810 1810 if not(self.setNextFile()):
1811 1811 print("[Writing] There isn't a next file")
1812 1812 return 0
1813 1813
1814 1814 self.setBlockDimension()
1815 1815
1816 1816 return 1
1817 1817
1818 1818 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1819 1819
1820 1820 if not(self.isConfig):
1821 1821
1822 1822 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1823 1823 set=set, ext=ext, datatype=datatype, **kwargs)
1824 1824 self.isConfig = True
1825 1825
1826 self.putData() No newline at end of file
1826 self.dataOut = dataOut
1827 self.putData()
1828 return self.dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1095 +1,1103
1 1 import numpy
2 2 import time
3 3 import os
4 4 import h5py
5 5 import re
6 6 import datetime
7 7
8 8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 10 # from .jroIO_base import *
11 11 from schainpy.model.io.jroIO_base import *
12 12 import schainpy
13 from schainpy.utils import log
13 14
14
15 class ParamReader(ProcessingUnit):
15 @MPDecorator
16 class ParamReader(JRODataReader,ProcessingUnit):
16 17 '''
17 18 Reads HDF5 format files
18 19
19 20 path
20 21
21 22 startDate
22 23
23 24 endDate
24 25
25 26 startTime
26 27
27 28 endTime
28 29 '''
29 30
30 31 ext = ".hdf5"
31 32
32 33 optchar = "D"
33 34
34 35 timezone = None
35 36
36 37 startTime = None
37 38
38 39 endTime = None
39 40
40 41 fileIndex = None
41 42
42 43 utcList = None #To select data in the utctime list
43 44
44 45 blockList = None #List to blocks to be read from the file
45 46
46 47 blocksPerFile = None #Number of blocks to be read
47 48
48 49 blockIndex = None
49 50
50 51 path = None
51 52
52 53 #List of Files
53 54
54 55 filenameList = None
55 56
56 57 datetimeList = None
57 58
58 59 #Hdf5 File
59 60
60 61 listMetaname = None
61 62
62 63 listMeta = None
63 64
64 65 listDataname = None
65 66
66 67 listData = None
67 68
68 69 listShapes = None
69 70
70 71 fp = None
71 72
72 73 #dataOut reconstruction
73 74
74 75 dataOut = None
75 76
76 77
77 def __init__(self, **kwargs):
78 ProcessingUnit.__init__(self, **kwargs)
78 def __init__(self):#, **kwargs):
79 ProcessingUnit.__init__(self) #, **kwargs)
79 80 self.dataOut = Parameters()
80 81 return
81 82
82 83 def setup(self, **kwargs):
83 84
84 85 path = kwargs['path']
85 86 startDate = kwargs['startDate']
86 87 endDate = kwargs['endDate']
87 88 startTime = kwargs['startTime']
88 89 endTime = kwargs['endTime']
89 90 walk = kwargs['walk']
90 91 if 'ext' in kwargs:
91 92 ext = kwargs['ext']
92 93 else:
93 94 ext = '.hdf5'
94 95 if 'timezone' in kwargs:
95 96 self.timezone = kwargs['timezone']
96 97 else:
97 98 self.timezone = 'lt'
98 99
99 100 print("[Reading] Searching files in offline mode ...")
100 101 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 102 startTime=startTime, endTime=endTime,
102 103 ext=ext, walk=walk)
103 104
104 105 if not(filenameList):
105 106 print("There is no files into the folder: %s"%(path))
106 107 sys.exit(-1)
107 108
108 109 self.fileIndex = -1
109 110 self.startTime = startTime
110 111 self.endTime = endTime
111 112
112 113 self.__readMetadata()
113 114
114 115 self.__setNextFileOffline()
115 116
116 117 return
117 118
118 119 def searchFilesOffLine(self,
119 120 path,
120 121 startDate=None,
121 122 endDate=None,
122 123 startTime=datetime.time(0,0,0),
123 124 endTime=datetime.time(23,59,59),
124 125 ext='.hdf5',
125 126 walk=True):
126 127
127 128 expLabel = ''
128 129 self.filenameList = []
129 130 self.datetimeList = []
130 131
131 132 pathList = []
132 133
133 134 JRODataObj = JRODataReader()
134 135 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
135 136
136 137 if dateList == []:
137 138 print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
138 139 datetime.datetime.combine(startDate,startTime).ctime(),
139 140 datetime.datetime.combine(endDate,endTime).ctime()))
140 141
141 142 return None, None
142 143
143 144 if len(dateList) > 1:
144 145 print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
145 146 else:
146 147 print("[Reading] data was found for the date %s" %(dateList[0]))
147 148
148 149 filenameList = []
149 150 datetimeList = []
150 151
151 152 #----------------------------------------------------------------------------------
152 153
153 154 for thisPath in pathList:
154 155 # thisPath = pathList[pathDict[file]]
155 156
156 157 fileList = glob.glob1(thisPath, "*%s" %ext)
157 158 fileList.sort()
158 159
159 160 for file in fileList:
160 161
161 162 filename = os.path.join(thisPath,file)
162 163
163 164 if not isFileInDateRange(filename, startDate, endDate):
164 165 continue
165 166
166 167 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
167 168
168 169 if not(thisDatetime):
169 170 continue
170 171
171 172 filenameList.append(filename)
172 173 datetimeList.append(thisDatetime)
173 174
174 175 if not(filenameList):
175 176 print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
176 177 return None, None
177 178
178 179 print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
179 180 print()
180 181
181 182 # for i in range(len(filenameList)):
182 183 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
183 184
184 185 self.filenameList = filenameList
185 186 self.datetimeList = datetimeList
186 187
187 188 return pathList, filenameList
188 189
189 190 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
190 191
191 192 """
192 193 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
193 194
194 195 Inputs:
195 196 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
196 197
197 198 startDate : fecha inicial del rango seleccionado en formato datetime.date
198 199
199 200 endDate : fecha final del rango seleccionado en formato datetime.date
200 201
201 202 startTime : tiempo inicial del rango seleccionado en formato datetime.time
202 203
203 204 endTime : tiempo final del rango seleccionado en formato datetime.time
204 205
205 206 Return:
206 207 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
207 208 fecha especificado, de lo contrario retorna False.
208 209
209 210 Excepciones:
210 211 Si el archivo no existe o no puede ser abierto
211 212 Si la cabecera no puede ser leida.
212 213
213 214 """
214 215
215 216 try:
216 217 fp = h5py.File(filename,'r')
217 218 grp1 = fp['Data']
218 219
219 220 except IOError:
220 221 traceback.print_exc()
221 222 raise IOError("The file %s can't be opened" %(filename))
222 223 #chino rata
223 224 #In case has utctime attribute
224 225 grp2 = grp1['utctime']
225 226 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
226 227 thisUtcTime = grp2.value[0]
227 228
228 229 fp.close()
229 230
230 231 if self.timezone == 'lt':
231 232 thisUtcTime -= 5*3600
232 233
233 234 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
234 235 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
235 236 thisDate = thisDatetime.date()
236 237 thisTime = thisDatetime.time()
237 238
238 239 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 240 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
240 241
241 242 #General case
242 243 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
243 244 #-----------o----------------------------o-----------
244 245 # startTime endTime
245 246
246 247 if endTime >= startTime:
247 248 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
248 249 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
249 250 return thisDatetime
250 251 return None
251 252
252 253 #If endTime < startTime then endTime belongs to the next day
253 254 #<<<<<<<<<<<o o>>>>>>>>>>>
254 255 #-----------o----------------------------o-----------
255 256 # endTime startTime
256 257
257 258 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
258 259 return None
259 260
260 261 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
261 262 return None
262 263
263 264 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
264 265 return None
265 266
266 267 return thisDatetime
267 268
268 269 def __setNextFileOffline(self):
269 270
270 271 self.fileIndex += 1
271 272 idFile = self.fileIndex
272 273
273 274 if not(idFile < len(self.filenameList)):
274 275 print("No more Files")
275 276 return 0
276 277
277 278 filename = self.filenameList[idFile]
278 279
279 280 filePointer = h5py.File(filename,'r')
280 281
281 282 self.filename = filename
282 283
283 284 self.fp = filePointer
284 285
285 286 print("Setting the file: %s"%self.filename)
286 287
287 288 # self.__readMetadata()
288 289 self.__setBlockList()
289 290 self.__readData()
290 291 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
291 292 # self.nRecords = self.fp['Data'].attrs['nRecords']
292 293 self.blockIndex = 0
293 294 return 1
294 295
295 296 def __setBlockList(self):
296 297 '''
297 298 Selects the data within the times defined
298 299
299 300 self.fp
300 301 self.startTime
301 302 self.endTime
302 303
303 304 self.blockList
304 305 self.blocksPerFile
305 306
306 307 '''
307 308 fp = self.fp
308 309 startTime = self.startTime
309 310 endTime = self.endTime
310 311
311 312 grp = fp['Data']
312 313 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
313 314
314 315 #ERROOOOR
315 316 if self.timezone == 'lt':
316 317 thisUtcTime -= 5*3600
317 318
318 319 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
319 320
320 321 thisDate = thisDatetime.date()
321 322 thisTime = thisDatetime.time()
322 323
323 324 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 325 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
325 326
326 327 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
327 328
328 329 self.blockList = ind
329 330 self.blocksPerFile = len(ind)
330 331
331 332 return
332 333
333 334 def __readMetadata(self):
334 335 '''
335 336 Reads Metadata
336 337
337 338 self.pathMeta
338 339
339 340 self.listShapes
340 341 self.listMetaname
341 342 self.listMeta
342 343
343 344 '''
344 345
345 346 # grp = self.fp['Data']
346 347 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
347 348 #
348 349 # if pathMeta == self.pathMeta:
349 350 # return
350 351 # else:
351 352 # self.pathMeta = pathMeta
352 353 #
353 354 # filePointer = h5py.File(self.pathMeta,'r')
354 355 # groupPointer = filePointer['Metadata']
355 356
356 357 filename = self.filenameList[0]
357 358
358 359 fp = h5py.File(filename,'r')
359 360
360 361 gp = fp['Metadata']
361 362
362 363 listMetaname = []
363 364 listMetadata = []
364 365 for item in list(gp.items()):
365 366 name = item[0]
366 367
367 368 if name=='array dimensions':
368 369 table = gp[name][:]
369 370 listShapes = {}
370 371 for shapes in table:
371 372 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
372 373 else:
373 374 data = gp[name].value
374 375 listMetaname.append(name)
375 376 listMetadata.append(data)
376 377
377 378 # if name=='type':
378 379 # self.__initDataOut(data)
379 380
380 381 self.listShapes = listShapes
381 382 self.listMetaname = listMetaname
382 383 self.listMeta = listMetadata
383 384
384 385 fp.close()
385 386 return
386 387
387 388 def __readData(self):
388 389 grp = self.fp['Data']
389 390 listdataname = []
390 391 listdata = []
391 392
392 393 for item in list(grp.items()):
393 394 name = item[0]
394 395 listdataname.append(name)
395 396
396 397 array = self.__setDataArray(grp[name],self.listShapes[name])
397 398 listdata.append(array)
398 399
399 400 self.listDataname = listdataname
400 401 self.listData = listdata
401 402 return
402 403
403 404 def __setDataArray(self, dataset, shapes):
404 405
405 406 nDims = shapes[0]
406 407
407 408 nDim2 = shapes[1] #Dimension 0
408 409
409 410 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
410 411
411 412 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
412 413
413 414 mode = shapes[4] #Mode of storing
414 415
415 416 blockList = self.blockList
416 417
417 418 blocksPerFile = self.blocksPerFile
418 419
419 420 #Depending on what mode the data was stored
420 421 if mode == 0: #Divided in channels
421 422 arrayData = dataset.value.astype(numpy.float)[0][blockList]
422 423 if mode == 1: #Divided in parameter
423 424 strds = 'table'
424 425 nDatas = nDim1
425 426 newShapes = (blocksPerFile,nDim2,nDim0)
426 427 elif mode==2: #Concatenated in a table
427 428 strds = 'table0'
428 429 arrayData = dataset[strds].value
429 430 #Selecting part of the dataset
430 431 utctime = arrayData[:,0]
431 432 u, indices = numpy.unique(utctime, return_index=True)
432 433
433 434 if blockList.size != indices.size:
434 435 indMin = indices[blockList[0]]
435 436 if blockList[1] + 1 >= indices.size:
436 437 arrayData = arrayData[indMin:,:]
437 438 else:
438 439 indMax = indices[blockList[1] + 1]
439 440 arrayData = arrayData[indMin:indMax,:]
440 441 return arrayData
441 442
442 443 # One dimension
443 444 if nDims == 0:
444 445 arrayData = dataset.value.astype(numpy.float)[0][blockList]
445 446
446 447 # Two dimensions
447 448 elif nDims == 2:
448 449 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
449 450 newShapes = (blocksPerFile,nDim0)
450 451 nDatas = nDim1
451 452
452 453 for i in range(nDatas):
453 454 data = dataset[strds + str(i)].value
454 455 arrayData[:,i,:] = data[blockList,:]
455 456
456 457 # Three dimensions
457 458 else:
458 459 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
459 460 for i in range(nDatas):
460 461
461 462 data = dataset[strds + str(i)].value
462 463
463 464 for b in range(blockList.size):
464 465 arrayData[b,:,i,:] = data[:,:,blockList[b]]
465 466
466 467 return arrayData
467 468
468 469 def __setDataOut(self):
469 470 listMeta = self.listMeta
470 471 listMetaname = self.listMetaname
471 472 listDataname = self.listDataname
472 473 listData = self.listData
473 474 listShapes = self.listShapes
474 475
475 476 blockIndex = self.blockIndex
476 477 # blockList = self.blockList
477 478
478 479 for i in range(len(listMeta)):
479 480 setattr(self.dataOut,listMetaname[i],listMeta[i])
480 481
481 482 for j in range(len(listData)):
482 483 nShapes = listShapes[listDataname[j]][0]
483 484 mode = listShapes[listDataname[j]][4]
484 485 if nShapes == 1:
485 486 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
486 487 elif nShapes > 1:
487 488 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
488 489 elif mode==0:
489 490 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
490 491 #Mode Meteors
491 492 elif mode ==2:
492 493 selectedData = self.__selectDataMode2(listData[j], blockIndex)
493 494 setattr(self.dataOut, listDataname[j], selectedData)
494 495 return
495 496
496 497 def __selectDataMode2(self, data, blockIndex):
497 498 utctime = data[:,0]
498 499 aux, indices = numpy.unique(utctime, return_inverse=True)
499 500 selInd = numpy.where(indices == blockIndex)[0]
500 501 selData = data[selInd,:]
501 502
502 503 return selData
503 504
504 505 def getData(self):
505 506
506 507 # if self.flagNoMoreFiles:
507 508 # self.dataOut.flagNoData = True
508 509 # print 'Process finished'
509 510 # return 0
510 511 #
511 512 if self.blockIndex==self.blocksPerFile:
512 513 if not( self.__setNextFileOffline() ):
513 514 self.dataOut.flagNoData = True
514 515 return 0
515 516
516 517 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
517 518 # self.dataOut.flagNoData = True
518 519 # return 0
519 520 # self.__readData()
520 521 self.__setDataOut()
521 522 self.dataOut.flagNoData = False
522 523
523 524 self.blockIndex += 1
524 525
525 526 return
526 527
527 528 def run(self, **kwargs):
528 529
529 530 if not(self.isConfig):
530 531 self.setup(**kwargs)
531 532 # self.setObjProperties()
532 533 self.isConfig = True
533 534
534 535 self.getData()
535 536
536 537 return
537
538 @MPDecorator
538 539 class ParamWriter(Operation):
539 540 '''
540 541 HDF5 Writer, stores parameters data in HDF5 format files
541 542
542 543 path: path where the files will be stored
543 544
544 545 blocksPerFile: number of blocks that will be saved in per HDF5 format file
545 546
546 547 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
547 548
548 549 metadataList: list of attributes that will be stored as metadata
549 550
550 551 dataList: list of attributes that will be stores as data
551 552
552 553 '''
553 554
554 555
555 556 ext = ".hdf5"
556 557
557 558 optchar = "D"
558 559
559 560 metaoptchar = "M"
560 561
561 562 metaFile = None
562 563
563 564 filename = None
564 565
565 566 path = None
566 567
567 568 setFile = None
568 569
569 570 fp = None
570 571
571 572 grp = None
572 573
573 574 ds = None
574 575
575 576 firsttime = True
576 577
577 578 #Configurations
578 579
579 580 blocksPerFile = None
580 581
581 582 blockIndex = None
582 583
583 584 dataOut = None
584 585
585 586 #Data Arrays
586 587
587 588 dataList = None
588 589
589 590 metadataList = None
590 591
591 592 # arrayDim = None
592 593
593 594 dsList = None #List of dictionaries with dataset properties
594 595
595 596 tableDim = None
596 597
597 598 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
598 599
599 600 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
600 601
601 602 currentDay = None
602 603
603 604 lastTime = None
604 605
605 def __init__(self, **kwargs):
606 Operation.__init__(self, **kwargs)
607 self.isConfig = False
606 def __init__(self):#, **kwargs):
607 Operation.__init__(self)#, **kwargs)
608 #self.isConfig = False
608 609 return
609 610
610 611 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
611 612 self.path = path
612 613 self.blocksPerFile = blocksPerFile
613 614 self.metadataList = metadataList
614 615 self.dataList = dataList
615 616 self.dataOut = dataOut
616 617 self.mode = mode
617
618 618 if self.mode is not None:
619 619 self.mode = numpy.zeros(len(self.dataList)) + mode
620 620 else:
621 #self.mode = numpy.ones(len(self.dataList),int)
621 622 self.mode = numpy.ones(len(self.dataList))
623 log.error(self.mode)#yong
622 624
623 625 arrayDim = numpy.zeros((len(self.dataList),5))
624 626
625 627 #Table dimensions
626 628 dtype0 = self.dtype
627 629 tableList = []
628 630
629 631 #Dictionary and list of tables
630 632 dsList = []
631 633
632 634 for i in range(len(self.dataList)):
633 635 dsDict = {}
634 636 dataAux = getattr(self.dataOut, self.dataList[i])
635 637 dsDict['variable'] = self.dataList[i]
636 638 #--------------------- Conditionals ------------------------
637 639 #There is no data
638 640 if dataAux is None:
639 641 return 0
640 642
641 643 #Not array, just a number
642 644 #Mode 0
645 #log.error(mode)#yong
646 #log.error(len(mode))#yong
647 #log.error(type(mode))#yong
643 648 if type(dataAux)==float or type(dataAux)==int:
644 649 dsDict['mode'] = 0
645 650 dsDict['nDim'] = 0
646 651 arrayDim[i,0] = 0
647 652 dsList.append(dsDict)
648 653
649 654 #Mode 2: meteors
650 elif mode[i] == 2:
655 elif self.mode[i] == 2:
651 656 # dsDict['nDim'] = 0
652 657 dsDict['dsName'] = 'table0'
653 658 dsDict['mode'] = 2 # Mode meteors
654 659 dsDict['shape'] = dataAux.shape[-1]
655 660 dsDict['nDim'] = 0
656 661 dsDict['dsNumber'] = 1
657 662
658 663 arrayDim[i,3] = dataAux.shape[-1]
659 arrayDim[i,4] = mode[i] #Mode the data was stored
664 arrayDim[i,4] = self.mode[i] #Mode the data was stored
660 665
661 666 dsList.append(dsDict)
662 667
663 668 #Mode 1
664 669 else:
665 670 arrayDim0 = dataAux.shape #Data dimensions
666 671 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
667 arrayDim[i,4] = mode[i] #Mode the data was stored
672 arrayDim[i,4] = self.mode[i] #Mode the data was stored
668 673
669 674 strtable = 'table'
670 675 dsDict['mode'] = 1 # Mode parameters
671 676
672 677 # Three-dimension arrays
673 678 if len(arrayDim0) == 3:
674 679 arrayDim[i,1:-1] = numpy.array(arrayDim0)
675 680 nTables = int(arrayDim[i,2])
676 681 dsDict['dsNumber'] = nTables
677 682 dsDict['shape'] = arrayDim[i,2:4]
678 683 dsDict['nDim'] = 3
679 684
680 685 for j in range(nTables):
681 686 dsDict = dsDict.copy()
682 687 dsDict['dsName'] = strtable + str(j)
683 688 dsList.append(dsDict)
684 689
685 690 # Two-dimension arrays
686 691 elif len(arrayDim0) == 2:
687 692 arrayDim[i,2:-1] = numpy.array(arrayDim0)
688 693 nTables = int(arrayDim[i,2])
689 694 dsDict['dsNumber'] = nTables
690 695 dsDict['shape'] = arrayDim[i,3]
691 696 dsDict['nDim'] = 2
692 697
693 698 for j in range(nTables):
694 699 dsDict = dsDict.copy()
695 700 dsDict['dsName'] = strtable + str(j)
696 701 dsList.append(dsDict)
697 702
698 703 # One-dimension arrays
699 704 elif len(arrayDim0) == 1:
700 705 arrayDim[i,3] = arrayDim0[0]
701 706 dsDict['shape'] = arrayDim0[0]
702 707 dsDict['dsNumber'] = 1
703 708 dsDict['dsName'] = strtable + str(0)
704 709 dsDict['nDim'] = 1
705 710 dsList.append(dsDict)
706 711
707 712 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
708 713 tableList.append(table)
709 714
710 715 # self.arrayDim = arrayDim
711 716 self.dsList = dsList
712 717 self.tableDim = numpy.array(tableList, dtype = dtype0)
713 718 self.blockIndex = 0
714 719
715 720 timeTuple = time.localtime(dataOut.utctime)
716 721 self.currentDay = timeTuple.tm_yday
717 722 return 1
718 723
719 724 def putMetadata(self):
720 725
721 726 fp = self.createMetadataFile()
722 727 self.writeMetadata(fp)
723 728 fp.close()
724 729 return
725 730
726 731 def createMetadataFile(self):
727 732 ext = self.ext
728 733 path = self.path
729 734 setFile = self.setFile
730 735
731 736 timeTuple = time.localtime(self.dataOut.utctime)
732 737
733 738 subfolder = ''
734 739 fullpath = os.path.join( path, subfolder )
735 740
736 741 if not( os.path.exists(fullpath) ):
737 742 os.mkdir(fullpath)
738 743 setFile = -1 #inicializo mi contador de seteo
739 744
740 745 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
741 746 fullpath = os.path.join( path, subfolder )
742 747
743 748 if not( os.path.exists(fullpath) ):
744 749 os.mkdir(fullpath)
745 750 setFile = -1 #inicializo mi contador de seteo
746 751
747 752 else:
748 753 filesList = os.listdir( fullpath )
749 754 filesList = sorted( filesList, key=str.lower )
750 755 if len( filesList ) > 0:
751 756 filesList = [k for k in filesList if 'M' in k]
752 757 filen = filesList[-1]
753 758 # el filename debera tener el siguiente formato
754 759 # 0 1234 567 89A BCDE (hex)
755 760 # x YYYY DDD SSS .ext
756 761 if isNumber( filen[8:11] ):
757 762 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
758 763 else:
759 764 setFile = -1
760 765 else:
761 766 setFile = -1 #inicializo mi contador de seteo
762 767
763 768 if self.setType is None:
764 769 setFile += 1
765 770 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
766 771 timeTuple.tm_year,
767 772 timeTuple.tm_yday,
768 773 setFile,
769 774 ext )
770 775 else:
771 776 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
772 777 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
773 778 timeTuple.tm_year,
774 779 timeTuple.tm_yday,
775 780 setFile,
776 781 ext )
777 782
778 783 filename = os.path.join( path, subfolder, file )
779 784 self.metaFile = file
780 785 #Setting HDF5 File
781 786 fp = h5py.File(filename,'w')
782 787
783 788 return fp
784 789
785 790 def writeMetadata(self, fp):
786 791
787 792 grp = fp.create_group("Metadata")
788 793 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
789 794
790 795 for i in range(len(self.metadataList)):
791 796 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
792 797 return
793 798
794 799 def timeFlag(self):
795 800 currentTime = self.dataOut.utctime
796 801
797 802 if self.lastTime is None:
798 803 self.lastTime = currentTime
799 804
800 805 #Day
801 806 timeTuple = time.localtime(currentTime)
802 807 dataDay = timeTuple.tm_yday
803 808
804 809 #Time
805 810 timeDiff = currentTime - self.lastTime
806 811
807 812 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
808 813 if dataDay != self.currentDay:
809 814 self.currentDay = dataDay
810 815 return True
811 816 elif timeDiff > 3*60*60:
812 817 self.lastTime = currentTime
813 818 return True
814 819 else:
815 820 self.lastTime = currentTime
816 821 return False
817 822
818 823 def setNextFile(self):
819 824
820 825 ext = self.ext
821 826 path = self.path
822 827 setFile = self.setFile
823 828 mode = self.mode
824 829
825 830 timeTuple = time.localtime(self.dataOut.utctime)
826 831 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
827 832
828 833 fullpath = os.path.join( path, subfolder )
829 834
830 835 if os.path.exists(fullpath):
831 836 filesList = os.listdir( fullpath )
832 837 filesList = [k for k in filesList if 'D' in k]
833 838 if len( filesList ) > 0:
834 839 filesList = sorted( filesList, key=str.lower )
835 840 filen = filesList[-1]
836 841 # el filename debera tener el siguiente formato
837 842 # 0 1234 567 89A BCDE (hex)
838 843 # x YYYY DDD SSS .ext
839 844 if isNumber( filen[8:11] ):
840 845 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
841 846 else:
842 847 setFile = -1
843 848 else:
844 849 setFile = -1 #inicializo mi contador de seteo
845 850 else:
846 851 os.makedirs(fullpath)
847 852 setFile = -1 #inicializo mi contador de seteo
848 853
849 if self.setType is None:
854 if None is None:
850 855 setFile += 1
851 856 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
852 857 timeTuple.tm_year,
853 858 timeTuple.tm_yday,
854 859 setFile,
855 860 ext )
856 861 else:
857 862 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
858 863 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
859 864 timeTuple.tm_year,
860 865 timeTuple.tm_yday,
861 866 setFile,
862 867 ext )
863 868
864 869 filename = os.path.join( path, subfolder, file )
865 870
866 871 #Setting HDF5 File
867 872 fp = h5py.File(filename,'w')
868 873 #write metadata
869 874 self.writeMetadata(fp)
870 875 #Write data
871 876 grp = fp.create_group("Data")
872 877 # grp.attrs['metadata'] = self.metaFile
873 878
874 879 # grp.attrs['blocksPerFile'] = 0
875 880 ds = []
876 881 data = []
877 882 dsList = self.dsList
878 883 i = 0
879 884 while i < len(dsList):
880 885 dsInfo = dsList[i]
881 886 #One-dimension data
882 887 if dsInfo['mode'] == 0:
883 888 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
884 889 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
885 890 ds.append(ds0)
886 891 data.append([])
887 892 i += 1
888 893 continue
889 894 # nDimsForDs.append(nDims[i])
890 895
891 896 elif dsInfo['mode'] == 2:
892 897 grp0 = grp.create_group(dsInfo['variable'])
893 898 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
894 899 ds.append(ds0)
895 900 data.append([])
896 901 i += 1
897 902 continue
898 903
899 904 elif dsInfo['mode'] == 1:
900 905 grp0 = grp.create_group(dsInfo['variable'])
901 906
902 907 for j in range(dsInfo['dsNumber']):
903 908 dsInfo = dsList[i]
904 909 tableName = dsInfo['dsName']
905 shape = int(dsInfo['shape'])
910
906 911
907 912 if dsInfo['nDim'] == 3:
913 shape = dsInfo['shape'].astype(int)
908 914 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
909 915 else:
916 shape = int(dsInfo['shape'])
910 917 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
911 918
912 919 ds.append(ds0)
913 920 data.append([])
914 921 i += 1
915 922 # nDimsForDs.append(nDims[i])
916 923
917 924 fp.flush()
918 925 fp.close()
919 926
920 927 # self.nDatas = nDatas
921 928 # self.nDims = nDims
922 929 # self.nDimsForDs = nDimsForDs
923 930 #Saving variables
924 931 print('Writing the file: %s'%filename)
925 932 self.filename = filename
926 933 # self.fp = fp
927 934 # self.grp = grp
928 935 # self.grp.attrs.modify('nRecords', 1)
929 936 self.ds = ds
930 937 self.data = data
931 938 # self.setFile = setFile
932 939 self.firsttime = True
933 940 self.blockIndex = 0
934 941 return
935 942
936 943 def putData(self):
937 944
938 945 if self.blockIndex == self.blocksPerFile or self.timeFlag():
939 946 self.setNextFile()
940 947
941 948 # if not self.firsttime:
942 949 self.readBlock()
943 950 self.setBlock() #Prepare data to be written
944 951 self.writeBlock() #Write data
945 952
946 953 return
947 954
948 955 def readBlock(self):
949 956
950 957 '''
951 958 data Array configured
952 959
953 960
954 961 self.data
955 962 '''
956 963 dsList = self.dsList
957 964 ds = self.ds
958 965 #Setting HDF5 File
959 966 fp = h5py.File(self.filename,'r+')
960 967 grp = fp["Data"]
961 968 ind = 0
962 969
963 970 # grp.attrs['blocksPerFile'] = 0
964 971 while ind < len(dsList):
965 972 dsInfo = dsList[ind]
966 973
967 974 if dsInfo['mode'] == 0:
968 975 ds0 = grp[dsInfo['variable']]
969 976 ds[ind] = ds0
970 977 ind += 1
971 978 else:
972 979
973 980 grp0 = grp[dsInfo['variable']]
974 981
975 982 for j in range(dsInfo['dsNumber']):
976 983 dsInfo = dsList[ind]
977 984 ds0 = grp0[dsInfo['dsName']]
978 985 ds[ind] = ds0
979 986 ind += 1
980 987
981 988 self.fp = fp
982 989 self.grp = grp
983 990 self.ds = ds
984 991
985 992 return
986 993
987 994 def setBlock(self):
988 995 '''
989 996 data Array configured
990 997
991 998
992 999 self.data
993 1000 '''
994 1001 #Creating Arrays
995 1002 dsList = self.dsList
996 1003 data = self.data
997 1004 ind = 0
998 1005
999 1006 while ind < len(dsList):
1000 1007 dsInfo = dsList[ind]
1001 1008 dataAux = getattr(self.dataOut, dsInfo['variable'])
1002 1009
1003 1010 mode = dsInfo['mode']
1004 1011 nDim = dsInfo['nDim']
1005 1012
1006 1013 if mode == 0 or mode == 2 or nDim == 1:
1007 1014 data[ind] = dataAux
1008 1015 ind += 1
1009 1016 # elif nDim == 1:
1010 1017 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1011 1018 # ind += 1
1012 1019 elif nDim == 2:
1013 1020 for j in range(dsInfo['dsNumber']):
1014 1021 data[ind] = dataAux[j,:]
1015 1022 ind += 1
1016 1023 elif nDim == 3:
1017 1024 for j in range(dsInfo['dsNumber']):
1018 1025 data[ind] = dataAux[:,j,:]
1019 1026 ind += 1
1020 1027
1021 1028 self.data = data
1022 1029 return
1023 1030
1024 1031 def writeBlock(self):
1025 1032 '''
1026 1033 Saves the block in the HDF5 file
1027 1034 '''
1028 1035 dsList = self.dsList
1029 1036
1030 1037 for i in range(len(self.ds)):
1031 1038 dsInfo = dsList[i]
1032 1039 nDim = dsInfo['nDim']
1033 1040 mode = dsInfo['mode']
1034 1041
1035 1042 # First time
1036 1043 if self.firsttime:
1037 1044 # self.ds[i].resize(self.data[i].shape)
1038 1045 # self.ds[i][self.blockIndex,:] = self.data[i]
1039 1046 if type(self.data[i]) == numpy.ndarray:
1040 1047
1041 1048 if nDim == 3:
1042 1049 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1043 1050 self.ds[i].resize(self.data[i].shape)
1044 1051 if mode == 2:
1045 1052 self.ds[i].resize(self.data[i].shape)
1046 1053 self.ds[i][:] = self.data[i]
1047 1054 else:
1048 1055
1049 1056 # From second time
1050 1057 # Meteors!
1051 1058 if mode == 2:
1052 1059 dataShape = self.data[i].shape
1053 1060 dsShape = self.ds[i].shape
1054 1061 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1055 1062 self.ds[i][dsShape[0]:,:] = self.data[i]
1056 1063 # No dimension
1057 1064 elif mode == 0:
1058 1065 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1059 1066 self.ds[i][0,-1] = self.data[i]
1060 1067 # One dimension
1061 1068 elif nDim == 1:
1062 1069 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1063 1070 self.ds[i][-1,:] = self.data[i]
1064 1071 # Two dimension
1065 1072 elif nDim == 2:
1066 1073 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1067 1074 self.ds[i][self.blockIndex,:] = self.data[i]
1068 1075 # Three dimensions
1069 1076 elif nDim == 3:
1070 1077 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1071 1078 self.ds[i][:,:,-1] = self.data[i]
1072 1079
1073 1080 self.firsttime = False
1074 1081 self.blockIndex += 1
1075 1082
1076 1083 #Close to save changes
1077 1084 self.fp.flush()
1078 1085 self.fp.close()
1079 1086 return
1080 1087
1081 def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1088 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1082 1089
1083 1090 if not(self.isConfig):
1084 1091 flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1085 1092 metadataList=metadataList, dataList=dataList, mode=mode, **kwargs)
1086 1093
1087 1094 if not(flagdata):
1088 1095 return
1089 1096
1090 1097 self.isConfig = True
1091 1098 # self.putMetadata()
1092 1099 self.setNextFile()
1093 1100
1094 1101 self.putData()
1095 return No newline at end of file
1102 return
1103 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,680 +1,679
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import numpy
7 7
8 8 from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 10 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 11 from schainpy.model.data.jrodata import Spectra
12 from schainpy.utils import log
12 13
13 14 @MPDecorator
14 15 class SpectraReader(JRODataReader, ProcessingUnit):
15 16 """
16 17 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
17 18 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
18 19 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
19 20
20 21 paresCanalesIguales * alturas * perfiles (Self Spectra)
21 22 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
22 23 canales * alturas (DC Channels)
23 24
24 25 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
25 26 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
26 27 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
27 28 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
28 29
29 30 Example:
30 31 dpath = "/home/myuser/data"
31 32
32 33 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
33 34
34 35 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
35 36
36 37 readerObj = SpectraReader()
37 38
38 39 readerObj.setup(dpath, startTime, endTime)
39 40
40 41 while(True):
41 42
42 43 readerObj.getData()
43 44
44 45 print readerObj.data_spc
45 46
46 47 print readerObj.data_cspc
47 48
48 49 print readerObj.data_dc
49 50
50 51 if readerObj.flagNoMoreFiles:
51 52 break
52 53
53 54 """
54 55
55 56 pts2read_SelfSpectra = 0
56 57
57 58 pts2read_CrossSpectra = 0
58 59
59 60 pts2read_DCchannels = 0
60 61
61 62 ext = ".pdata"
62 63
63 64 optchar = "P"
64 65
65 66 dataOut = None
66 67
67 68 nRdChannels = None
68 69
69 70 nRdPairs = None
70 71
71 72 rdPairList = []
72 73
73 74 def __init__(self):#, **kwargs):
74 75 """
75 76 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
76 77
77 78 Inputs:
78 79 dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para
79 80 almacenar un perfil de datos cada vez que se haga un requerimiento
80 81 (getData). El perfil sera obtenido a partir del buffer de datos,
81 82 si el buffer esta vacio se hara un nuevo proceso de lectura de un
82 83 bloque de datos.
83 84 Si este parametro no es pasado se creara uno internamente.
84 85
85 86 Affected:
86 87 self.dataOut
87 88
88 89 Return : None
89 90 """
90 91
91 92 #Eliminar de la base la herencia
92 93 ProcessingUnit.__init__(self)#, **kwargs)
93 94
94 95 # self.isConfig = False
95 96
96 97 self.pts2read_SelfSpectra = 0
97 98
98 99 self.pts2read_CrossSpectra = 0
99 100
100 101 self.pts2read_DCchannels = 0
101 102
102 103 self.datablock = None
103 104
104 105 self.utc = None
105 106
106 107 self.ext = ".pdata"
107 108
108 109 self.optchar = "P"
109 110
110 111 self.basicHeaderObj = BasicHeader(LOCALTIME)
111 112
112 113 self.systemHeaderObj = SystemHeader()
113 114
114 115 self.radarControllerHeaderObj = RadarControllerHeader()
115 116
116 117 self.processingHeaderObj = ProcessingHeader()
117 118
118 119 self.online = 0
119 120
120 121 self.fp = None
121 122
122 123 self.idFile = None
123 124
124 125 self.dtype = None
125 126
126 127 self.fileSizeByHeader = None
127 128
128 129 self.filenameList = []
129 130
130 131 self.filename = None
131 132
132 133 self.fileSize = None
133 134
134 135 self.firstHeaderSize = 0
135 136
136 137 self.basicHeaderSize = 24
137 138
138 139 self.pathList = []
139 140
140 141 self.lastUTTime = 0
141 142
142 143 self.maxTimeStep = 30
143 144
144 145 self.flagNoMoreFiles = 0
145 146
146 147 self.set = 0
147 148
148 149 self.path = None
149 150
150 151 self.delay = 60 #seconds
151 152
152 153 self.nTries = 3 #quantity tries
153 154
154 155 self.nFiles = 3 #number of files for searching
155 156
156 157 self.nReadBlocks = 0
157 158
158 159 self.flagIsNewFile = 1
159 160
160 161 self.__isFirstTimeOnline = 1
161 162
162 163 # self.ippSeconds = 0
163 164
164 165 self.flagDiscontinuousBlock = 0
165 166
166 167 self.flagIsNewBlock = 0
167 168
168 169 self.nTotalBlocks = 0
169 170
170 171 self.blocksize = 0
171 172
172 173 self.dataOut = self.createObjByDefault()
173 174
174 175 self.profileIndex = 1 #Always
175 176
176 177
177 178 def createObjByDefault(self):
178 179
179 180 dataObj = Spectra()
180 181
181 182 return dataObj
182 183
183 184 def __hasNotDataInBuffer(self):
184 185 return 1
185 186
186 187
187 188 def getBlockDimension(self):
188 189 """
189 190 Obtiene la cantidad de puntos a leer por cada bloque de datos
190 191
191 192 Affected:
192 193 self.nRdChannels
193 194 self.nRdPairs
194 195 self.pts2read_SelfSpectra
195 196 self.pts2read_CrossSpectra
196 197 self.pts2read_DCchannels
197 198 self.blocksize
198 199 self.dataOut.nChannels
199 200 self.dataOut.nPairs
200 201
201 202 Return:
202 203 None
203 204 """
204 205 self.nRdChannels = 0
205 206 self.nRdPairs = 0
206 207 self.rdPairList = []
207 208
208 209 for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
209 210 if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
210 211 self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
211 212 else:
212 213 self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
213 214 self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
214 215
215 216 pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
216 217
217 218 self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
218 219 self.blocksize = self.pts2read_SelfSpectra
219 220
220 221 if self.processingHeaderObj.flag_cspc:
221 222 self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
222 223 self.blocksize += self.pts2read_CrossSpectra
223 224
224 225 if self.processingHeaderObj.flag_dc:
225 226 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
226 227 self.blocksize += self.pts2read_DCchannels
227 228
228 229 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
229 230
230 231
231 232 def readBlock(self):
232 233 """
233 234 Lee el bloque de datos desde la posicion actual del puntero del archivo
234 235 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
235 236 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
236 237 es seteado a 0
237 238
238 239 Return: None
239 240
240 241 Variables afectadas:
241 242
242 243 self.flagIsNewFile
243 244 self.flagIsNewBlock
244 245 self.nTotalBlocks
245 246 self.data_spc
246 247 self.data_cspc
247 248 self.data_dc
248 249
249 250 Exceptions:
250 251 Si un bloque leido no es un bloque valido
251 252 """
252 253 blockOk_flag = False
253 254 fpointer = self.fp.tell()
254 255
255 256 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
256 257 spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
257 258
258 259 if self.processingHeaderObj.flag_cspc:
259 260 cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
260 261 cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
261 262
262 263 if self.processingHeaderObj.flag_dc:
263 264 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
264 265 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
265 266
266 267
267 268 if not self.processingHeaderObj.shif_fft:
268 269 #desplaza a la derecha en el eje 2 determinadas posiciones
269 270 shift = int(self.processingHeaderObj.profilesPerBlock/2)
270 271 spc = numpy.roll( spc, shift , axis=2 )
271 272
272 273 if self.processingHeaderObj.flag_cspc:
273 274 #desplaza a la derecha en el eje 2 determinadas posiciones
274 275 cspc = numpy.roll( cspc, shift, axis=2 )
275 276
276 277 #Dimensions : nChannels, nProfiles, nSamples
277 278 spc = numpy.transpose( spc, (0,2,1) )
278 279 self.data_spc = spc
279 280
280 281 if self.processingHeaderObj.flag_cspc:
281 282 cspc = numpy.transpose( cspc, (0,2,1) )
282 283 self.data_cspc = cspc['real'] + cspc['imag']*1j
283 284 else:
284 285 self.data_cspc = None
285 286
286 287 if self.processingHeaderObj.flag_dc:
287 288 self.data_dc = dc['real'] + dc['imag']*1j
288 289 else:
289 290 self.data_dc = None
290 291
291 292 self.flagIsNewFile = 0
292 293 self.flagIsNewBlock = 1
293 294
294 295 self.nTotalBlocks += 1
295 296 self.nReadBlocks += 1
296 297
297 298 return 1
298 299
299 300 def getFirstHeader(self):
300 301
301 302 self.getBasicHeader()
302 303
303 304 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
304 305
305 306 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
306 307
307 308 # self.dataOut.ippSeconds = self.ippSeconds
308 309
309 310 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.processingHeaderObj.profilesPerBlock
310 311
311 312 self.dataOut.dtype = self.dtype
312 313
313 314 # self.dataOut.nPairs = self.nPairs
314 315
315 316 self.dataOut.pairsList = self.rdPairList
316 317
317 318 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
318 319
319 320 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
320 321
321 322 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
322 323
323 324 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
324 325
325 326 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
326 327
327 328 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
328 329
329 330 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
330 331
331 332 self.dataOut.flagShiftFFT = True #Data is always shifted
332 333
333 334 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
334 335
335 336 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
336 337
337 338 def getData(self):
338 339 """
339 340 First method to execute before "RUN" is called.
340 341
341 342 Copia el buffer de lectura a la clase "Spectra",
342 343 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
343 344 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
344 345
345 346 Return:
346 347 0 : Si no hay mas archivos disponibles
347 348 1 : Si hizo una buena copia del buffer
348 349
349 350 Affected:
350 351 self.dataOut
351 352
352 353 self.flagDiscontinuousBlock
353 354 self.flagIsNewBlock
354 355 """
355 356
356 357 if self.flagNoMoreFiles:
357 358 self.dataOut.flagNoData = True
358 359 print('Process finished')
359 360 return 0
360 361
361 362 self.flagDiscontinuousBlock = 0
362 363 self.flagIsNewBlock = 0
363 364
364 365 if self.__hasNotDataInBuffer():
365 366
366 367 if not( self.readNextBlock() ):
367 368 self.dataOut.flagNoData = True
368 369 return 0
369 370
370 371 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
371 372
372 373 if self.data_spc is None:
373 374 self.dataOut.flagNoData = True
374 375 return 0
375 376
376 377 self.getBasicHeader()
377 378
378 379 self.getFirstHeader()
379 380
380 381 self.dataOut.data_spc = self.data_spc
381 382
382 383 self.dataOut.data_cspc = self.data_cspc
383 384
384 385 self.dataOut.data_dc = self.data_dc
385 386
386 387 self.dataOut.flagNoData = False
387 388
388 389 self.dataOut.realtime = self.online
389 390
390 391 return self.dataOut.data_spc
391
392 @MPDecorator
392 393 class SpectraWriter(JRODataWriter, Operation):
393 394
394 395 """
395 396 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
396 397 de los datos siempre se realiza por bloques.
397 398 """
398 399
399 400 ext = ".pdata"
400 401
401 402 optchar = "P"
402 403
403 404 shape_spc_Buffer = None
404 405
405 406 shape_cspc_Buffer = None
406 407
407 408 shape_dc_Buffer = None
408 409
409 410 data_spc = None
410 411
411 412 data_cspc = None
412 413
413 414 data_dc = None
414 415
415 416 # dataOut = None
416 417
417 def __init__(self, **kwargs):
418 def __init__(self):#, **kwargs):
418 419 """
419 420 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
420 421
421 422 Affected:
422 423 self.dataOut
423 424 self.basicHeaderObj
424 425 self.systemHeaderObj
425 426 self.radarControllerHeaderObj
426 427 self.processingHeaderObj
427 428
428 429 Return: None
429 430 """
430 431
431 Operation.__init__(self, **kwargs)
432 Operation.__init__(self)#, **kwargs)
432 433
433 self.isConfig = False
434 #self.isConfig = False
434 435
435 436 self.nTotalBlocks = 0
436 437
437 438 self.data_spc = None
438 439
439 440 self.data_cspc = None
440 441
441 442 self.data_dc = None
442 443
443 444 self.fp = None
444 445
445 446 self.flagIsNewFile = 1
446 447
447 448 self.nTotalBlocks = 0
448 449
449 450 self.flagIsNewBlock = 0
450 451
451 452 self.setFile = None
452 453
453 454 self.dtype = None
454 455
455 456 self.path = None
456 457
457 458 self.noMoreFiles = 0
458 459
459 460 self.filename = None
460 461
461 462 self.basicHeaderObj = BasicHeader(LOCALTIME)
462 463
463 464 self.systemHeaderObj = SystemHeader()
464 465
465 466 self.radarControllerHeaderObj = RadarControllerHeader()
466 467
467 468 self.processingHeaderObj = ProcessingHeader()
468 469
469 470
470 471 def hasAllDataInBuffer(self):
471 472 return 1
472 473
473 474
474 475 def setBlockDimension(self):
475 476 """
476 477 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
477 478
478 479 Affected:
479 480 self.shape_spc_Buffer
480 481 self.shape_cspc_Buffer
481 482 self.shape_dc_Buffer
482 483
483 484 Return: None
484 485 """
485 486 self.shape_spc_Buffer = (self.dataOut.nChannels,
486 487 self.processingHeaderObj.nHeights,
487 488 self.processingHeaderObj.profilesPerBlock)
488 489
489 490 self.shape_cspc_Buffer = (self.dataOut.nPairs,
490 491 self.processingHeaderObj.nHeights,
491 492 self.processingHeaderObj.profilesPerBlock)
492 493
493 494 self.shape_dc_Buffer = (self.dataOut.nChannels,
494 495 self.processingHeaderObj.nHeights)
495 496
496 497
497 498 def writeBlock(self):
498 499 """
499 500 Escribe el buffer en el file designado
500 501
501 502 Affected:
502 503 self.data_spc
503 504 self.data_cspc
504 505 self.data_dc
505 506 self.flagIsNewFile
506 507 self.flagIsNewBlock
507 508 self.nTotalBlocks
508 509 self.nWriteBlocks
509 510
510 511 Return: None
511 512 """
512 513
513 514 spc = numpy.transpose( self.data_spc, (0,2,1) )
514 515 if not self.processingHeaderObj.shif_fft:
515 516 spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
516 517 data = spc.reshape((-1))
517 518 data = data.astype(self.dtype[0])
518 519 data.tofile(self.fp)
519 520
520 521 if self.data_cspc is not None:
521 522 data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
522 523 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
523 524 if not self.processingHeaderObj.shif_fft:
524 525 cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
525 526 data['real'] = cspc.real
526 527 data['imag'] = cspc.imag
527 528 data = data.reshape((-1))
528 529 data.tofile(self.fp)
529 530
530 531 if self.data_dc is not None:
531 532 data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
532 533 dc = self.data_dc
533 534 data['real'] = dc.real
534 535 data['imag'] = dc.imag
535 536 data = data.reshape((-1))
536 537 data.tofile(self.fp)
537 538
538 539 # self.data_spc.fill(0)
539 540 #
540 541 # if self.data_dc is not None:
541 542 # self.data_dc.fill(0)
542 543 #
543 544 # if self.data_cspc is not None:
544 545 # self.data_cspc.fill(0)
545 546
546 547 self.flagIsNewFile = 0
547 548 self.flagIsNewBlock = 1
548 549 self.nTotalBlocks += 1
549 550 self.nWriteBlocks += 1
550 551 self.blockIndex += 1
551 552
552 553 # print "[Writing] Block = %d04" %self.blockIndex
553 554
554 555 def putData(self):
555 556 """
556 557 Setea un bloque de datos y luego los escribe en un file
557 558
558 559 Affected:
559 560 self.data_spc
560 561 self.data_cspc
561 562 self.data_dc
562 563
563 564 Return:
564 565 0 : Si no hay data o no hay mas files que puedan escribirse
565 566 1 : Si se escribio la data de un bloque en un file
566 567 """
567 568
568 569 if self.dataOut.flagNoData:
569 570 return 0
570 571
571 572 self.flagIsNewBlock = 0
572 573
573 574 if self.dataOut.flagDiscontinuousBlock:
574 575 self.data_spc.fill(0)
575 576 if self.dataOut.data_cspc is not None:
576 577 self.data_cspc.fill(0)
577 578 if self.dataOut.data_dc is not None:
578 579 self.data_dc.fill(0)
579 580 self.setNextFile()
580 581
581 582 if self.flagIsNewFile == 0:
582 583 self.setBasicHeader()
583 584
584 585 self.data_spc = self.dataOut.data_spc.copy()
585 586
586 587 if self.dataOut.data_cspc is not None:
587 588 self.data_cspc = self.dataOut.data_cspc.copy()
588 589
589 590 if self.dataOut.data_dc is not None:
590 591 self.data_dc = self.dataOut.data_dc.copy()
591 592
592 593 # #self.processingHeaderObj.dataBlocksPerFile)
593 594 if self.hasAllDataInBuffer():
594 595 # self.setFirstHeader()
595 596 self.writeNextBlock()
596 597
597 return 1
598
599 598 def __getBlockSize(self):
600 599 '''
601 600 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
602 601 '''
603 602
604 603 dtype_width = self.getDtypeWidth()
605 604
606 605 pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
607 606
608 607 pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
609 608 blocksize = (pts2write_SelfSpectra*dtype_width)
610 609
611 610 if self.dataOut.data_cspc is not None:
612 611 pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
613 612 blocksize += (pts2write_CrossSpectra*dtype_width*2)
614 613
615 614 if self.dataOut.data_dc is not None:
616 615 pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
617 616 blocksize += (pts2write_DCchannels*dtype_width*2)
618 617
619 618 # blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
620 619
621 620 return blocksize
622 621
623 622 def setFirstHeader(self):
624 623
625 624 """
626 625 Obtiene una copia del First Header
627 626
628 627 Affected:
629 628 self.systemHeaderObj
630 629 self.radarControllerHeaderObj
631 630 self.dtype
632 631
633 632 Return:
634 633 None
635 634 """
636 635
637 636 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
638 637 self.systemHeaderObj.nChannels = self.dataOut.nChannels
639 638 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
640 639
641 640 self.processingHeaderObj.dtype = 1 # Spectra
642 641 self.processingHeaderObj.blockSize = self.__getBlockSize()
643 642 self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
644 643 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
645 644 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
646 645 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
647 646 self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
648 647 self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
649 648 self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
650 649
651 650 if self.processingHeaderObj.totalSpectra > 0:
652 651 channelList = []
653 652 for channel in range(self.dataOut.nChannels):
654 653 channelList.append(channel)
655 654 channelList.append(channel)
656 655
657 656 pairsList = []
658 657 if self.dataOut.nPairs > 0:
659 658 for pair in self.dataOut.pairsList:
660 659 pairsList.append(pair[0])
661 660 pairsList.append(pair[1])
662 661
663 662 spectraComb = channelList + pairsList
664 663 spectraComb = numpy.array(spectraComb, dtype="u1")
665 664 self.processingHeaderObj.spectraComb = spectraComb
666 665
667 666 if self.dataOut.code is not None:
668 667 self.processingHeaderObj.code = self.dataOut.code
669 668 self.processingHeaderObj.nCode = self.dataOut.nCode
670 669 self.processingHeaderObj.nBaud = self.dataOut.nBaud
671 670
672 671 if self.processingHeaderObj.nWindows != 0:
673 672 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
674 673 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
675 674 self.processingHeaderObj.nHeights = self.dataOut.nHeights
676 675 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
677 676
678 677 self.processingHeaderObj.processFlags = self.getProcessFlags()
679 678
680 679 self.setBasicHeader() No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,765 +1,765
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6
7 7 import numpy
8 8
9 9 from .jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 12 from schainpy.model.data.jrodata import Voltage
13 13 import zmq
14 14 import tempfile
15 15 from io import StringIO
16 16 # from _sha import blocksize
17 17
18 18 @MPDecorator
19 19 class VoltageReader(JRODataReader, ProcessingUnit):
20 20 """
21 21 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
22 22 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
23 23 perfiles*alturas*canales) son almacenados en la variable "buffer".
24 24
25 25 perfiles * alturas * canales
26 26
27 27 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
28 28 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
29 29 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
30 30 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
31 31
32 32 Example:
33 33
34 34 dpath = "/home/myuser/data"
35 35
36 36 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
37 37
38 38 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
39 39
40 40 readerObj = VoltageReader()
41 41
42 42 readerObj.setup(dpath, startTime, endTime)
43 43
44 44 while(True):
45 45
46 46 #to get one profile
47 47 profile = readerObj.getData()
48 48
49 49 #print the profile
50 50 print profile
51 51
52 52 #If you want to see all datablock
53 53 print readerObj.datablock
54 54
55 55 if readerObj.flagNoMoreFiles:
56 56 break
57 57
58 58 """
59 59
60 60 ext = ".r"
61 61
62 62 optchar = "D"
63 63 dataOut = None
64 64
65 65 def __init__(self):#, **kwargs):
66 66 """
67 67 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
68 68
69 69 Input:
70 70 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
71 71 almacenar un perfil de datos cada vez que se haga un requerimiento
72 72 (getData). El perfil sera obtenido a partir del buffer de datos,
73 73 si el buffer esta vacio se hara un nuevo proceso de lectura de un
74 74 bloque de datos.
75 75 Si este parametro no es pasado se creara uno internamente.
76 76
77 77 Variables afectadas:
78 78 self.dataOut
79 79
80 80 Return:
81 81 None
82 82 """
83 83
84 84 ProcessingUnit.__init__(self)#, **kwargs)
85 85
86 86 self.isConfig = False
87 87
88 88 self.datablock = None
89 89
90 90 self.utc = 0
91 91
92 92 self.ext = ".r"
93 93
94 94 self.optchar = "D"
95 95
96 96 self.basicHeaderObj = BasicHeader(LOCALTIME)
97 97
98 98 self.systemHeaderObj = SystemHeader()
99 99
100 100 self.radarControllerHeaderObj = RadarControllerHeader()
101 101
102 102 self.processingHeaderObj = ProcessingHeader()
103 103
104 104 self.online = 0
105 105
106 106 self.fp = None
107 107
108 108 self.idFile = None
109 109
110 110 self.dtype = None
111 111
112 112 self.fileSizeByHeader = None
113 113
114 114 self.filenameList = []
115 115
116 116 self.filename = None
117 117
118 118 self.fileSize = None
119 119
120 120 self.firstHeaderSize = 0
121 121
122 122 self.basicHeaderSize = 24
123 123
124 124 self.pathList = []
125 125
126 126 self.filenameList = []
127 127
128 128 self.lastUTTime = 0
129 129
130 130 self.maxTimeStep = 30
131 131
132 132 self.flagNoMoreFiles = 0
133 133
134 134 self.set = 0
135 135
136 136 self.path = None
137 137
138 138 self.profileIndex = 2**32 - 1
139 139
140 140 self.delay = 3 # seconds
141 141
142 142 self.nTries = 3 # quantity tries
143 143
144 144 self.nFiles = 3 # number of files for searching
145 145
146 146 self.nReadBlocks = 0
147 147
148 148 self.flagIsNewFile = 1
149 149
150 150 self.__isFirstTimeOnline = 1
151 151
152 152 # self.ippSeconds = 0
153 153
154 154 self.flagDiscontinuousBlock = 0
155 155
156 156 self.flagIsNewBlock = 0
157 157
158 158 self.nTotalBlocks = 0
159 159
160 160 self.blocksize = 0
161 161
162 162 self.dataOut = self.createObjByDefault()
163 163
164 164 self.nTxs = 1
165 165
166 166 self.txIndex = 0
167 167
168 168 def createObjByDefault(self):
169 169
170 170 dataObj = Voltage()
171 171
172 172 return dataObj
173 173
174 174 def __hasNotDataInBuffer(self):
175 175
176 176 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock * self.nTxs:
177 177 return 1
178 178
179 179 return 0
180 180
181 181 def getBlockDimension(self):
182 182 """
183 183 Obtiene la cantidad de puntos a leer por cada bloque de datos
184 184
185 185 Affected:
186 186 self.blocksize
187 187
188 188 Return:
189 189 None
190 190 """
191 191 pts2read = self.processingHeaderObj.profilesPerBlock * \
192 192 self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
193 193 self.blocksize = pts2read
194 194
195 195 def readBlock(self):
196 196 """
197 197 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
198 198 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
199 199 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
200 200 es seteado a 0
201 201
202 202 Inputs:
203 203 None
204 204
205 205 Return:
206 206 None
207 207
208 208 Affected:
209 209 self.profileIndex
210 210 self.datablock
211 211 self.flagIsNewFile
212 212 self.flagIsNewBlock
213 213 self.nTotalBlocks
214 214
215 215 Exceptions:
216 216 Si un bloque leido no es un bloque valido
217 217 """
218 218
219 219 # if self.server is not None:
220 220 # self.zBlock = self.receiver.recv()
221 221 # self.zHeader = self.zBlock[:24]
222 222 # self.zDataBlock = self.zBlock[24:]
223 223 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
224 224 # self.processingHeaderObj.profilesPerBlock = 240
225 225 # self.processingHeaderObj.nHeights = 248
226 226 # self.systemHeaderObj.nChannels
227 227 # else:
228 228 current_pointer_location = self.fp.tell()
229 229 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
230 230
231 231 try:
232 232 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
233 233 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
234 234 except:
235 235 # print "The read block (%3d) has not enough data" %self.nReadBlocks
236 236
237 237 if self.waitDataBlock(pointer_location=current_pointer_location):
238 238 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
239 239 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
240 240 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
241 241 # return 0
242 242
243 243 # Dimensions : nChannels, nProfiles, nSamples
244 244
245 245 junk = numpy.transpose(junk, (2, 0, 1))
246 246 self.datablock = junk['real'] + junk['imag'] * 1j
247 247
248 248 self.profileIndex = 0
249 249
250 250 self.flagIsNewFile = 0
251 251 self.flagIsNewBlock = 1
252 252
253 253 self.nTotalBlocks += 1
254 254 self.nReadBlocks += 1
255 255
256 256 return 1
257 257
258 258 def getFirstHeader(self):
259 259
260 260 self.getBasicHeader()
261 261
262 262 self.dataOut.processingHeaderObj = self.processingHeaderObj.copy()
263 263
264 264 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
265 265
266 266 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
267 267
268 268 if self.nTxs > 1:
269 269 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
270 270 # Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
271 271
272 272 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
273 273 #
274 274 # if self.radarControllerHeaderObj.code is not None:
275 275 #
276 276 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
277 277 #
278 278 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
279 279 #
280 280 # self.dataOut.code = self.radarControllerHeaderObj.code
281 281
282 282 self.dataOut.dtype = self.dtype
283 283
284 284 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
285 285
286 286 self.dataOut.heightList = numpy.arange(
287 287 self.processingHeaderObj.nHeights) * self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
288 288
289 289 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
290 290
291 291 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
292 292
293 293 # asumo q la data no esta decodificada
294 294 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode
295 295
296 296 # asumo q la data no esta sin flip
297 297 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip
298 298
299 299 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
300 300
301 301 def reshapeData(self):
302 302
303 303 if self.nTxs < 0:
304 304 return
305 305
306 306 if self.nTxs == 1:
307 307 return
308 308
309 309 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1. / self.nTxs) != 0:
310 310 raise ValueError("1./nTxs (=%f), should be a multiple of nProfiles (=%d)" % (
311 311 1. / self.nTxs, self.processingHeaderObj.profilesPerBlock))
312 312
313 313 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
314 314 raise ValueError("nTxs (=%d), should be a multiple of nHeights (=%d)" % (
315 315 self.nTxs, self.processingHeaderObj.nHeights))
316 316
317 317 self.datablock = self.datablock.reshape(
318 318 (self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock * self.nTxs, self.processingHeaderObj.nHeights / self.nTxs))
319 319
320 320 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock * self.nTxs
321 321 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights / self.nTxs) * \
322 322 self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
323 323 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
324 324
325 325 return
326 326
327 327 def readFirstHeaderFromServer(self):
328 328
329 329 self.getFirstHeader()
330 330
331 331 self.firstHeaderSize = self.basicHeaderObj.size
332 332
333 333 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
334 334 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
335 335 if datatype == 0:
336 336 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
337 337 elif datatype == 1:
338 338 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
339 339 elif datatype == 2:
340 340 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
341 341 elif datatype == 3:
342 342 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
343 343 elif datatype == 4:
344 344 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
345 345 elif datatype == 5:
346 346 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
347 347 else:
348 348 raise ValueError('Data type was not defined')
349 349
350 350 self.dtype = datatype_str
351 351 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
352 352 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
353 353 self.firstHeaderSize + self.basicHeaderSize * \
354 354 (self.processingHeaderObj.dataBlocksPerFile - 1)
355 355 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
356 356 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
357 357 self.getBlockDimension()
358 358
359 359 def getFromServer(self):
360 360 self.flagDiscontinuousBlock = 0
361 361 self.profileIndex = 0
362 362 self.flagIsNewBlock = 1
363 363 self.dataOut.flagNoData = False
364 364 self.nTotalBlocks += 1
365 365 self.nReadBlocks += 1
366 366 self.blockPointer = 0
367 367
368 368 block = self.receiver.recv()
369 369
370 370 self.basicHeaderObj.read(block[self.blockPointer:])
371 371 self.blockPointer += self.basicHeaderObj.length
372 372 self.systemHeaderObj.read(block[self.blockPointer:])
373 373 self.blockPointer += self.systemHeaderObj.length
374 374 self.radarControllerHeaderObj.read(block[self.blockPointer:])
375 375 self.blockPointer += self.radarControllerHeaderObj.length
376 376 self.processingHeaderObj.read(block[self.blockPointer:])
377 377 self.blockPointer += self.processingHeaderObj.length
378 378 self.readFirstHeaderFromServer()
379 379
380 380 timestamp = self.basicHeaderObj.get_datatime()
381 381 print('[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp))
382 382 current_pointer_location = self.blockPointer
383 383 junk = numpy.fromstring(
384 384 block[self.blockPointer:], self.dtype, self.blocksize)
385 385
386 386 try:
387 387 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
388 388 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
389 389 except:
390 390 # print "The read block (%3d) has not enough data" %self.nReadBlocks
391 391 if self.waitDataBlock(pointer_location=current_pointer_location):
392 392 junk = numpy.fromstring(
393 393 block[self.blockPointer:], self.dtype, self.blocksize)
394 394 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
395 395 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
396 396 # return 0
397 397
398 398 # Dimensions : nChannels, nProfiles, nSamples
399 399
400 400 junk = numpy.transpose(junk, (2, 0, 1))
401 401 self.datablock = junk['real'] + junk['imag'] * 1j
402 402 self.profileIndex = 0
403 403 if self.selBlocksize == None:
404 404 self.selBlocksize = self.dataOut.nProfiles
405 405 if self.selBlocktime != None:
406 406 if self.dataOut.nCohInt is not None:
407 407 nCohInt = self.dataOut.nCohInt
408 408 else:
409 409 nCohInt = 1
410 410 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
411 411 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
412 412 self.dataOut.data = self.datablock[:,
413 413 self.profileIndex:self.profileIndex + self.selBlocksize, :]
414 414 datasize = self.dataOut.data.shape[1]
415 415 if datasize < self.selBlocksize:
416 416 buffer = numpy.zeros(
417 417 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
418 418 buffer[:, :datasize, :] = self.dataOut.data
419 419 self.dataOut.data = buffer
420 420 self.profileIndex = blockIndex
421 421
422 422 self.dataOut.flagDataAsBlock = True
423 423 self.flagIsNewBlock = 1
424 424 self.dataOut.realtime = self.online
425 425
426 426 return self.dataOut.data
427 427
428 428 def getData(self):
429 429 """
430 430 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
431 431 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
432 432 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
433 433 "readNextBlock"
434 434
435 435 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
436 436
437 437 Return:
438 438
439 439 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
440 440 es igual al total de perfiles leidos desde el archivo.
441 441
442 442 Si self.getByBlock == False:
443 443
444 444 self.dataOut.data = buffer[:, thisProfile, :]
445 445
446 446 shape = [nChannels, nHeis]
447 447
448 448 Si self.getByBlock == True:
449 449
450 450 self.dataOut.data = buffer[:, :, :]
451 451
452 452 shape = [nChannels, nProfiles, nHeis]
453 453
454 454 Variables afectadas:
455 455 self.dataOut
456 456 self.profileIndex
457 457
458 458 Affected:
459 459 self.dataOut
460 460 self.profileIndex
461 461 self.flagDiscontinuousBlock
462 462 self.flagIsNewBlock
463 463 """
464 464 if self.flagNoMoreFiles:
465 465 self.dataOut.flagNoData = True
466 466 print('Process finished')
467 467 return 0
468 468 self.flagDiscontinuousBlock = 0
469 469 self.flagIsNewBlock = 0
470 470 if self.__hasNotDataInBuffer():
471 471 if not(self.readNextBlock()):
472 472 return 0
473 473
474 474 self.getFirstHeader()
475 475
476 476 self.reshapeData()
477 477 if self.datablock is None:
478 478 self.dataOut.flagNoData = True
479 479 return 0
480 480
481 481 if not self.getByBlock:
482 482
483 483 """
484 484 Return profile by profile
485 485
486 486 If nTxs > 1 then one profile is divided by nTxs and number of total
487 487 blocks is increased by nTxs (nProfiles *= nTxs)
488 488 """
489 489 self.dataOut.flagDataAsBlock = False
490 490 self.dataOut.data = self.datablock[:, self.profileIndex, :]
491 491 self.dataOut.profileIndex = self.profileIndex
492 492
493 493 self.profileIndex += 1
494 494
495 495 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
496 496 # """
497 497 # Return all block
498 498 # """
499 499 # self.dataOut.flagDataAsBlock = True
500 500 # self.dataOut.data = self.datablock
501 501 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
502 502 #
503 503 # self.profileIndex = self.dataOut.nProfiles
504 504
505 505 else:
506 506 """
507 507 Return a block
508 508 """
509 509 if self.selBlocksize == None:
510 510 self.selBlocksize = self.dataOut.nProfiles
511 511 if self.selBlocktime != None:
512 512 if self.dataOut.nCohInt is not None:
513 513 nCohInt = self.dataOut.nCohInt
514 514 else:
515 515 nCohInt = 1
516 516 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
517 517 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
518 518
519 519 self.dataOut.data = self.datablock[:,
520 520 self.profileIndex:self.profileIndex + self.selBlocksize, :]
521 521 self.profileIndex += self.selBlocksize
522 522 datasize = self.dataOut.data.shape[1]
523 523
524 524 if datasize < self.selBlocksize:
525 525 buffer = numpy.zeros(
526 526 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
527 527 buffer[:, :datasize, :] = self.dataOut.data
528 528
529 529 while datasize < self.selBlocksize: # Not enough profiles to fill the block
530 530 if not(self.readNextBlock()):
531 531 return 0
532 532 self.getFirstHeader()
533 533 self.reshapeData()
534 534 if self.datablock is None:
535 535 self.dataOut.flagNoData = True
536 536 return 0
537 537 # stack data
538 538 blockIndex = self.selBlocksize - datasize
539 539 datablock1 = self.datablock[:, :blockIndex, :]
540 540
541 541 buffer[:, datasize:datasize +
542 542 datablock1.shape[1], :] = datablock1
543 543 datasize += datablock1.shape[1]
544 544
545 545 self.dataOut.data = buffer
546 546 self.profileIndex = blockIndex
547 547
548 548 self.dataOut.flagDataAsBlock = True
549 549 self.dataOut.nProfiles = self.dataOut.data.shape[1]
550 550
551 551 self.dataOut.flagNoData = False
552 552
553 553 self.getBasicHeader()
554 554
555 555 self.dataOut.realtime = self.online
556 556
557 557 return self.dataOut.data
558 558
559
559 @MPDecorator
560 560 class VoltageWriter(JRODataWriter, Operation):
561 561 """
562 562 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
563 563 de los datos siempre se realiza por bloques.
564 564 """
565 565
566 566 ext = ".r"
567 567
568 568 optchar = "D"
569 569
570 570 shapeBuffer = None
571 571
572 def __init__(self, **kwargs):
572 def __init__(self):#, **kwargs):
573 573 """
574 574 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
575 575
576 576 Affected:
577 577 self.dataOut
578 578
579 579 Return: None
580 580 """
581 Operation.__init__(self, **kwargs)
581 Operation.__init__(self)#, **kwargs)
582 582
583 583 self.nTotalBlocks = 0
584 584
585 585 self.profileIndex = 0
586 586
587 587 self.isConfig = False
588 588
589 589 self.fp = None
590 590
591 591 self.flagIsNewFile = 1
592 592
593 593 self.blockIndex = 0
594 594
595 595 self.flagIsNewBlock = 0
596 596
597 597 self.setFile = None
598 598
599 599 self.dtype = None
600 600
601 601 self.path = None
602 602
603 603 self.filename = None
604 604
605 605 self.basicHeaderObj = BasicHeader(LOCALTIME)
606 606
607 607 self.systemHeaderObj = SystemHeader()
608 608
609 609 self.radarControllerHeaderObj = RadarControllerHeader()
610 610
611 611 self.processingHeaderObj = ProcessingHeader()
612 612
613 613 def hasAllDataInBuffer(self):
614 614 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
615 615 return 1
616 616 return 0
617 617
618 618 def setBlockDimension(self):
619 619 """
620 620 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
621 621
622 622 Affected:
623 623 self.shape_spc_Buffer
624 624 self.shape_cspc_Buffer
625 625 self.shape_dc_Buffer
626 626
627 627 Return: None
628 628 """
629 629 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
630 630 self.processingHeaderObj.nHeights,
631 631 self.systemHeaderObj.nChannels)
632 632
633 633 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
634 634 self.processingHeaderObj.profilesPerBlock,
635 635 self.processingHeaderObj.nHeights),
636 636 dtype=numpy.dtype('complex64'))
637 637
638 638 def writeBlock(self):
639 639 """
640 640 Escribe el buffer en el file designado
641 641
642 642 Affected:
643 643 self.profileIndex
644 644 self.flagIsNewFile
645 645 self.flagIsNewBlock
646 646 self.nTotalBlocks
647 647 self.blockIndex
648 648
649 649 Return: None
650 650 """
651 651 data = numpy.zeros(self.shapeBuffer, self.dtype)
652 652
653 653 junk = numpy.transpose(self.datablock, (1, 2, 0))
654 654
655 655 data['real'] = junk.real
656 656 data['imag'] = junk.imag
657 657
658 658 data = data.reshape((-1))
659 659
660 660 data.tofile(self.fp)
661 661
662 662 self.datablock.fill(0)
663 663
664 664 self.profileIndex = 0
665 665 self.flagIsNewFile = 0
666 666 self.flagIsNewBlock = 1
667 667
668 668 self.blockIndex += 1
669 669 self.nTotalBlocks += 1
670 670
671 671 # print "[Writing] Block = %04d" %self.blockIndex
672 672
673 673 def putData(self):
674 674 """
675 675 Setea un bloque de datos y luego los escribe en un file
676 676
677 677 Affected:
678 678 self.flagIsNewBlock
679 679 self.profileIndex
680 680
681 681 Return:
682 682 0 : Si no hay data o no hay mas files que puedan escribirse
683 683 1 : Si se escribio la data de un bloque en un file
684 684 """
685 685 if self.dataOut.flagNoData:
686 686 return 0
687 687
688 688 self.flagIsNewBlock = 0
689 689
690 690 if self.dataOut.flagDiscontinuousBlock:
691 691 self.datablock.fill(0)
692 692 self.profileIndex = 0
693 693 self.setNextFile()
694 694
695 695 if self.profileIndex == 0:
696 696 self.setBasicHeader()
697 697
698 698 self.datablock[:, self.profileIndex, :] = self.dataOut.data
699 699
700 700 self.profileIndex += 1
701 701
702 702 if self.hasAllDataInBuffer():
703 703 # if self.flagIsNewFile:
704 704 self.writeNextBlock()
705 705 # self.setFirstHeader()
706 706
707 707 return 1
708 708
709 709 def __getBlockSize(self):
710 710 '''
711 711 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
712 712 '''
713 713
714 714 dtype_width = self.getDtypeWidth()
715 715
716 716 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels *
717 717 self.profilesPerBlock * dtype_width * 2)
718 718
719 719 return blocksize
720 720
721 721 def setFirstHeader(self):
722 722 """
723 723 Obtiene una copia del First Header
724 724
725 725 Affected:
726 726 self.systemHeaderObj
727 727 self.radarControllerHeaderObj
728 728 self.dtype
729 729
730 730 Return:
731 731 None
732 732 """
733 733
734 734 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
735 735 self.systemHeaderObj.nChannels = self.dataOut.nChannels
736 736 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
737 737
738 738 self.processingHeaderObj.dtype = 0 # Voltage
739 739 self.processingHeaderObj.blockSize = self.__getBlockSize()
740 740 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
741 741 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
742 742 # podria ser 1 o self.dataOut.processingHeaderObj.nWindows
743 743 self.processingHeaderObj.nWindows = 1
744 744 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
745 745 # Cuando la data de origen es de tipo Voltage
746 746 self.processingHeaderObj.nIncohInt = 1
747 747 # Cuando la data de origen es de tipo Voltage
748 748 self.processingHeaderObj.totalSpectra = 0
749 749
750 750 if self.dataOut.code is not None:
751 751 self.processingHeaderObj.code = self.dataOut.code
752 752 self.processingHeaderObj.nCode = self.dataOut.nCode
753 753 self.processingHeaderObj.nBaud = self.dataOut.nBaud
754 754
755 755 if self.processingHeaderObj.nWindows != 0:
756 756 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
757 757 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - \
758 758 self.dataOut.heightList[0]
759 759 self.processingHeaderObj.nHeights = self.dataOut.nHeights
760 760 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
761 761
762 762 self.processingHeaderObj.processFlags = self.getProcessFlags()
763 763
764 764 self.setBasicHeader()
765 765 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,367 +1,367
1 1 '''
2 2 Updated for multiprocessing
3 3 Author : Sergio Cortez
4 4 Jan 2018
5 5 Abstract:
6 6 Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
7 7 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
8 8 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
9 9
10 10 Based on:
11 11 $Author: murco $
12 12 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
13 13 '''
14 14 from platform import python_version
15 15 import inspect
16 16 import zmq
17 17 import time
18 18 import pickle
19 19 import os
20 20 from multiprocessing import Process
21 21 from schainpy.utils import log
22 22
23 23
24 24 class ProcessingUnit(object):
25 25
26 26 """
27 27 Update - Jan 2018 - MULTIPROCESSING
28 28 All the "call" methods present in the previous base were removed.
29 29 The majority of operations are independant processes, thus
30 30 the decorator is in charge of communicate the operation processes
31 31 with the proccessing unit via IPC.
32 32
33 33 The constructor does not receive any argument. The remaining methods
34 34 are related with the operations to execute.
35 35
36 36
37 37 """
38 38
39 39 METHODS = {}
40 40 dataIn = None
41 41 dataInList = []
42 42 id = None
43 43 inputId = None
44 44 dataOut = None
45 45 dictProcs = None
46 46 isConfig = False
47 47
48 48 def __init__(self):
49 49
50 50 self.dataIn = None
51 51 self.dataOut = None
52 52 self.isConfig = False
53 53 self.operations = []
54 54
55 55 def getAllowedArgs(self):
56 56 if hasattr(self, '__attrs__'):
57 57 return self.__attrs__
58 58 else:
59 59 return inspect.getargspec(self.run).args
60 60
61 61 def addOperation(self, conf, operation):
62 62
63 63 """
64 64 This method is used in the controller, and update the dictionary containing the operations to execute. The dict
65 65 posses the id of the operation process (IPC purposes)
66 66
67 67 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
68 68 identificador asociado a este objeto.
69 69
70 70 Input:
71 71
72 72 object : objeto de la clase "Operation"
73 73
74 74 Return:
75 75
76 76 objId : identificador del objeto, necesario para comunicar con master(procUnit)
77 77 """
78 78
79 79 self.operations.append((operation, conf.type, conf.id, conf.getKwargs()))
80 80
81 81 def getOperationObj(self, objId):
82 82
83 83 if objId not in list(self.operations.keys()):
84 84 return None
85 85
86 86 return self.operations[objId]
87 87
88 88 def operation(self, **kwargs):
89 89
90 90 """
91 91 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
92 92 atributos del objeto dataOut
93 93
94 94 Input:
95 95
96 96 **kwargs : Diccionario de argumentos de la funcion a ejecutar
97 97 """
98 98
99 99 raise NotImplementedError
100 100
101 101 def setup(self):
102 102
103 103 raise NotImplementedError
104 104
105 105 def run(self):
106 106
107 107 raise NotImplementedError
108 108
109 109 def close(self):
110 110 #Close every thread, queue or any other object here is it is neccesary.
111 111 return
112 112
113 113 class Operation(object):
114 114
115 115 """
116 116 Update - Jan 2018 - MULTIPROCESSING
117 117
118 118 Most of the methods remained the same. The decorator parse the arguments and executed the run() method for each process.
119 119 The constructor doe snot receive any argument, neither the baseclass.
120 120
121 121
122 122 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
123 123 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
124 124 acumulacion dentro de esta clase
125 125
126 126 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
127 127
128 128 """
129 129 id = None
130 130 __buffer = None
131 131 dest = None
132 132 isConfig = False
133 133 readyFlag = None
134 134
135 135 def __init__(self):
136 136
137 137 self.buffer = None
138 138 self.dest = None
139 139 self.isConfig = False
140 140 self.readyFlag = False
141 141
142 142 if not hasattr(self, 'name'):
143 143 self.name = self.__class__.__name__
144 144
145 145 def getAllowedArgs(self):
146 146 if hasattr(self, '__attrs__'):
147 147 return self.__attrs__
148 148 else:
149 149 return inspect.getargspec(self.run).args
150 150
151 151 def setup(self):
152 152
153 153 self.isConfig = True
154 154
155 155 raise NotImplementedError
156 156
157 157
158 158 def run(self, dataIn, **kwargs):
159 159
160 160 """
161 161 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
162 162 atributos del objeto dataIn.
163 163
164 164 Input:
165 165
166 166 dataIn : objeto del tipo JROData
167 167
168 168 Return:
169 169
170 170 None
171 171
172 172 Affected:
173 173 __buffer : buffer de recepcion de datos.
174 174
175 175 """
176 176 if not self.isConfig:
177 177 self.setup(**kwargs)
178 178
179 179 raise NotImplementedError
180 180
181 181 def close(self):
182 182
183 183 pass
184 184
185 185
186 186 def MPDecorator(BaseClass):
187 187
188 188 """
189 189 Multiprocessing class decorator
190 190
191 191 This function add multiprocessing features to a BaseClass. Also, it handle
192 192 the communication beetween processes (readers, procUnits and operations).
193 193 """
194 194
195 195 class MPClass(BaseClass, Process):
196 196
197 197 def __init__(self, *args, **kwargs):
198 198 super(MPClass, self).__init__()
199 199 Process.__init__(self)
200 200 self.operationKwargs = {}
201 201 self.args = args
202 202 self.kwargs = kwargs
203 203 self.sender = None
204 204 self.receiver = None
205 205 self.name = BaseClass.__name__
206 206
207 207 if len(self.args) is 3:
208 208 self.typeProc = "ProcUnit"
209 209 self.id = args[0]
210 210 self.inputId = args[1]
211 211 self.project_id = args[2]
212 212 else:
213 213 self.id = args[0]
214 214 self.inputId = args[0]
215 215 self.project_id = args[1]
216 216 self.typeProc = "Operation"
217 217
218 218 def getAllowedArgs(self):
219 219
220 220 if hasattr(self, '__attrs__'):
221 221 return self.__attrs__
222 222 else:
223 223 return inspect.getargspec(BaseClass.run).args
224 224
225 225 def subscribe(self):
226 226 '''
227 227 This function create a socket to receive objects from the
228 228 topic `inputId`.
229 229 '''
230 230
231 231 c = zmq.Context()
232 232 self.receiver = c.socket(zmq.SUB)
233 233 self.receiver.connect('ipc:///tmp/schain/{}_pub'.format(self.project_id))
234 234 self.receiver.setsockopt(zmq.SUBSCRIBE, self.inputId.encode())
235 235
236 236 def listen(self):
237 237 '''
238 238 This function waits for objects and deserialize using pickle
239 239 '''
240 240
241 241 data = pickle.loads(self.receiver.recv_multipart()[1])
242 242 return data
243 243
244 244 def set_publisher(self):
245 245 '''
246 246 This function create a socket for publishing purposes.
247 247 '''
248 248
249 249 time.sleep(1)
250 250 c = zmq.Context()
251 251 self.sender = c.socket(zmq.PUB)
252 252 self.sender.connect('ipc:///tmp/schain/{}_sub'.format(self.project_id))
253 253
254 254 def publish(self, data, id):
255 255 '''
256 256 This function publish an object, to a specific topic.
257 257 '''
258 258
259 259 self.sender.send_multipart([str(id).encode(), pickle.dumps(data)])
260 260
261 261 def runReader(self):
262 262 '''
263 263 Run fuction for read units
264 264 '''
265 265 while True:
266 266
267 267 BaseClass.run(self, **self.kwargs)
268 268
269 269 if self.dataOut.error[0] == -1:
270 270 log.error(self.dataOut.error[1])
271 271 self.publish('end', self.id)
272 272 #self.sender.send_multipart([str(self.project_id).encode(), 'end'.encode()])
273 273 break
274 274
275 275 for op, optype, id, kwargs in self.operations:
276 276 if optype=='self':
277 277 op(**kwargs)
278 278 elif optype=='other':
279 279 self.dataOut = op.run(self.dataOut, **self.kwargs)
280 280 elif optype=='external':
281 281 self.publish(self.dataOut, opId)
282 282
283 283 if self.dataOut.flagNoData:
284 284 continue
285 285
286 286 self.publish(self.dataOut, self.id)
287 287
288 288 def runProc(self):
289 289 '''
290 290 Run function for proccessing units
291 291 '''
292 292
293 293 while True:
294 294 self.dataIn = self.listen()
295 295
296 296 if self.dataIn == 'end':
297 297 self.publish('end', self.id)
298 298 for op, optype, opId, kwargs in self.operations:
299 299 if optype == 'external':
300 300 self.publish('end', opId)
301 301 break
302 302
303 303 if self.dataIn.flagNoData:
304 304 continue
305 305
306 306 BaseClass.run(self, **self.kwargs)
307 307
308 308 for op, optype, opId, kwargs in self.operations:
309 309 if optype=='self':
310 310 op(**kwargs)
311 311 elif optype=='other':
312 312 self.dataOut = op.run(self.dataOut, **kwargs)
313 313 elif optype=='external':
314 314 self.publish(self.dataOut, opId)
315 315
316 316 if self.dataOut.flagNoData:
317 317 continue
318 318
319 319 self.publish(self.dataOut, self.id)
320 320
321 321 def runOp(self):
322 322 '''
323 323 Run function for operations
324 324 '''
325 325
326 326 while True:
327 327
328 328 dataOut = self.listen()
329
329
330 330 if dataOut == 'end':
331 331 break
332 332
333 333 BaseClass.run(self, dataOut, **self.kwargs)
334 334
335 335 def run(self):
336 336
337 337 if self.typeProc is "ProcUnit":
338 338
339 339 if self.inputId is not None:
340 340 self.subscribe()
341 341 self.set_publisher()
342 342
343 343 if 'Reader' not in BaseClass.__name__:
344 344 self.runProc()
345 345 else:
346 346 self.runReader()
347 347
348 348 elif self.typeProc is "Operation":
349 349
350 350 self.subscribe()
351 351 self.runOp()
352 352
353 353 else:
354 354 raise ValueError("Unknown type")
355 355
356 356 print("%s done" % BaseClass.__name__)
357 357 self.close()
358 358
359 359 def close(self):
360 360
361 361 if self.sender:
362 362 self.sender.close()
363 363
364 364 if self.receiver:
365 365 self.receiver.close()
366 366
367 367 return MPClass No newline at end of file
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now