##// END OF EJS Templates
schain
RSS
updates real multiprocess, save hdf5, utils_Io
joabAM -
r1559:6d931be610e9
parent child
Show More
Side by Side Unified
Context file:
r1559:6d931be610e9 Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,1136 +1,1154
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Definition of diferent Data objects for different types of data
6 6
7 7 Here you will find the diferent data objects for the different types
8 8 of data, this data objects must be used as dataIn or dataOut objects in
9 9 processing units and operations. Currently the supported data objects are:
10 10 Voltage, Spectra, SpectraHeis, Fits, Correlation and Parameters
11 11 """
12 12
13 13 import copy
14 14 import numpy
15 15 import datetime
16 16 import json
17 17
18 18 import schainpy.admin
19 19 from schainpy.utils import log
20 from .jroheaderIO import SystemHeader, RadarControllerHeader
20 from .jroheaderIO import SystemHeader, RadarControllerHeader,ProcessingHeader
21 21 from schainpy.model.data import _noise
22
22 SPEED_OF_LIGHT = 3e8
23 23
24 24 def getNumpyDtype(dataTypeCode):
25 25
26 26 if dataTypeCode == 0:
27 27 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
28 28 elif dataTypeCode == 1:
29 29 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
30 30 elif dataTypeCode == 2:
31 31 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
32 32 elif dataTypeCode == 3:
33 33 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
34 34 elif dataTypeCode == 4:
35 35 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
36 36 elif dataTypeCode == 5:
37 37 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
38 38 else:
39 39 raise ValueError('dataTypeCode was not defined')
40 40
41 41 return numpyDtype
42 42
43 43
44 44 def getDataTypeCode(numpyDtype):
45 45
46 46 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
47 47 datatype = 0
48 48 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
49 49 datatype = 1
50 50 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
51 51 datatype = 2
52 52 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
53 53 datatype = 3
54 54 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
55 55 datatype = 4
56 56 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
57 57 datatype = 5
58 58 else:
59 59 datatype = None
60 60
61 61 return datatype
62 62
63 63
64 64 def hildebrand_sekhon(data, navg):
65 65 """
66 66 This method is for the objective determination of the noise level in Doppler spectra. This
67 67 implementation technique is based on the fact that the standard deviation of the spectral
68 68 densities is equal to the mean spectral density for white Gaussian noise
69 69
70 70 Inputs:
71 71 Data : heights
72 72 navg : numbers of averages
73 73
74 74 Return:
75 75 mean : noise's level
76 76 """
77 77
78 78 sortdata = numpy.sort(data, axis=None)
79 79 '''
80 80 lenOfData = len(sortdata)
81 81 nums_min = lenOfData*0.5
82 82
83 83 if nums_min <= 5:
84 84
85 85 nums_min = 5
86 86
87 87 sump = 0.
88 88 sumq = 0.
89 89
90 90 j = 0
91 91 cont = 1
92 92
93 93 while((cont == 1)and(j < lenOfData)):
94 94
95 95 sump += sortdata[j]
96 96 sumq += sortdata[j]**2
97 97
98 98 if j > nums_min:
99 99 rtest = float(j)/(j-1) + 1.0/navg
100 100 if ((sumq*j) > (rtest*sump**2)):
101 101 j = j - 1
102 102 sump = sump - sortdata[j]
103 103 sumq = sumq - sortdata[j]**2
104 104 cont = 0
105 105
106 106 j += 1
107 107
108 108 lnoise = sump / j
109 109 return lnoise
110 110 '''
111 111 return _noise.hildebrand_sekhon(sortdata, navg)
112 112
113 113
114 114 class Beam:
115 115
116 116 def __init__(self):
117 117 self.codeList = []
118 118 self.azimuthList = []
119 119 self.zenithList = []
120 120
121 121
122 122
123 123 class GenericData(object):
124 124
125 125 flagNoData = True
126 126
127 127 def copy(self, inputObj=None):
128 128
129 129 if inputObj == None:
130 130 return copy.deepcopy(self)
131 131
132 132 for key in list(inputObj.__dict__.keys()):
133 133
134 134 attribute = inputObj.__dict__[key]
135 135
136 136 # If this attribute is a tuple or list
137 137 if type(inputObj.__dict__[key]) in (tuple, list):
138 138 self.__dict__[key] = attribute[:]
139 139 continue
140 140
141 141 # If this attribute is another object or instance
142 142 if hasattr(attribute, '__dict__'):
143 143 self.__dict__[key] = attribute.copy()
144 144 continue
145 145
146 146 self.__dict__[key] = inputObj.__dict__[key]
147 147
148 148 def deepcopy(self):
149 149
150 150 return copy.deepcopy(self)
151 151
152 152 def isEmpty(self):
153 153
154 154 return self.flagNoData
155 155
156 156 def isReady(self):
157 157
158 158 return not self.flagNoData
159 159
160 160
161 161 class JROData(GenericData):
162 162
163 163 useInputBuffer = False
164 164 buffer_empty = True
165 165
166 166 systemHeaderObj = SystemHeader()
167 167 radarControllerHeaderObj = RadarControllerHeader()
168 168 type = None
169 169 datatype = None # dtype but in string
170 170 nProfiles = None
171 171 heightList = None
172 172 channelList = None
173 173 flagDiscontinuousBlock = False
174 174 useLocalTime = False
175 175 utctime = None
176 176 timeZone = None
177 177 dstFlag = None
178 178 errorCount = None
179 179 blocksize = None
180 180 flagDecodeData = False # asumo q la data no esta decodificada
181 181 flagDeflipData = False # asumo q la data no esta sin flip
182 182 flagShiftFFT = False
183 183 nCohInt = None
184 184 windowOfFilter = 1
185 185 C = 3e8
186 186 frequency = 49.92e6
187 187 realtime = False
188 188 beacon_heiIndexList = None
189 189 last_block = None
190 190 blocknow = None
191 191 azimuth = None
192 192 zenith = None
193 beam = Beam()
193
194 194 profileIndex = None
195 195 error = None
196 196 data = None
197 197 nmodes = None
198 198 metadata_list = ['heightList', 'timeZone', 'type']
199 199 codeList = []
200 200 azimuthList = []
201 201 elevationList = []
202 202 last_noise = None
203 radar_ipp = None
203 __ipp = None
204 __ippSeconds = None
204 205 sampled_heightsFFT = None
205 206 pulseLength_TxA = None
206 207 deltaHeight = None
208 code = None
209 nCode = None
210 nBaud = None
211 unitsDescription = "The units of the parameters are according to the International System of units (Seconds, Meter, Hertz, ...), except \
212 the parameters related to distances such as heightList, or heightResolution wich are in Km"
213
207 214
208 215 def __str__(self):
209 216
210 217 return '{} - {}'.format(self.type, self.datatime())
211 218
212 219 def getNoise(self):
213 220
214 221 raise NotImplementedError
215 222
216 223 @property
217 224 def nChannels(self):
218 225
219 226 return len(self.channelList)
220 227
221 228 @property
222 229 def channelIndexList(self):
223 230
224 231 return list(range(self.nChannels))
225 232
226 233 @property
227 234 def nHeights(self):
228 235
229 236 return len(self.heightList)
230 237
231 238 def getDeltaH(self):
232 239
233 240 return self.heightList[1] - self.heightList[0]
234 241
235 242 @property
236 243 def ltctime(self):
237 244
238 245 if self.useLocalTime:
239 246 return self.utctime - self.timeZone * 60
240 247
241 248 return self.utctime
242 249
243 250 @property
244 251 def datatime(self):
245 252
246 253 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
247 254 return datatimeValue
248 255
249 256 def getTimeRange(self):
250 257
251 258 datatime = []
252 259
253 260 datatime.append(self.ltctime)
254 261 datatime.append(self.ltctime + self.timeInterval + 1)
255 262
256 263 datatime = numpy.array(datatime)
257 264
258 265 return datatime
259 266
260 267 def getFmaxTimeResponse(self):
261 268
262 269 period = (10**-6) * self.getDeltaH() / (0.15)
263 270
264 271 PRF = 1. / (period * self.nCohInt)
265 272
266 273 fmax = PRF
267 274
268 275 return fmax
269 276
270 277 def getFmax(self):
271 PRF = 1. / (self.ippSeconds * self.nCohInt)
278 PRF = 1. / (self.__ippSeconds * self.nCohInt)
272 279
273 280 fmax = PRF
274 281 return fmax
275 282
276 283 def getVmax(self):
277 284
278 285 _lambda = self.C / self.frequency
279 286
280 287 vmax = self.getFmax() * _lambda / 2
281 288
282 289 return vmax
283 290
284 291 @property
285 292 def ippSeconds(self):
286 293 '''
287 294 '''
288 return self.radarControllerHeaderObj.ippSeconds
295 #return self.radarControllerHeaderObj.ippSeconds
296 return self.__ippSeconds
289 297
290 298 @ippSeconds.setter
291 299 def ippSeconds(self, ippSeconds):
292 300 '''
293 301 '''
294 self.radarControllerHeaderObj.ippSeconds = ippSeconds
295
296 @property
297 def code(self):
298 '''
299 '''
300 return self.radarControllerHeaderObj.code
301
302 @code.setter
303 def code(self, code):
304 '''
305 '''
306 self.radarControllerHeaderObj.code = code
302 #self.radarControllerHeaderObj.ippSeconds = ippSeconds
303 self.__ippSeconds = ippSeconds
304 self.__ipp = ippSeconds*SPEED_OF_LIGHT/2000.0
307 305
308 @property
309 def nCode(self):
310 '''
311 '''
312 return self.radarControllerHeaderObj.nCode
313
314 @nCode.setter
315 def nCode(self, ncode):
316 '''
317 '''
318 self.radarControllerHeaderObj.nCode = ncode
319
320 @property
321 def nBaud(self):
322 '''
323 '''
324 return self.radarControllerHeaderObj.nBaud
325 306
326 @nBaud.setter
327 def nBaud(self, nbaud):
328 '''
329 '''
330 self.radarControllerHeaderObj.nBaud = nbaud
307 # @property
308 # def code(self):
309 # '''
310 # '''
311 # return self.radarControllerHeaderObj.code
312 #
313 # @code.setter
314 # def code(self, code):
315 # '''
316 # '''
317 # self.radarControllerHeaderObj.code = code
318 #
319 # @property
320 # def nCode(self):
321 # '''
322 # '''
323 # return self.radarControllerHeaderObj.nCode
324 #
325 # @nCode.setter
326 # def nCode(self, ncode):
327 # '''
328 # '''
329 # self.radarControllerHeaderObj.nCode = ncode
330 #
331 # @property
332 # def nBaud(self):
333 # '''
334 # '''
335 # return self.radarControllerHeaderObj.nBaud
336 #
337 # @nBaud.setter
338 # def nBaud(self, nbaud):
339 # '''
340 # '''
341 # self.radarControllerHeaderObj.nBaud = nbaud
331 342
332 343 @property
333 344 def ipp(self):
334 345 '''
335 346 '''
336 return self.radarControllerHeaderObj.ipp
347 return self.__ipp
348 #return self.radarControllerHeaderObj.ipp
337 349
338 350 @ipp.setter
339 351 def ipp(self, ipp):
340 352 '''
341 353 '''
342 self.radarControllerHeaderObj.ipp = ipp
354 self.__ipp = ipp
355 #self.radarControllerHeaderObj.ipp = ipp
343 356
344 357 @property
345 358 def metadata(self):
346 359 '''
347 360 '''
348 361
349 362 return {attr: getattr(self, attr) for attr in self.metadata_list}
350 363
351 364
352 365 class Voltage(JROData):
353 366
354 367 dataPP_POW = None
355 368 dataPP_DOP = None
356 369 dataPP_WIDTH = None
357 370 dataPP_SNR = None
358 371
359 372 def __init__(self):
360 373 '''
361 374 Constructor
362 375 '''
363 376
364 377 self.useLocalTime = True
365 378 self.radarControllerHeaderObj = RadarControllerHeader()
366 379 self.systemHeaderObj = SystemHeader()
380 self.processingHeaderObj = ProcessingHeader()
367 381 self.type = "Voltage"
368 382 self.data = None
369 383 self.nProfiles = None
370 384 self.heightList = None
371 385 self.channelList = None
372 386 self.flagNoData = True
373 387 self.flagDiscontinuousBlock = False
374 388 self.utctime = None
375 389 self.timeZone = 0
376 390 self.dstFlag = None
377 391 self.errorCount = None
378 392 self.nCohInt = None
379 393 self.blocksize = None
380 394 self.flagCohInt = False
381 395 self.flagDecodeData = False # asumo q la data no esta decodificada
382 396 self.flagDeflipData = False # asumo q la data no esta sin flip
383 397 self.flagShiftFFT = False
384 398 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
385 399 self.profileIndex = 0
386 400 self.metadata_list = ['type', 'heightList', 'timeZone', 'nProfiles', 'channelList', 'nCohInt',
387 401 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp']
388 402
389 403 def getNoisebyHildebrand(self, channel=None, ymin_index=None, ymax_index=None):
390 404 """
391 405 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
392 406
393 407 Return:
394 408 noiselevel
395 409 """
396 410
397 411 if channel != None:
398 412 data = self.data[channel,ymin_index:ymax_index]
399 413 nChannels = 1
400 414 else:
401 415 data = self.data[:,ymin_index:ymax_index]
402 416 nChannels = self.nChannels
403 417
404 418 noise = numpy.zeros(nChannels)
405 419 power = data * numpy.conjugate(data)
406 420
407 421 for thisChannel in range(nChannels):
408 422 if nChannels == 1:
409 423 daux = power[:].real
410 424 else:
411 425 daux = power[thisChannel, :].real
412 426 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
413 427
414 428 return noise
415 429
416 430 def getNoise(self, type=1, channel=None,ymin_index=None, ymax_index=None):
417 431
418 432 if type == 1:
419 433 noise = self.getNoisebyHildebrand(channel,ymin_index, ymax_index)
420 434
421 435 return noise
422 436
423 437 def getPower(self, channel=None):
424 438
425 439 if channel != None:
426 440 data = self.data[channel]
427 441 else:
428 442 data = self.data
429 443
430 444 power = data * numpy.conjugate(data)
431 445 powerdB = 10 * numpy.log10(power.real)
432 446 powerdB = numpy.squeeze(powerdB)
433 447
434 448 return powerdB
435 449
436 450 @property
437 451 def timeInterval(self):
438 452
439 453 return self.ippSeconds * self.nCohInt
440 454
441 455 noise = property(getNoise, "I'm the 'nHeights' property.")
442 456
443 457
444 458 class Spectra(JROData):
445 459
446 460 data_outlier = None
447 461
448 462 def __init__(self):
449 463 '''
450 464 Constructor
451 465 '''
452 466
453 467 self.data_dc = None
454 468 self.data_spc = None
455 469 self.data_cspc = None
456 470 self.useLocalTime = True
457 471 self.radarControllerHeaderObj = RadarControllerHeader()
458 472 self.systemHeaderObj = SystemHeader()
473 self.processingHeaderObj = ProcessingHeader()
459 474 self.type = "Spectra"
460 475 self.timeZone = 0
461 476 self.nProfiles = None
462 477 self.heightList = None
463 478 self.channelList = None
464 479 self.pairsList = None
465 480 self.flagNoData = True
466 481 self.flagDiscontinuousBlock = False
467 482 self.utctime = None
468 483 self.nCohInt = None
469 484 self.nIncohInt = None
470 485 self.blocksize = None
471 486 self.nFFTPoints = None
472 487 self.wavelength = None
473 488 self.flagDecodeData = False # asumo q la data no esta decodificada
474 489 self.flagDeflipData = False # asumo q la data no esta sin flip
475 490 self.flagShiftFFT = False
476 491 self.ippFactor = 1
477 492 self.beacon_heiIndexList = []
478 493 self.noise_estimation = None
479 494 self.codeList = []
480 495 self.azimuthList = []
481 496 self.elevationList = []
482 497 self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt',
483 498 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp','nIncohInt', 'nFFTPoints', 'nProfiles']
484 499
485 500
486 501
487 502
488 503 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
489 504 """
490 505 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
491 506
492 507 Return:
493 508 noiselevel
494 509 """
495 510 # if hasattr(self.nIncohInt, "__len__"): #nIncohInt is a matrix
496 511 #
497 512 # heis = self.data_spc.shape[2]
498 513 #
499 514 # noise = numpy.zeros((self.nChannels, heis))
500 515 # for hei in range(heis):
501 516 # for channel in range(self.nChannels):
502 517 # daux = self.data_spc[channel, xmin_index:xmax_index, hei]
503 518 #
504 519 # noise[channel,hei] = hildebrand_sekhon(daux, self.nIncohInt[channel,hei])
505 520 #
506 521 # else:
507 522 # noise = numpy.zeros(self.nChannels)
508 523 # for channel in range(self.nChannels):
509 524 # daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
510 525 #
511 526 # noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
512 527 noise = numpy.zeros(self.nChannels)
513 528
514 529 for channel in range(self.nChannels):
515 530 daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
516 531
517 532 noise[channel] = hildebrand_sekhon(daux, self.max_nIncohInt[channel])
518 533
519 534 return noise
520 535
521 536 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
522 537
523 538 if self.noise_estimation is not None:
524 539 # this was estimated by getNoise Operation defined in jroproc_spectra.py
525 540 return self.noise_estimation
526 541 else:
527 542 noise = self.getNoisebyHildebrand(
528 543 xmin_index, xmax_index, ymin_index, ymax_index)
529 544 return noise
530 545
531 546 def getFreqRangeTimeResponse(self, extrapoints=0):
532 547
533 548 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
534 549 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
535 550
536 551 return freqrange
537 552
538 553 def getAcfRange(self, extrapoints=0):
539 554
540 555 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
541 556 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
542 557
543 558 return freqrange
544 559
545 560 def getFreqRange(self, extrapoints=0):
546 561
547 562 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
548 563 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
549 564
550 565 return freqrange
551 566
552 567 def getVelRange(self, extrapoints=0):
553 568
554 569 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
555 570 velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
556 571
557 572 if self.nmodes:
558 573 return velrange/self.nmodes
559 574 else:
560 575 return velrange
561 576
562 577 @property
563 578 def nPairs(self):
564 579
565 580 return len(self.pairsList)
566 581
567 582 @property
568 583 def pairsIndexList(self):
569 584
570 585 return list(range(self.nPairs))
571 586
572 587 @property
573 588 def normFactor(self):
574 589
575 590 pwcode = 1
576 591
577 592 if self.flagDecodeData:
578 593 pwcode = numpy.sum(self.code[0]**2)
579 594 #print(self.flagDecodeData, pwcode)
580 595 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
581 596 normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
582 597
583 598
584 599 return normFactor
585 600
586 601 @property
587 602 def flag_cspc(self):
588 603
589 604 if self.data_cspc is None:
590 605 return True
591 606
592 607 return False
593 608
594 609 @property
595 610 def flag_dc(self):
596 611
597 612 if self.data_dc is None:
598 613 return True
599 614
600 615 return False
601 616
602 617 @property
603 618 def timeInterval(self):
604 619
605 620 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
606 621 if self.nmodes:
607 622 return self.nmodes*timeInterval
608 623 else:
609 624 return timeInterval
610 625
611 626 def getPower(self):
612 627
613 628 factor = self.normFactor
614 629 power = numpy.zeros( (self.nChannels,self.nHeights) )
615 630 for ch in range(self.nChannels):
616 631 z = None
617 632 if hasattr(factor,'shape'):
618 633 if factor.ndim > 1:
619 634 z = self.data_spc[ch]/factor[ch]
620 635 else:
621 636 z = self.data_spc[ch]/factor
622 637 else:
623 638 z = self.data_spc[ch]/factor
624 639 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
625 640 avg = numpy.average(z, axis=0)
626 641 power[ch] = 10 * numpy.log10(avg)
627 642 return power
628 643
629 644 @property
630 645 def max_nIncohInt(self):
631 646
632 647 ints = numpy.zeros(self.nChannels)
633 648 for ch in range(self.nChannels):
634 649 if hasattr(self.nIncohInt,'shape'):
635 650 if self.nIncohInt.ndim > 1:
636 651 ints[ch,] = self.nIncohInt[ch].max()
637 652 else:
638 653 ints[ch,] = self.nIncohInt
639 654 self.nIncohInt = int(self.nIncohInt)
640 655 else:
641 656 ints[ch,] = self.nIncohInt
642 657
643 658 return ints
644 659
645 660
646 661 def getCoherence(self, pairsList=None, phase=False):
647 662
648 663 z = []
649 664 if pairsList is None:
650 665 pairsIndexList = self.pairsIndexList
651 666 else:
652 667 pairsIndexList = []
653 668 for pair in pairsList:
654 669 if pair not in self.pairsList:
655 670 raise ValueError("Pair %s is not in dataOut.pairsList" % (
656 671 pair))
657 672 pairsIndexList.append(self.pairsList.index(pair))
658 673 for i in range(len(pairsIndexList)):
659 674 pair = self.pairsList[pairsIndexList[i]]
660 675 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
661 676 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
662 677 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
663 678 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
664 679 if phase:
665 680 data = numpy.arctan2(avgcoherenceComplex.imag,
666 681 avgcoherenceComplex.real) * 180 / numpy.pi
667 682 else:
668 683 data = numpy.abs(avgcoherenceComplex)
669 684
670 685 z.append(data)
671 686
672 687 return numpy.array(z)
673 688
674 689 def setValue(self, value):
675 690
676 691 print("This property should not be initialized", value)
677 692
678 693 return
679 694
680 695 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
681 696
682 697
683 698 class SpectraHeis(Spectra):
684 699
685 700 def __init__(self):
686 701
687 702 self.radarControllerHeaderObj = RadarControllerHeader()
688 703 self.systemHeaderObj = SystemHeader()
689 704 self.type = "SpectraHeis"
690 705 self.nProfiles = None
691 706 self.heightList = None
692 707 self.channelList = None
693 708 self.flagNoData = True
694 709 self.flagDiscontinuousBlock = False
695 710 self.utctime = None
696 711 self.blocksize = None
697 712 self.profileIndex = 0
698 713 self.nCohInt = 1
699 714 self.nIncohInt = 1
700 715
701 716 @property
702 717 def normFactor(self):
703 718 pwcode = 1
704 719 if self.flagDecodeData:
705 720 pwcode = numpy.sum(self.code[0]**2)
706 721
707 722 normFactor = self.nIncohInt * self.nCohInt * pwcode
708 723
709 724 return normFactor
710 725
711 726 @property
712 727 def timeInterval(self):
713 728
714 729 return self.ippSeconds * self.nCohInt * self.nIncohInt
715 730
716 731
717 732 class Fits(JROData):
718 733
719 734 def __init__(self):
720 735
721 736 self.type = "Fits"
722 737 self.nProfiles = None
723 738 self.heightList = None
724 739 self.channelList = None
725 740 self.flagNoData = True
726 741 self.utctime = None
727 742 self.nCohInt = 1
728 743 self.nIncohInt = 1
729 744 self.useLocalTime = True
730 745 self.profileIndex = 0
731 746 self.timeZone = 0
732 747
733 748 def getTimeRange(self):
734 749
735 750 datatime = []
736 751
737 752 datatime.append(self.ltctime)
738 753 datatime.append(self.ltctime + self.timeInterval)
739 754
740 755 datatime = numpy.array(datatime)
741 756
742 757 return datatime
743 758
744 759 def getChannelIndexList(self):
745 760
746 761 return list(range(self.nChannels))
747 762
748 763 def getNoise(self, type=1):
749 764
750 765
751 766 if type == 1:
752 767 noise = self.getNoisebyHildebrand()
753 768
754 769 if type == 2:
755 770 noise = self.getNoisebySort()
756 771
757 772 if type == 3:
758 773 noise = self.getNoisebyWindow()
759 774
760 775 return noise
761 776
762 777 @property
763 778 def timeInterval(self):
764 779
765 780 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
766 781
767 782 return timeInterval
768 783
769 784 @property
770 785 def ippSeconds(self):
771 786 '''
772 787 '''
773 788 return self.ipp_sec
774 789
775 790 noise = property(getNoise, "I'm the 'nHeights' property.")
776 791
777 792
778 793 class Correlation(JROData):
779 794
780 795 def __init__(self):
781 796 '''
782 797 Constructor
783 798 '''
784 799 self.radarControllerHeaderObj = RadarControllerHeader()
785 800 self.systemHeaderObj = SystemHeader()
786 801 self.type = "Correlation"
787 802 self.data = None
788 803 self.dtype = None
789 804 self.nProfiles = None
790 805 self.heightList = None
791 806 self.channelList = None
792 807 self.flagNoData = True
793 808 self.flagDiscontinuousBlock = False
794 809 self.utctime = None
795 810 self.timeZone = 0
796 811 self.dstFlag = None
797 812 self.errorCount = None
798 813 self.blocksize = None
799 814 self.flagDecodeData = False # asumo q la data no esta decodificada
800 815 self.flagDeflipData = False # asumo q la data no esta sin flip
801 816 self.pairsList = None
802 817 self.nPoints = None
803 818
804 819 def getPairsList(self):
805 820
806 821 return self.pairsList
807 822
808 823 def getNoise(self, mode=2):
809 824
810 825 indR = numpy.where(self.lagR == 0)[0][0]
811 826 indT = numpy.where(self.lagT == 0)[0][0]
812 827
813 828 jspectra0 = self.data_corr[:, :, indR, :]
814 829 jspectra = copy.copy(jspectra0)
815 830
816 831 num_chan = jspectra.shape[0]
817 832 num_hei = jspectra.shape[2]
818 833
819 834 freq_dc = jspectra.shape[1] / 2
820 835 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
821 836
822 837 if ind_vel[0] < 0:
823 838 ind_vel[list(range(0, 1))] = ind_vel[list(
824 839 range(0, 1))] + self.num_prof
825 840
826 841 if mode == 1:
827 842 jspectra[:, freq_dc, :] = (
828 843 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
829 844
830 845 if mode == 2:
831 846
832 847 vel = numpy.array([-2, -1, 1, 2])
833 848 xx = numpy.zeros([4, 4])
834 849
835 850 for fil in range(4):
836 851 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
837 852
838 853 xx_inv = numpy.linalg.inv(xx)
839 854 xx_aux = xx_inv[0, :]
840 855
841 856 for ich in range(num_chan):
842 857 yy = jspectra[ich, ind_vel, :]
843 858 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
844 859
845 860 junkid = jspectra[ich, freq_dc, :] <= 0
846 861 cjunkid = sum(junkid)
847 862
848 863 if cjunkid.any():
849 864 jspectra[ich, freq_dc, junkid.nonzero()] = (
850 865 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
851 866
852 867 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
853 868
854 869 return noise
855 870
856 871 @property
857 872 def timeInterval(self):
858 873
859 874 return self.ippSeconds * self.nCohInt * self.nProfiles
860 875
861 876 def splitFunctions(self):
862 877
863 878 pairsList = self.pairsList
864 879 ccf_pairs = []
865 880 acf_pairs = []
866 881 ccf_ind = []
867 882 acf_ind = []
868 883 for l in range(len(pairsList)):
869 884 chan0 = pairsList[l][0]
870 885 chan1 = pairsList[l][1]
871 886
872 887 # Obteniendo pares de Autocorrelacion
873 888 if chan0 == chan1:
874 889 acf_pairs.append(chan0)
875 890 acf_ind.append(l)
876 891 else:
877 892 ccf_pairs.append(pairsList[l])
878 893 ccf_ind.append(l)
879 894
880 895 data_acf = self.data_cf[acf_ind]
881 896 data_ccf = self.data_cf[ccf_ind]
882 897
883 898 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
884 899
885 900 @property
886 901 def normFactor(self):
887 902 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
888 903 acf_pairs = numpy.array(acf_pairs)
889 904 normFactor = numpy.zeros((self.nPairs, self.nHeights))
890 905
891 906 for p in range(self.nPairs):
892 907 pair = self.pairsList[p]
893 908
894 909 ch0 = pair[0]
895 910 ch1 = pair[1]
896 911
897 912 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
898 913 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
899 914 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
900 915
901 916 return normFactor
902 917
903 918
904 919 class Parameters(Spectra):
905 920
906 921 radarControllerHeaderTxt=None #header Controller like text
907 922 groupList = None # List of Pairs, Groups, etc
908 923 data_param = None # Parameters obtained
909 924 data_pre = None # Data Pre Parametrization
910 925 data_SNR = None # Signal to Noise Ratio
911 926 data_outlier = None
912 927 abscissaList = None # Abscissa, can be velocities, lags or time
913 928 utctimeInit = None # Initial UTC time
914 929 paramInterval = None # Time interval to calculate Parameters in seconds
915 930 useLocalTime = True
916 931 # Fitting
917 932 data_error = None # Error of the estimation
918 933 constants = None
919 934 library = None
920 935 # Output signal
921 936 outputInterval = None # Time interval to calculate output signal in seconds
922 937 data_output = None # Out signal
923 938 nAvg = None
924 939 noise_estimation = None
925 940 GauSPC = None # Fit gaussian SPC
926 941
942
943
927 944 def __init__(self):
928 945 '''
929 946 Constructor
930 947 '''
931 948 self.radarControllerHeaderObj = RadarControllerHeader()
932 949 self.systemHeaderObj = SystemHeader()
950 self.processingHeaderObj = ProcessingHeader()
933 951 self.type = "Parameters"
934 952 self.timeZone = 0
935 953
936 954 def getTimeRange1(self, interval):
937 955
938 956 datatime = []
939 957
940 958 if self.useLocalTime:
941 959 time1 = self.utctimeInit - self.timeZone * 60
942 960 else:
943 961 time1 = self.utctimeInit
944 962
945 963 datatime.append(time1)
946 964 datatime.append(time1 + interval)
947 965 datatime = numpy.array(datatime)
948 966
949 967 return datatime
950 968
951 969 @property
952 970 def timeInterval(self):
953 971
954 972 if hasattr(self, 'timeInterval1'):
955 973 return self.timeInterval1
956 974 else:
957 975 return self.paramInterval
958 976
959 977 def setValue(self, value):
960 978
961 979 print("This property should not be initialized")
962 980
963 981 return
964 982
965 983 def getNoise(self):
966 984
967 985 return self.spc_noise
968 986
969 987 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
970 988
971 989
972 990 class PlotterData(object):
973 991 '''
974 992 Object to hold data to be plotted
975 993 '''
976 994
977 995 MAXNUMX = 200
978 996 MAXNUMY = 200
979 997
980 998 def __init__(self, code, exp_code, localtime=True):
981 999
982 1000 self.key = code
983 1001 self.exp_code = exp_code
984 1002 self.ready = False
985 1003 self.flagNoData = False
986 1004 self.localtime = localtime
987 1005 self.data = {}
988 1006 self.meta = {}
989 1007 self.__heights = []
990 1008
991 1009 def __str__(self):
992 1010 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
993 1011 return 'Data[{}][{}]'.format(';'.join(dum), len(self.times))
994 1012
995 1013 def __len__(self):
996 1014 return len(self.data)
997 1015
998 1016 def __getitem__(self, key):
999 1017 if isinstance(key, int):
1000 1018 return self.data[self.times[key]]
1001 1019 elif isinstance(key, str):
1002 1020 ret = numpy.array([self.data[x][key] for x in self.times])
1003 1021 if ret.ndim > 1:
1004 1022 ret = numpy.swapaxes(ret, 0, 1)
1005 1023 return ret
1006 1024
1007 1025 def __contains__(self, key):
1008 1026 return key in self.data[self.min_time]
1009 1027
1010 1028 def setup(self):
1011 1029 '''
1012 1030 Configure object
1013 1031 '''
1014 1032 self.type = ''
1015 1033 self.ready = False
1016 1034 del self.data
1017 1035 self.data = {}
1018 1036 self.__heights = []
1019 1037 self.__all_heights = set()
1020 1038
1021 1039 def shape(self, key):
1022 1040 '''
1023 1041 Get the shape of the one-element data for the given key
1024 1042 '''
1025 1043
1026 1044 if len(self.data[self.min_time][key]):
1027 1045 return self.data[self.min_time][key].shape
1028 1046 return (0,)
1029 1047
1030 1048 def update(self, data, tm, meta={}):
1031 1049 '''
1032 1050 Update data object with new dataOut
1033 1051 '''
1034 1052
1035 1053 self.data[tm] = data
1036 1054
1037 1055 for key, value in meta.items():
1038 1056 setattr(self, key, value)
1039 1057
1040 1058 def normalize_heights(self):
1041 1059 '''
1042 1060 Ensure same-dimension of the data for different heighList
1043 1061 '''
1044 1062
1045 1063 H = numpy.array(list(self.__all_heights))
1046 1064 H.sort()
1047 1065 for key in self.data:
1048 1066 shape = self.shape(key)[:-1] + H.shape
1049 1067 for tm, obj in list(self.data[key].items()):
1050 1068 h = self.__heights[self.times.tolist().index(tm)]
1051 1069 if H.size == h.size:
1052 1070 continue
1053 1071 index = numpy.where(numpy.in1d(H, h))[0]
1054 1072 dummy = numpy.zeros(shape) + numpy.nan
1055 1073 if len(shape) == 2:
1056 1074 dummy[:, index] = obj
1057 1075 else:
1058 1076 dummy[index] = obj
1059 1077 self.data[key][tm] = dummy
1060 1078
1061 1079 self.__heights = [H for tm in self.times]
1062 1080
1063 1081 def jsonify(self, tm, plot_name, plot_type, decimate=False):
1064 1082 '''
1065 1083 Convert data to json
1066 1084 '''
1067 1085
1068 1086 meta = {}
1069 1087 meta['xrange'] = []
1070 1088 dy = int(len(self.yrange)/self.MAXNUMY) + 1
1071 1089 tmp = self.data[tm][self.key]
1072 1090 shape = tmp.shape
1073 1091 if len(shape) == 2:
1074 1092 data = self.roundFloats(self.data[tm][self.key][::, ::dy].tolist())
1075 1093 elif len(shape) == 3:
1076 1094 dx = int(self.data[tm][self.key].shape[1]/self.MAXNUMX) + 1
1077 1095 data = self.roundFloats(
1078 1096 self.data[tm][self.key][::, ::dx, ::dy].tolist())
1079 1097 meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1080 1098 else:
1081 1099 data = self.roundFloats(self.data[tm][self.key].tolist())
1082 1100
1083 1101 ret = {
1084 1102 'plot': plot_name,
1085 1103 'code': self.exp_code,
1086 1104 'time': float(tm),
1087 1105 'data': data,
1088 1106 }
1089 1107 meta['type'] = plot_type
1090 1108 meta['interval'] = float(self.interval)
1091 1109 meta['localtime'] = self.localtime
1092 1110 meta['yrange'] = self.roundFloats(self.yrange[::dy].tolist())
1093 1111 meta.update(self.meta)
1094 1112 ret['metadata'] = meta
1095 1113 return json.dumps(ret)
1096 1114
1097 1115 @property
1098 1116 def times(self):
1099 1117 '''
1100 1118 Return the list of times of the current data
1101 1119 '''
1102 1120
1103 1121 ret = [t for t in self.data]
1104 1122 ret.sort()
1105 1123 return numpy.array(ret)
1106 1124
1107 1125 @property
1108 1126 def min_time(self):
1109 1127 '''
1110 1128 Return the minimun time value
1111 1129 '''
1112 1130
1113 1131 return self.times[0]
1114 1132
1115 1133 @property
1116 1134 def max_time(self):
1117 1135 '''
1118 1136 Return the maximun time value
1119 1137 '''
1120 1138
1121 1139 return self.times[-1]
1122 1140
1123 1141 # @property
1124 1142 # def heights(self):
1125 1143 # '''
1126 1144 # Return the list of heights of the current data
1127 1145 # '''
1128 1146
1129 1147 # return numpy.array(self.__heights[-1])
1130 1148
1131 1149 @staticmethod
1132 1150 def roundFloats(obj):
1133 1151 if isinstance(obj, list):
1134 1152 return list(map(PlotterData.roundFloats, obj))
1135 1153 elif isinstance(obj, float):
1136 1154 return round(obj, 2)
Show file before | Show file after | Hide comments
@@ -1,915 +1,948
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
5 5 '''
6 6 import sys
7 7 import numpy
8 8 import copy
9 9 import datetime
10 10 import inspect
11 11 from schainpy.utils import log
12 12
13 13 SPEED_OF_LIGHT = 299792458
14 14 SPEED_OF_LIGHT = 3e8
15 15
16 16 BASIC_STRUCTURE = numpy.dtype([
17 17 ('nSize', '<u4'),
18 18 ('nVersion', '<u2'),
19 19 ('nDataBlockId', '<u4'),
20 20 ('nUtime', '<u4'),
21 21 ('nMilsec', '<u2'),
22 22 ('nTimezone', '<i2'),
23 23 ('nDstflag', '<i2'),
24 24 ('nErrorCount', '<u4')
25 25 ])
26 26
27 27 SYSTEM_STRUCTURE = numpy.dtype([
28 28 ('nSize', '<u4'),
29 29 ('nNumSamples', '<u4'),
30 30 ('nNumProfiles', '<u4'),
31 31 ('nNumChannels', '<u4'),
32 32 ('nADCResolution', '<u4'),
33 33 ('nPCDIOBusWidth', '<u4'),
34 34 ])
35 35
36 36 RADAR_STRUCTURE = numpy.dtype([
37 37 ('nSize', '<u4'),
38 38 ('nExpType', '<u4'),
39 39 ('nNTx', '<u4'),
40 40 ('fIpp', '<f4'),
41 41 ('fTxA', '<f4'),
42 42 ('fTxB', '<f4'),
43 43 ('nNumWindows', '<u4'),
44 44 ('nNumTaus', '<u4'),
45 45 ('nCodeType', '<u4'),
46 46 ('nLine6Function', '<u4'),
47 47 ('nLine5Function', '<u4'),
48 48 ('fClock', '<f4'),
49 49 ('nPrePulseBefore', '<u4'),
50 50 ('nPrePulseAfter', '<u4'),
51 51 ('sRangeIPP', '<a20'),
52 52 ('sRangeTxA', '<a20'),
53 53 ('sRangeTxB', '<a20'),
54 54 ])
55 55
56 56 SAMPLING_STRUCTURE = numpy.dtype(
57 57 [('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')])
58 58
59 59
60 60 PROCESSING_STRUCTURE = numpy.dtype([
61 61 ('nSize', '<u4'),
62 62 ('nDataType', '<u4'),
63 63 ('nSizeOfDataBlock', '<u4'),
64 64 ('nProfilesperBlock', '<u4'),
65 65 ('nDataBlocksperFile', '<u4'),
66 66 ('nNumWindows', '<u4'),
67 67 ('nProcessFlags', '<u4'),
68 68 ('nCoherentIntegrations', '<u4'),
69 69 ('nIncoherentIntegrations', '<u4'),
70 70 ('nTotalSpectra', '<u4')
71 71 ])
72 72
73 73
74 74 class Header(object):
75 75
76 76 def __init__(self):
77 77 raise NotImplementedError
78 78
79 79 def copy(self):
80 80 return copy.deepcopy(self)
81 81
82 82 def read(self):
83 83
84 84 raise NotImplementedError
85 85
86 86 def write(self):
87 87
88 88 raise NotImplementedError
89 89
90 90 def getAllowedArgs(self):
91 91 args = inspect.getargspec(self.__init__).args
92 92 try:
93 93 args.remove('self')
94 94 except:
95 95 pass
96 96 return args
97 97
98 98 def getAsDict(self):
99 99 args = self.getAllowedArgs()
100 100 asDict = {}
101 101 for x in args:
102 102 asDict[x] = self[x]
103 103 return asDict
104 104
105 105 def __getitem__(self, name):
106 106 return getattr(self, name)
107 107
108 108 def printInfo(self):
109 109
110 110 message = "#" * 50 + "\n"
111 111 message += self.__class__.__name__.upper() + "\n"
112 112 message += "#" * 50 + "\n"
113 113
114 114 keyList = list(self.__dict__.keys())
115 115 keyList.sort()
116 116
117 117 for key in keyList:
118 118 message += "%s = %s" % (key, self.__dict__[key]) + "\n"
119 119
120 120 if "size" not in keyList:
121 121 attr = getattr(self, "size")
122 122
123 123 if attr:
124 124 message += "%s = %s" % ("size", attr) + "\n"
125 125
126 126 print(message)
127 127
128 128
129 129 class BasicHeader(Header):
130 130
131 131 size = None
132 132 version = None
133 133 dataBlock = None
134 134 utc = None
135 135 ltc = None
136 136 miliSecond = None
137 137 timeZone = None
138 138 dstFlag = None
139 139 errorCount = None
140 140 datatime = None
141 141 structure = BASIC_STRUCTURE
142 142 __LOCALTIME = None
143 143
144 144 def __init__(self, useLocalTime=True):
145 145
146 146 self.size = 24
147 147 self.version = 0
148 148 self.dataBlock = 0
149 149 self.utc = 0
150 150 self.miliSecond = 0
151 151 self.timeZone = 0
152 152 self.dstFlag = 0
153 153 self.errorCount = 0
154 154
155 155 self.useLocalTime = useLocalTime
156 156
157 157 def read(self, fp):
158 158
159 159 self.length = 0
160 160 try:
161 161 if hasattr(fp, 'read'):
162 162 header = numpy.fromfile(fp, BASIC_STRUCTURE, 1)
163 163 else:
164 164 header = numpy.fromstring(fp, BASIC_STRUCTURE, 1)
165 165 except Exception as e:
166 166 print("BasicHeader: ")
167 167 print(e)
168 168 return 0
169 169
170 170 self.size = int(header['nSize'][0])
171 171 self.version = int(header['nVersion'][0])
172 172 self.dataBlock = int(header['nDataBlockId'][0])
173 173 self.utc = int(header['nUtime'][0])
174 174 self.miliSecond = int(header['nMilsec'][0])
175 175 self.timeZone = int(header['nTimezone'][0])
176 176 self.dstFlag = int(header['nDstflag'][0])
177 177 self.errorCount = int(header['nErrorCount'][0])
178 178
179 179 if self.size < 24:
180 180 return 0
181 181
182 182 self.length = header.nbytes
183 183 return 1
184 184
185 185 def write(self, fp):
186 186
187 187 headerTuple = (self.size, self.version, self.dataBlock, self.utc,
188 188 self.miliSecond, self.timeZone, self.dstFlag, self.errorCount)
189 189 header = numpy.array(headerTuple, BASIC_STRUCTURE)
190 190 header.tofile(fp)
191 191
192 192 return 1
193 193
194 194 def get_ltc(self):
195 195
196 196 return self.utc - self.timeZone * 60
197 197
198 198 def set_ltc(self, value):
199 199
200 200 self.utc = value + self.timeZone * 60
201 201
202 202 def get_datatime(self):
203 203
204 204 return datetime.datetime.utcfromtimestamp(self.ltc)
205 205
206 206 ltc = property(get_ltc, set_ltc)
207 207 datatime = property(get_datatime)
208 208
209 209
210 210 class SystemHeader(Header):
211 211
212 212 size = None
213 213 nSamples = None
214 214 nProfiles = None
215 215 nChannels = None
216 216 adcResolution = None
217 217 pciDioBusWidth = None
218 218 structure = SYSTEM_STRUCTURE
219 219
220 220 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWidth=0):
221 221
222 222 self.size = 24
223 223 self.nSamples = nSamples
224 224 self.nProfiles = nProfiles
225 225 self.nChannels = nChannels
226 226 self.adcResolution = adcResolution
227 227 self.pciDioBusWidth = pciDioBusWidth
228 228
229 229 def read(self, fp):
230 230 self.length = 0
231 231 try:
232 232 startFp = fp.tell()
233 233 except Exception as e:
234 234 startFp = None
235 235 pass
236 236
237 237 try:
238 238 if hasattr(fp, 'read'):
239 239 header = numpy.fromfile(fp, SYSTEM_STRUCTURE, 1)
240 240 else:
241 241 header = numpy.fromstring(fp, SYSTEM_STRUCTURE, 1)
242 242 except Exception as e:
243 243 print("System Header: " + str(e))
244 244 return 0
245 245
246 246 self.size = header['nSize'][0]
247 247 self.nSamples = header['nNumSamples'][0]
248 248 self.nProfiles = header['nNumProfiles'][0]
249 249 self.nChannels = header['nNumChannels'][0]
250 250 self.adcResolution = header['nADCResolution'][0]
251 251 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
252 252
253 253 if startFp is not None:
254 254 endFp = self.size + startFp
255 255
256 256 if fp.tell() > endFp:
257 257 sys.stderr.write(
258 258 "Warning %s: Size value read from System Header is lower than it has to be\n" % fp.name)
259 259 return 0
260 260
261 261 if fp.tell() < endFp:
262 262 sys.stderr.write(
263 263 "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp.name)
264 264 return 0
265 265
266 266 self.length = header.nbytes
267 267 return 1
268 268
269 269 def write(self, fp):
270 270
271 271 headerTuple = (self.size, self.nSamples, self.nProfiles,
272 272 self.nChannels, self.adcResolution, self.pciDioBusWidth)
273 273 header = numpy.array(headerTuple, SYSTEM_STRUCTURE)
274 274 header.tofile(fp)
275 275
276 276 return 1
277 277
278 278
279 279 class RadarControllerHeader(Header):
280 280
281 281 expType = None
282 dtype = ""
282 283 nTx = None
283 284 ipp = None
284 285 txA = None
285 286 txB = None
286 287 nWindows = None
287 288 numTaus = None
288 289 codeType = None
289 290 line6Function = None
290 291 line5Function = None
291 292 fClock = None
292 293 prePulseBefore = None
293 294 prePulseAfter = None
294 rangeIpp = None
295 rangeIpp = None #variables innecesarias?
295 296 rangeTxA = None
296 297 rangeTxB = None
297 298 structure = RADAR_STRUCTURE
298 299 __size = None
300 ################################################
301 ippSeconds = None
302 frequency = None
303 sampleRate = None
304 nOsamp = None
305 channelList = []
306 azimuthList = []
307 elevationList =[]
308 codeList = []
309 nChannels = 1
310 heightList = []
311 heightResolution = None
312
299 313
300 314 def __init__(self, expType=2, nTx=1,
301 315 ipp=None, txA=0, txB=0,
302 316 nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
303 317 numTaus=0, line6Function=0, line5Function=0, fClock=None,
304 318 prePulseBefore=0, prePulseAfter=0,
305 codeType=0, nCode=0, nBaud=0, code=[],
306 flip1=0, flip2=0):
319 codeType=0, nCode=0, nBaud=0, code=[],nOsamp = None, frequency = None,sampleRate=None,
320 flip1=0, flip2=0, nChannels=1):
307 321
308 322 # self.size = 116
309 323 self.expType = expType
310 324 self.nTx = nTx
311 325 self.ipp = ipp
312 326 self.txA = txA
313 327 self.txB = txB
314 328 self.rangeIpp = ipp
315 329 self.rangeTxA = txA
316 330 self.rangeTxB = txB
317
331 self.frequency = frequency
332 self.sampleRate = sampleRate
318 333 self.nWindows = nWindows
319 334 self.numTaus = numTaus
320 335 self.codeType = codeType
321 336 self.line6Function = line6Function
322 337 self.line5Function = line5Function
323 338 self.fClock = fClock
324 339 self.prePulseBefore = prePulseBefore
325 340 self.prePulseAfter = prePulseAfter
326 341
327 342 self.nHeights = nHeights
328 343 self.firstHeight = firstHeight
329 344 self.deltaHeight = deltaHeight
330 345 self.samplesWin = nHeights
331 346
332 347 self.nCode = nCode
333 348 self.nBaud = nBaud
334 349 self.code = code
350 self.nOsamp = nOsamp
335 351 self.flip1 = flip1
336 352 self.flip2 = flip2
337 353
338 354 self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
339 355 # self.dynamic = numpy.array([],numpy.dtype('byte'))
340 356
341 357 if self.fClock is None and self.deltaHeight is not None:
342 358 self.fClock = 0.15 / (deltaHeight * 1e-6) # 0.15Km / (height * 1u)
343 359
344 360 def read(self, fp):
345 361 self.length = 0
346 362 try:
347 363 startFp = fp.tell()
348 364 except Exception as e:
349 365 startFp = None
350 366 pass
351 367
352 368 try:
353 369 if hasattr(fp, 'read'):
354 370 header = numpy.fromfile(fp, RADAR_STRUCTURE, 1)
355 371 else:
356 372 header = numpy.fromstring(fp, RADAR_STRUCTURE, 1)
357 373 self.length += header.nbytes
358 374 except Exception as e:
359 375 print("RadarControllerHeader: " + str(e))
360 376 return 0
361 377
362 378 size = int(header['nSize'][0])
363 379 self.expType = int(header['nExpType'][0])
364 380 self.nTx = int(header['nNTx'][0])
365 381 self.ipp = float(header['fIpp'][0])
366 382 self.txA = float(header['fTxA'][0])
367 383 self.txB = float(header['fTxB'][0])
368 384 self.nWindows = int(header['nNumWindows'][0])
369 385 self.numTaus = int(header['nNumTaus'][0])
370 386 self.codeType = int(header['nCodeType'][0])
371 387 self.line6Function = int(header['nLine6Function'][0])
372 388 self.line5Function = int(header['nLine5Function'][0])
373 389 self.fClock = float(header['fClock'][0])
374 390 self.prePulseBefore = int(header['nPrePulseBefore'][0])
375 391 self.prePulseAfter = int(header['nPrePulseAfter'][0])
376 392 self.rangeIpp = header['sRangeIPP'][0]
377 393 self.rangeTxA = header['sRangeTxA'][0]
378 394 self.rangeTxB = header['sRangeTxB'][0]
379 395
380 396 try:
381 397 if hasattr(fp, 'read'):
382 398 samplingWindow = numpy.fromfile(
383 399 fp, SAMPLING_STRUCTURE, self.nWindows)
384 400 else:
385 401 samplingWindow = numpy.fromstring(
386 402 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
387 403 self.length += samplingWindow.nbytes
388 404 except Exception as e:
389 405 print("RadarControllerHeader: " + str(e))
390 406 return 0
391 407 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
392 408 self.firstHeight = samplingWindow['h0']
393 409 self.deltaHeight = samplingWindow['dh']
394 410 self.samplesWin = samplingWindow['nsa']
395 411
396 412 try:
397 413 if hasattr(fp, 'read'):
398 414 self.Taus = numpy.fromfile(fp, '<f4', self.numTaus)
399 415 else:
400 416 self.Taus = numpy.fromstring(
401 417 fp[self.length:], '<f4', self.numTaus)
402 418 self.length += self.Taus.nbytes
403 419 except Exception as e:
404 420 print("RadarControllerHeader: " + str(e))
405 421 return 0
406 422
407 423 self.code_size = 0
408 424 if self.codeType != 0:
409 425
410 426 try:
411 427 if hasattr(fp, 'read'):
412 428 self.nCode = numpy.fromfile(fp, '<u4', 1)[0]
413 429 self.length += self.nCode.nbytes
414 430 self.nBaud = numpy.fromfile(fp, '<u4', 1)[0]
415 431 self.length += self.nBaud.nbytes
416 432 else:
417 433 self.nCode = numpy.fromstring(
418 434 fp[self.length:], '<u4', 1)[0]
419 435 self.length += self.nCode.nbytes
420 436 self.nBaud = numpy.fromstring(
421 437 fp[self.length:], '<u4', 1)[0]
422 438 self.length += self.nBaud.nbytes
423 439 except Exception as e:
424 440 print("RadarControllerHeader: " + str(e))
425 441 return 0
426 442 code = numpy.empty([self.nCode, self.nBaud], dtype='i1')
427 443
428 444 for ic in range(self.nCode):
429 445 try:
430 446 if hasattr(fp, 'read'):
431 447 temp = numpy.fromfile(fp, 'u4', int(
432 448 numpy.ceil(self.nBaud / 32.)))
433 449 else:
434 450 temp = numpy.fromstring(
435 451 fp, 'u4', int(numpy.ceil(self.nBaud / 32.)))
436 452 self.length += temp.nbytes
437 453 except Exception as e:
438 454 print("RadarControllerHeader: " + str(e))
439 455 return 0
440 456
441 457 for ib in range(self.nBaud - 1, -1, -1):
442 458 code[ic, ib] = temp[int(ib / 32)] % 2
443 459 temp[int(ib / 32)] = temp[int(ib / 32)] / 2
444 460
445 461 self.code = 2.0 * code - 1.0
446 462 self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
447 463
448 464 # if self.line5Function == RCfunction.FLIP:
449 465 # self.flip1 = numpy.fromfile(fp,'<u4',1)
450 466 #
451 467 # if self.line6Function == RCfunction.FLIP:
452 468 # self.flip2 = numpy.fromfile(fp,'<u4',1)
453 469 if startFp is not None:
454 470 endFp = size + startFp
455 471
456 472 if fp.tell() != endFp:
457 473 # fp.seek(endFp)
458 474 print("%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" % (fp.name, fp.tell() - startFp, size))
459 475 # return 0
460 476
461 477 if fp.tell() > endFp:
462 478 sys.stderr.write(
463 479 "Warning %s: Size value read from Radar Controller header is lower than it has to be\n" % fp.name)
464 480 # return 0
465 481
466 482 if fp.tell() < endFp:
467 483 sys.stderr.write(
468 484 "Warning %s: Size value read from Radar Controller header is greater than it has to be\n" % fp.name)
469 485
470 486 return 1
471 487
472 488 def toString(self):
473 489 #attrs = dir(self)
474 490 s = ""
475 491 for attribute, value in self.__dict__.items():
476 492 if value!=None and value!= 0:
477 493 s += '{:18s}'.format(str(attribute)) +'\t'+str(value)+'\n'
478 494 #s += str(att) +'\t'+str(getattr(self, att))+'\n'
479 495 return s
480 496
481 497 def write(self, fp):
482 498
483 499 headerTuple = (self.size,
484 500 self.expType,
485 501 self.nTx,
486 502 self.ipp,
487 503 self.txA,
488 504 self.txB,
489 505 self.nWindows,
490 506 self.numTaus,
491 507 self.codeType,
492 508 self.line6Function,
493 509 self.line5Function,
494 510 self.fClock,
495 511 self.prePulseBefore,
496 512 self.prePulseAfter,
497 513 self.rangeIpp,
498 514 self.rangeTxA,
499 515 self.rangeTxB)
500 516
501 517 header = numpy.array(headerTuple, RADAR_STRUCTURE)
502 518 header.tofile(fp)
503 519
504 520 sampleWindowTuple = (
505 521 self.firstHeight, self.deltaHeight, self.samplesWin)
506 522 samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
507 523 samplingWindow.tofile(fp)
508 524
509 525 if self.numTaus > 0:
510 526 self.Taus.tofile(fp)
511 527
512 528 if self.codeType != 0:
513 529 nCode = numpy.array(self.nCode, '<u4')
514 530 nCode.tofile(fp)
515 531 nBaud = numpy.array(self.nBaud, '<u4')
516 532 nBaud.tofile(fp)
517 533 code1 = (self.code + 1.0) / 2.
518 534
519 535 for ic in range(self.nCode):
520 536 tempx = numpy.zeros(int(numpy.ceil(self.nBaud / 32.)))
521 537 start = 0
522 538 end = 32
523 539 for i in range(len(tempx)):
524 540 code_selected = code1[ic, start:end]
525 541 for j in range(len(code_selected) - 1, -1, -1):
526 542 if code_selected[j] == 1:
527 543 tempx[i] = tempx[i] + \
528 544 2**(len(code_selected) - 1 - j)
529 545 start = start + 32
530 546 end = end + 32
531 547
532 548 tempx = tempx.astype('u4')
533 549 tempx.tofile(fp)
534 550
535 551 # if self.line5Function == RCfunction.FLIP:
536 552 # self.flip1.tofile(fp)
537 553 #
538 554 # if self.line6Function == RCfunction.FLIP:
539 555 # self.flip2.tofile(fp)
540 556
541 557 return 1
542 558
543 559 def get_ippSeconds(self):
544 560 '''
545 561 '''
546 562 ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT
547 563
548 564 return ippSeconds
549 565
550 566 def set_ippSeconds(self, ippSeconds):
551 567 '''
552 568 '''
553 569
554 570 self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0 * 1000)
555 571
556 572 return
557 573
558 574 def get_size(self):
559 575
560 576 self.__size = 116 + 12 * self.nWindows + 4 * self.numTaus
561 577
562 578 if self.codeType != 0:
563 579 self.__size += 4 + 4 + 4 * self.nCode * \
564 580 numpy.ceil(self.nBaud / 32.)
565 581
566 582 return self.__size
567 583
568 584 def set_size(self, value):
569 585
570 586 raise IOError("size is a property and it cannot be set, just read")
571 587
572 588 return
573 589
574 590 ippSeconds = property(get_ippSeconds, set_ippSeconds)
575 591 size = property(get_size, set_size)
576 592
577 593
578 594 class ProcessingHeader(Header):
579 595
580 596 # size = None
597
581 598 dtype = None
582 599 blockSize = None
583 600 profilesPerBlock = None
584 601 dataBlocksPerFile = None
585 602 nWindows = None
586 603 processFlags = None
587 604 nCohInt = None
588 605 nIncohInt = None
589 606 totalSpectra = None
590 607 structure = PROCESSING_STRUCTURE
591 608 flag_dc = None
592 609 flag_cspc = None
610 #########################################################
611 nFFTPoints = None
612 nSamplesFFT = None
613 channelList = []
614 azimuthList = []
615 elevationList =[]
616 codeList = []
617 nChannels = 1
618 heightList = []
619 ipp = None
620 ippSeconds = None
621 timeIncohInt = None
622 #################
623 rangeIpp = None
624 heightResolution = None
593 625
594 626 def __init__(self, dtype=0, blockSize=0, profilesPerBlock=0, dataBlocksPerFile=0, nWindows=0, processFlags=0, nCohInt=0,
595 627 nIncohInt=0, totalSpectra=0, nHeights=0, firstHeight=0, deltaHeight=0, samplesWin=0, spectraComb=0, nCode=0,
596 628 code=0, nBaud=None, shif_fft=False, flag_dc=False, flag_cspc=False, flag_decode=False, flag_deflip=False
597 629 ):
598 630
599 631 # self.size = 0
600 632 self.dtype = dtype
601 633 self.blockSize = blockSize
602 634 self.profilesPerBlock = 0
603 635 self.dataBlocksPerFile = 0
604 636 self.nWindows = 0
605 637 self.processFlags = 0
606 638 self.nCohInt = 0
607 639 self.nIncohInt = 0
608 640 self.totalSpectra = 0
609 641
610 642 self.nHeights = 0
611 643 self.firstHeight = 0
612 644 self.deltaHeight = 0
613 645 self.samplesWin = 0
614 646 self.spectraComb = 0
615 647 self.nCode = None
616 648 self.code = None
617 649 self.nBaud = None
618 650
619 651 self.shif_fft = False
620 652 self.flag_dc = False
621 653 self.flag_cspc = False
622 654 self.flag_decode = False
623 655 self.flag_deflip = False
624 656 self.length = 0
625 657
658
626 659 def read(self, fp):
627 660 self.length = 0
628 661 try:
629 662 startFp = fp.tell()
630 663 except Exception as e:
631 664 startFp = None
632 665 pass
633 666
634 667 try:
635 668 if hasattr(fp, 'read'):
636 669 header = numpy.fromfile(fp, PROCESSING_STRUCTURE, 1)
637 670 else:
638 671 header = numpy.fromstring(fp, PROCESSING_STRUCTURE, 1)
639 672 self.length += header.nbytes
640 673 except Exception as e:
641 674 print("ProcessingHeader: " + str(e))
642 675 return 0
643 676
644 677 size = int(header['nSize'][0])
645 678 self.dtype = int(header['nDataType'][0])
646 679 self.blockSize = int(header['nSizeOfDataBlock'][0])
647 680 self.profilesPerBlock = int(header['nProfilesperBlock'][0])
648 681 self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
649 682 self.nWindows = int(header['nNumWindows'][0])
650 683 self.processFlags = header['nProcessFlags']
651 684 self.nCohInt = int(header['nCoherentIntegrations'][0])
652 685 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
653 686 self.totalSpectra = int(header['nTotalSpectra'][0])
654 687
655 688 try:
656 689 if hasattr(fp, 'read'):
657 690 samplingWindow = numpy.fromfile(
658 691 fp, SAMPLING_STRUCTURE, self.nWindows)
659 692 else:
660 693 samplingWindow = numpy.fromstring(
661 694 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
662 695 self.length += samplingWindow.nbytes
663 696 except Exception as e:
664 697 print("ProcessingHeader: " + str(e))
665 698 return 0
666 699
667 700 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
668 701 self.firstHeight = float(samplingWindow['h0'][0])
669 702 self.deltaHeight = float(samplingWindow['dh'][0])
670 703 self.samplesWin = samplingWindow['nsa'][0]
671 704
672 705 try:
673 706 if hasattr(fp, 'read'):
674 707 self.spectraComb = numpy.fromfile(
675 708 fp, 'u1', 2 * self.totalSpectra)
676 709 else:
677 710 self.spectraComb = numpy.fromstring(
678 711 fp[self.length:], 'u1', 2 * self.totalSpectra)
679 712 self.length += self.spectraComb.nbytes
680 713 except Exception as e:
681 714 print("ProcessingHeader: " + str(e))
682 715 return 0
683 716
684 717 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
685 718 self.nCode = int(numpy.fromfile(fp, '<u4', 1))
686 719 self.nBaud = int(numpy.fromfile(fp, '<u4', 1))
687 720 self.code = numpy.fromfile(
688 721 fp, '<f4', self.nCode * self.nBaud).reshape(self.nCode, self.nBaud)
689 722
690 723 if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP):
691 724 exp_name_len = int(numpy.fromfile(fp, '<u4', 1))
692 725 exp_name = numpy.fromfile(fp, 'u1', exp_name_len + 1)
693 726
694 727 if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
695 728 self.shif_fft = True
696 729 else:
697 730 self.shif_fft = False
698 731
699 732 if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
700 733 self.flag_dc = True
701 734 else:
702 735 self.flag_dc = False
703 736
704 737 if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA):
705 738 self.flag_decode = True
706 739 else:
707 740 self.flag_decode = False
708 741
709 742 if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA):
710 743 self.flag_deflip = True
711 744 else:
712 745 self.flag_deflip = False
713 746
714 747 nChannels = 0
715 748 nPairs = 0
716 749 pairList = []
717 750
718 751 for i in range(0, self.totalSpectra * 2, 2):
719 752 if self.spectraComb[i] == self.spectraComb[i + 1]:
720 753 nChannels = nChannels + 1 # par de canales iguales
721 754 else:
722 755 nPairs = nPairs + 1 # par de canales diferentes
723 756 pairList.append((self.spectraComb[i], self.spectraComb[i + 1]))
724 757
725 758 self.flag_cspc = False
726 759 if nPairs > 0:
727 760 self.flag_cspc = True
728 761
729 762 if startFp is not None:
730 763 endFp = size + startFp
731 764 if fp.tell() > endFp:
732 765 sys.stderr.write(
733 766 "Warning: Processing header size is lower than it has to be")
734 767 return 0
735 768
736 769 if fp.tell() < endFp:
737 770 sys.stderr.write(
738 771 "Warning: Processing header size is greater than it is considered")
739 772
740 773 return 1
741 774
742 775 def write(self, fp):
743 776 # Clear DEFINE_PROCESS_CODE
744 777 self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE)
745 778
746 779 headerTuple = (self.size,
747 780 self.dtype,
748 781 self.blockSize,
749 782 self.profilesPerBlock,
750 783 self.dataBlocksPerFile,
751 784 self.nWindows,
752 785 self.processFlags,
753 786 self.nCohInt,
754 787 self.nIncohInt,
755 788 self.totalSpectra)
756 789
757 790 header = numpy.array(headerTuple, PROCESSING_STRUCTURE)
758 791 header.tofile(fp)
759 792
760 793 if self.nWindows != 0:
761 794 sampleWindowTuple = (
762 795 self.firstHeight, self.deltaHeight, self.samplesWin)
763 796 samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
764 797 samplingWindow.tofile(fp)
765 798
766 799 if self.totalSpectra != 0:
767 800 # spectraComb = numpy.array([],numpy.dtype('u1'))
768 801 spectraComb = self.spectraComb
769 802 spectraComb.tofile(fp)
770 803
771 804 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
772 805 # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
773 806 # nCode.tofile(fp)
774 807 #
775 808 # nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
776 809 # nBaud.tofile(fp)
777 810 #
778 811 # code = self.code.reshape(self.nCode*self.nBaud)
779 812 # code = code.astype(numpy.dtype('<f4'))
780 813 # code.tofile(fp)
781 814
782 815 return 1
783 816
784 817 def get_size(self):
785 818
786 819 self.__size = 40 + 12 * self.nWindows + 2 * self.totalSpectra
787 820
788 821 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
789 822 # self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
790 823 # self.__size += 4 + 4 + 4 * self.nCode * self.nBaud
791 824
792 825 return self.__size
793 826
794 827 def set_size(self, value):
795 828
796 829 raise IOError("size is a property and it cannot be set, just read")
797 830
798 831 return
799 832
800 833 size = property(get_size, set_size)
801 834
802 835
803 836 class RCfunction:
804 837 NONE = 0
805 838 FLIP = 1
806 839 CODE = 2
807 840 SAMPLING = 3
808 841 LIN6DIV256 = 4
809 842 SYNCHRO = 5
810 843
811 844
812 845 class nCodeType:
813 846 NONE = 0
814 847 USERDEFINE = 1
815 848 BARKER2 = 2
816 849 BARKER3 = 3
817 850 BARKER4 = 4
818 851 BARKER5 = 5
819 852 BARKER7 = 6
820 853 BARKER11 = 7
821 854 BARKER13 = 8
822 855 AC128 = 9
823 856 COMPLEMENTARYCODE2 = 10
824 857 COMPLEMENTARYCODE4 = 11
825 858 COMPLEMENTARYCODE8 = 12
826 859 COMPLEMENTARYCODE16 = 13
827 860 COMPLEMENTARYCODE32 = 14
828 861 COMPLEMENTARYCODE64 = 15
829 862 COMPLEMENTARYCODE128 = 16
830 863 CODE_BINARY28 = 17
831 864
832 865
833 866 class PROCFLAG:
834 867
835 868 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
836 869 DECODE_DATA = numpy.uint32(0x00000002)
837 870 SPECTRA_CALC = numpy.uint32(0x00000004)
838 871 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
839 872 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
840 873 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
841 874
842 875 DATATYPE_CHAR = numpy.uint32(0x00000040)
843 876 DATATYPE_SHORT = numpy.uint32(0x00000080)
844 877 DATATYPE_LONG = numpy.uint32(0x00000100)
845 878 DATATYPE_INT64 = numpy.uint32(0x00000200)
846 879 DATATYPE_FLOAT = numpy.uint32(0x00000400)
847 880 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
848 881
849 882 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
850 883 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
851 884 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
852 885
853 886 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
854 887 DEFLIP_DATA = numpy.uint32(0x00010000)
855 888 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
856 889
857 890 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
858 891 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
859 892 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
860 893 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
861 894 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
862 895
863 896 EXP_NAME_ESP = numpy.uint32(0x00200000)
864 897 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
865 898
866 899 OPERATION_MASK = numpy.uint32(0x0000003F)
867 900 DATATYPE_MASK = numpy.uint32(0x00000FC0)
868 901 DATAARRANGE_MASK = numpy.uint32(0x00007000)
869 902 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
870 903
871 904
872 905 dtype0 = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
873 906 dtype1 = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
874 907 dtype2 = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
875 908 dtype3 = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
876 909 dtype4 = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
877 910 dtype5 = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
878 911
879 912 NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
880 913
881 914 PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR,
882 915 PROCFLAG.DATATYPE_SHORT,
883 916 PROCFLAG.DATATYPE_LONG,
884 917 PROCFLAG.DATATYPE_INT64,
885 918 PROCFLAG.DATATYPE_FLOAT,
886 919 PROCFLAG.DATATYPE_DOUBLE]
887 920
888 921 DTYPE_WIDTH = [1, 2, 4, 8, 4, 8]
889 922
890 923
891 924 def get_dtype_index(numpy_dtype):
892 925
893 926 index = None
894 927
895 928 for i in range(len(NUMPY_DTYPE_LIST)):
896 929 if numpy_dtype == NUMPY_DTYPE_LIST[i]:
897 930 index = i
898 931 break
899 932
900 933 return index
901 934
902 935
903 936 def get_numpy_dtype(index):
904 937
905 938 return NUMPY_DTYPE_LIST[index]
906 939
907 940
908 941 def get_procflag_dtype(index):
909 942
910 943 return PROCFLAG_DTYPE_LIST[index]
911 944
912 945
913 946 def get_dtype_width(index):
914 947
915 948 return DTYPE_WIDTH[index]
Show file before | Show file after | Hide comments
@@ -1,1234 +1,1241
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Classes to plot Spectra data
6 6
7 7 """
8 8
9 9 import os
10 10 import numpy
11 11
12 12 from schainpy.model.graphics.jroplot_base import Plot, plt, log
13 13 from itertools import combinations
14 14 from matplotlib.ticker import LinearLocator
15 15
16 16 class SpectraPlot(Plot):
17 17 '''
18 18 Plot for Spectra data
19 19 '''
20 20
21 21 CODE = 'spc'
22 22 colormap = 'jet'
23 23 plot_type = 'pcolor'
24 24 buffering = False
25 25 channelList = []
26 26 elevationList = []
27 27 azimuthList = []
28 28
29 29 def setup(self):
30 30
31 31 self.nplots = len(self.data.channels)
32 32 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
33 33 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
34 34 self.height = 3.4 * self.nrows
35 35
36 36 self.cb_label = 'dB'
37 37 if self.showprofile:
38 38 self.width = 5.2 * self.ncols
39 39 else:
40 40 self.width = 4.2* self.ncols
41 41 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.12})
42 42 self.ylabel = 'Range [km]'
43 43
44 44
45 45 def update_list(self,dataOut):
46 46 if len(self.channelList) == 0:
47 47 self.channelList = dataOut.channelList
48 48 if len(self.elevationList) == 0:
49 49 self.elevationList = dataOut.elevationList
50 50 if len(self.azimuthList) == 0:
51 51 self.azimuthList = dataOut.azimuthList
52 52
53 53 def update(self, dataOut):
54 54
55 55 self.update_list(dataOut)
56 56 data = {}
57 57 meta = {}
58 58
59 59 #data['rti'] = dataOut.getPower()
60 60 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
61 61 noise = 10*numpy.log10(dataOut.getNoise()/norm)
62 62
63 63
64 64 z = numpy.zeros((dataOut.nChannels, dataOut.nFFTPoints, dataOut.nHeights))
65 65 for ch in range(dataOut.nChannels):
66 66 if hasattr(dataOut.normFactor,'ndim'):
67 67 if dataOut.normFactor.ndim > 1:
68 68 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
69 69
70 70 else:
71 71 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
72 72 else:
73 73 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
74 74
75 75
76 76 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
77 77 spc = 10*numpy.log10(z)
78 78
79 79 data['spc'] = spc
80 print(spc[0].shape)
80 #print(spc[0].shape)
81 81 data['rti'] = spc.mean(axis=1)
82 82 data['noise'] = noise
83 83 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
84 84 if self.CODE == 'spc_moments':
85 85 data['moments'] = dataOut.moments
86 86
87 87 return data, meta
88 88
89 89 def plot(self):
90 90 if self.xaxis == "frequency":
91 91 x = self.data.xrange[0]
92 92 self.xlabel = "Frequency (kHz)"
93 93 elif self.xaxis == "time":
94 94 x = self.data.xrange[1]
95 95 self.xlabel = "Time (ms)"
96 96 else:
97 97 x = self.data.xrange[2]
98 98 self.xlabel = "Velocity (m/s)"
99 99
100 100 if self.CODE == 'spc_moments':
101 101 x = self.data.xrange[2]
102 102 self.xlabel = "Velocity (m/s)"
103 103
104 104 self.titles = []
105 105 y = self.data.yrange
106 106 self.y = y
107 107
108 108 data = self.data[-1]
109 109 z = data['spc']
110 110 #print(z.shape, x.shape, y.shape)
111 111 for n, ax in enumerate(self.axes):
112 112 noise = self.data['noise'][n][0]
113 113 #print(noise)
114 114 if self.CODE == 'spc_moments':
115 115 mean = data['moments'][n, 1]
116 116 if ax.firsttime:
117 117 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
118 118 self.xmin = self.xmin if self.xmin else -self.xmax
119 119 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
120 120 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
121 121 ax.plt = ax.pcolormesh(x, y, z[n].T,
122 122 vmin=self.zmin,
123 123 vmax=self.zmax,
124 124 cmap=plt.get_cmap(self.colormap)
125 125 )
126 126
127 127 if self.showprofile:
128 128 ax.plt_profile = self.pf_axes[n].plot(
129 129 data['rti'][n], y)[0]
130 130 # ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
131 131 # color="k", linestyle="dashed", lw=1)[0]
132 132 if self.CODE == 'spc_moments':
133 133 ax.plt_mean = ax.plot(mean, y, color='k')[0]
134 134 else:
135 135 ax.plt.set_array(z[n].T.ravel())
136 136 if self.showprofile:
137 137 ax.plt_profile.set_data(data['rti'][n], y)
138 138 #ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
139 139 if self.CODE == 'spc_moments':
140 140 ax.plt_mean.set_data(mean, y)
141 141 if len(self.azimuthList) > 0 and len(self.elevationList) > 0:
142 142 self.titles.append('CH {}: {:2.1f}elv {:2.1f}az {:3.2f}dB'.format(self.channelList[n], noise, self.elevationList[n], self.azimuthList[n]))
143 143 else:
144 144 self.titles.append('CH {}: {:3.2f}dB'.format(self.channelList[n], noise))
145 145
146 146
147 147 class CrossSpectraPlot(Plot):
148 148
149 149 CODE = 'cspc'
150 150 colormap = 'jet'
151 151 plot_type = 'pcolor'
152 152 zmin_coh = None
153 153 zmax_coh = None
154 154 zmin_phase = None
155 155 zmax_phase = None
156 156 realChannels = None
157 157 crossPairs = None
158 158
159 159 def setup(self):
160 160
161 161 self.ncols = 4
162 162 self.nplots = len(self.data.pairs) * 2
163 163 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
164 164 self.width = 3.1 * self.ncols
165 165 self.height = 2.6 * self.nrows
166 166 self.ylabel = 'Range [km]'
167 167 self.showprofile = False
168 168 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
169 169
170 170 def update(self, dataOut):
171 171
172 172 data = {}
173 173 meta = {}
174 174
175 175 spc = dataOut.data_spc
176 176 cspc = dataOut.data_cspc
177 177 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
178 178 rawPairs = list(combinations(list(range(dataOut.nChannels)), 2))
179 179 meta['pairs'] = rawPairs
180 180
181 181 if self.crossPairs == None:
182 182 self.crossPairs = dataOut.pairsList
183 183
184 184 tmp = []
185 185
186 186 for n, pair in enumerate(meta['pairs']):
187 187
188 188 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
189 189 coh = numpy.abs(out)
190 190 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
191 191 tmp.append(coh)
192 192 tmp.append(phase)
193 193
194 194 data['cspc'] = numpy.array(tmp)
195 195
196 196 return data, meta
197 197
198 198 def plot(self):
199 199
200 200 if self.xaxis == "frequency":
201 201 x = self.data.xrange[0]
202 202 self.xlabel = "Frequency (kHz)"
203 203 elif self.xaxis == "time":
204 204 x = self.data.xrange[1]
205 205 self.xlabel = "Time (ms)"
206 206 else:
207 207 x = self.data.xrange[2]
208 208 self.xlabel = "Velocity (m/s)"
209 209
210 210 self.titles = []
211 211
212 212 y = self.data.yrange
213 213 self.y = y
214 214
215 215 data = self.data[-1]
216 216 cspc = data['cspc']
217 217
218 218 for n in range(len(self.data.pairs)):
219 219
220 220 pair = self.crossPairs[n]
221 221
222 222 coh = cspc[n*2]
223 223 phase = cspc[n*2+1]
224 224 ax = self.axes[2 * n]
225 225
226 226 if ax.firsttime:
227 227 ax.plt = ax.pcolormesh(x, y, coh.T,
228 228 vmin=self.zmin_coh,
229 229 vmax=self.zmax_coh,
230 230 cmap=plt.get_cmap(self.colormap_coh)
231 231 )
232 232 else:
233 233 ax.plt.set_array(coh.T.ravel())
234 234 self.titles.append(
235 235 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
236 236
237 237 ax = self.axes[2 * n + 1]
238 238 if ax.firsttime:
239 239 ax.plt = ax.pcolormesh(x, y, phase.T,
240 240 vmin=-180,
241 241 vmax=180,
242 242 cmap=plt.get_cmap(self.colormap_phase)
243 243 )
244 244 else:
245 245 ax.plt.set_array(phase.T.ravel())
246 246
247 247 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
248 248
249 249
250 250 class RTIPlot(Plot):
251 251 '''
252 252 Plot for RTI data
253 253 '''
254 254
255 255 CODE = 'rti'
256 256 colormap = 'jet'
257 257 plot_type = 'pcolorbuffer'
258 258 titles = None
259 259 channelList = []
260 260 elevationList = []
261 261 azimuthList = []
262 262
263 263 def setup(self):
264 264 self.xaxis = 'time'
265 265 self.ncols = 1
266 266 #print("dataChannels ",self.data.channels)
267 267 self.nrows = len(self.data.channels)
268 268 self.nplots = len(self.data.channels)
269 269 self.ylabel = 'Range [km]'
270 270 self.xlabel = 'Time'
271 271 self.cb_label = 'dB'
272 272 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
273 273 self.titles = ['{} Channel {}'.format(
274 274 self.CODE.upper(), x) for x in range(self.nplots)]
275 275
276 276 def update_list(self,dataOut):
277 277
278 278 if len(self.channelList) == 0:
279 279 self.channelList = dataOut.channelList
280 280 if len(self.elevationList) == 0:
281 281 self.elevationList = dataOut.elevationList
282 282 if len(self.azimuthList) == 0:
283 283 self.azimuthList = dataOut.azimuthList
284 284
285 285
286 286 def update(self, dataOut):
287 287 if len(self.channelList) == 0:
288 288 self.update_list(dataOut)
289 289 data = {}
290 290 meta = {}
291 291
292 292 data['rti'] = dataOut.getPower()
293 293
294 294 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
295 295 noise = 10*numpy.log10(dataOut.getNoise()/norm)
296 296 data['noise'] = noise
297 297
298 298 return data, meta
299 299
300 300 def plot(self):
301 301
302 302 self.x = self.data.times
303 303 self.y = self.data.yrange
304 304 #print(" x, y: ",self.x, self.y)
305 305 self.z = self.data[self.CODE]
306 306 self.z = numpy.array(self.z, dtype=float)
307 307 self.z = numpy.ma.masked_invalid(self.z)
308 308
309 309 try:
310 310 if self.channelList != None:
311 311 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
312 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
312 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
313 313 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
314 314 else:
315 self.titles = ['{} Channel {}'.format(
315 self.titles = ['{} Channel {}'.format(
316 316 self.CODE.upper(), x) for x in self.channelList]
317 317 except:
318 318 if self.channelList.any() != None:
319
320 self.titles = ['{} Channel {}'.format(
321 self.CODE.upper(), x) for x in self.channelList]
319 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
320 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
321 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
322 else:
323 self.titles = ['{} Channel {}'.format(
324 self.CODE.upper(), x) for x in self.channelList]
322 325
323 326 if self.decimation is None:
324 327 x, y, z = self.fill_gaps(self.x, self.y, self.z)
325 328 else:
326 329 x, y, z = self.fill_gaps(*self.decimate())
327 330
328 331 #dummy_var = self.axes #Extrañamente esto actualiza el valor axes
329 332 for n, ax in enumerate(self.axes):
330 333 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
331 334 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
332 335 data = self.data[-1]
333 336
334 337 if ax.firsttime:
335 338 ax.plt = ax.pcolormesh(x, y, z[n].T,
336 339 vmin=self.zmin,
337 340 vmax=self.zmax,
338 341 cmap=plt.get_cmap(self.colormap)
339 342 )
340 343 if self.showprofile:
341 344 ax.plot_profile = self.pf_axes[n].plot(data[self.CODE][n], self.y)[0]
342 345 if "noise" in self.data:
343 346
344 347 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
345 348 color="k", linestyle="dashed", lw=1)[0]
346 349 else:
347 350 ax.collections.remove(ax.collections[0])
348 351 ax.plt = ax.pcolormesh(x, y, z[n].T,
349 352 vmin=self.zmin,
350 353 vmax=self.zmax,
351 354 cmap=plt.get_cmap(self.colormap)
352 355 )
353 356 if self.showprofile:
354 357 ax.plot_profile.set_data(data[self.CODE][n], self.y)
355 358 if "noise" in self.data:
356 359 ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
357 360
358 361
359 362 class CoherencePlot(RTIPlot):
360 363 '''
361 364 Plot for Coherence data
362 365 '''
363 366
364 367 CODE = 'coh'
365 368
366 369 def setup(self):
367 370 self.xaxis = 'time'
368 371 self.ncols = 1
369 372 self.nrows = len(self.data.pairs)
370 373 self.nplots = len(self.data.pairs)
371 374 self.ylabel = 'Range [km]'
372 375 self.xlabel = 'Time'
373 376 self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1,'right':0.95})
374 377 if self.CODE == 'coh':
375 378 self.cb_label = ''
376 379 self.titles = [
377 380 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
378 381 else:
379 382 self.cb_label = 'Degrees'
380 383 self.titles = [
381 384 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
382 385
383 386 def update(self, dataOut):
384 387 self.update_list(dataOut)
385 388 data = {}
386 389 meta = {}
387 390 data['coh'] = dataOut.getCoherence()
388 391 meta['pairs'] = dataOut.pairsList
389 392
390 393
391 394 return data, meta
392 395
393 396 class PhasePlot(CoherencePlot):
394 397 '''
395 398 Plot for Phase map data
396 399 '''
397 400
398 401 CODE = 'phase'
399 402 colormap = 'seismic'
400 403
401 404 def update(self, dataOut):
402 405
403 406 data = {}
404 407 meta = {}
405 408 data['phase'] = dataOut.getCoherence(phase=True)
406 409 meta['pairs'] = dataOut.pairsList
407 410
408 411 return data, meta
409 412
410 413 class NoisePlot(Plot):
411 414 '''
412 415 Plot for noise
413 416 '''
414 417
415 418 CODE = 'noise'
416 419 plot_type = 'scatterbuffer'
417 420
418 421 def setup(self):
419 422 self.xaxis = 'time'
420 423 self.ncols = 1
421 424 self.nrows = 1
422 425 self.nplots = 1
423 426 self.ylabel = 'Intensity [dB]'
424 427 self.xlabel = 'Time'
425 428 self.titles = ['Noise']
426 429 self.colorbar = False
427 430 self.plots_adjust.update({'right': 0.85 })
428 431 #if not self.titles:
429 432 self.titles = ['Noise Plot']
430 433
431 434 def update(self, dataOut):
432 435
433 436 data = {}
434 437 meta = {}
435 438 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
436 439 noise = 10*numpy.log10(dataOut.getNoise())
437 440 noise = noise.reshape(dataOut.nChannels, 1)
438 441 data['noise'] = noise
439 442 meta['yrange'] = numpy.array([])
440 443
441 444 return data, meta
442 445
443 446 def plot(self):
444 447
445 448 x = self.data.times
446 449 xmin = self.data.min_time
447 450 xmax = xmin + self.xrange * 60 * 60
448 451 Y = self.data['noise']
449 452
450 453 if self.axes[0].firsttime:
451 454 if self.ymin is None: self.ymin = numpy.nanmin(Y) - 5
452 455 if self.ymax is None: self.ymax = numpy.nanmax(Y) + 5
453 456 for ch in self.data.channels:
454 457 y = Y[ch]
455 458 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
456 459 plt.legend(bbox_to_anchor=(1.18, 1.0))
457 460 else:
458 461 for ch in self.data.channels:
459 462 y = Y[ch]
460 463 self.axes[0].lines[ch].set_data(x, y)
461 464
462 465
463 466 class PowerProfilePlot(Plot):
464 467
465 468 CODE = 'pow_profile'
466 469 plot_type = 'scatter'
467 470
468 471 def setup(self):
469 472
470 473 self.ncols = 1
471 474 self.nrows = 1
472 475 self.nplots = 1
473 476 self.height = 4
474 477 self.width = 3
475 478 self.ylabel = 'Range [km]'
476 479 self.xlabel = 'Intensity [dB]'
477 480 self.titles = ['Power Profile']
478 481 self.colorbar = False
479 482
480 483 def update(self, dataOut):
481 484
482 485 data = {}
483 486 meta = {}
484 487 data[self.CODE] = dataOut.getPower()
485 488
486 489 return data, meta
487 490
488 491 def plot(self):
489 492
490 493 y = self.data.yrange
491 494 self.y = y
492 495
493 496 x = self.data[-1][self.CODE]
494 497
495 498 if self.xmin is None: self.xmin = numpy.nanmin(x)*0.9
496 499 if self.xmax is None: self.xmax = numpy.nanmax(x)*1.1
497 500
498 501 if self.axes[0].firsttime:
499 502 for ch in self.data.channels:
500 503 self.axes[0].plot(x[ch], y, lw=1, label='Ch{}'.format(ch))
501 504 plt.legend()
502 505 else:
503 506 for ch in self.data.channels:
504 507 self.axes[0].lines[ch].set_data(x[ch], y)
505 508
506 509
507 510 class SpectraCutPlot(Plot):
508 511
509 512 CODE = 'spc_cut'
510 513 plot_type = 'scatter'
511 514 buffering = False
512 515 heights = []
513 516 channelList = []
514 517 maintitle = "Spectra Cuts"
515 518 flag_setIndex = False
516 519
517 520 def setup(self):
518 521
519 522 self.nplots = len(self.data.channels)
520 523 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
521 524 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
522 525 self.width = 4.5 * self.ncols + 2.5
523 526 self.height = 4.8 * self.nrows
524 527 self.ylabel = 'Power [dB]'
525 528 self.colorbar = False
526 529 self.plots_adjust.update({'left':0.1, 'hspace':0.3, 'right': 0.9, 'bottom':0.08})
527 530
528 531 if len(self.selectedHeightsList) > 0:
529 532 self.maintitle = "Spectra Cut"# for %d km " %(int(self.selectedHeight))
530 533
531 534
532 535
533 536 def update(self, dataOut):
534 537 if len(self.channelList) == 0:
535 538 self.channelList = dataOut.channelList
536 539
537 540 self.heights = dataOut.heightList
538 541 #print("sels: ",self.selectedHeightsList)
539 542 if len(self.selectedHeightsList)>0 and not self.flag_setIndex:
540 543
541 544 for sel_height in self.selectedHeightsList:
542 545 index_list = numpy.where(self.heights >= sel_height)
543 546 index_list = index_list[0]
544 547 self.height_index.append(index_list[0])
545 548 #print("sels i:"", self.height_index)
546 549 self.flag_setIndex = True
547 550 #print(self.height_index)
548 551 data = {}
549 552 meta = {}
550 553
551 554 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter#*dataOut.nFFTPoints
552 555 n0 = 10*numpy.log10(dataOut.getNoise()/norm)
553 556 noise = numpy.repeat(n0,(dataOut.nFFTPoints*dataOut.nHeights)).reshape(dataOut.nChannels,dataOut.nFFTPoints,dataOut.nHeights)
554 557
555 558
556 559 z = []
557 560 for ch in range(dataOut.nChannels):
558 561 if hasattr(dataOut.normFactor,'shape'):
559 562 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
560 563 else:
561 564 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
562 565
563 566 z = numpy.asarray(z)
564 567 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
565 568 spc = 10*numpy.log10(z)
566 569
567 570
568 571 data['spc'] = spc - noise
569 572 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
570 573
571 574 return data, meta
572 575
573 576 def plot(self):
574 577 if self.xaxis == "frequency":
575 578 x = self.data.xrange[0][1:]
576 579 self.xlabel = "Frequency (kHz)"
577 580 elif self.xaxis == "time":
578 581 x = self.data.xrange[1]
579 582 self.xlabel = "Time (ms)"
580 583 else:
581 584 x = self.data.xrange[2]
582 585 self.xlabel = "Velocity (m/s)"
583 586
584 587 self.titles = []
585 588
586 589 y = self.data.yrange
587 590 z = self.data[-1]['spc']
588 591 #print(z.shape)
589 592 if len(self.height_index) > 0:
590 593 index = self.height_index
591 594 else:
592 595 index = numpy.arange(0, len(y), int((len(y))/9))
593 596 #print("inde x ", index, self.axes)
594 597
595 598 for n, ax in enumerate(self.axes):
596 599
597 600 if ax.firsttime:
598 601
599 602
600 603 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
601 604 self.xmin = self.xmin if self.xmin else -self.xmax
602 605 self.ymin = self.ymin if self.ymin else numpy.nanmin(z)
603 606 self.ymax = self.ymax if self.ymax else numpy.nanmax(z)
604 607
605 608
606 609 ax.plt = ax.plot(x, z[n, :, index].T)
607 610 labels = ['Range = {:2.1f}km'.format(y[i]) for i in index]
608 611 self.figures[0].legend(ax.plt, labels, loc='center right', prop={'size': 8})
609 612 ax.minorticks_on()
610 613 ax.grid(which='major', axis='both')
611 614 ax.grid(which='minor', axis='x')
612 615 else:
613 616 for i, line in enumerate(ax.plt):
614 617 line.set_data(x, z[n, :, index[i]])
615 618
616 619
617 620 self.titles.append('CH {}'.format(self.channelList[n]))
618 621 plt.suptitle(self.maintitle, fontsize=10)
619 622
620 623
621 624 class BeaconPhase(Plot):
622 625
623 626 __isConfig = None
624 627 __nsubplots = None
625 628
626 629 PREFIX = 'beacon_phase'
627 630
628 631 def __init__(self):
629 632 Plot.__init__(self)
630 633 self.timerange = 24*60*60
631 634 self.isConfig = False
632 635 self.__nsubplots = 1
633 636 self.counter_imagwr = 0
634 637 self.WIDTH = 800
635 638 self.HEIGHT = 400
636 639 self.WIDTHPROF = 120
637 640 self.HEIGHTPROF = 0
638 641 self.xdata = None
639 642 self.ydata = None
640 643
641 644 self.PLOT_CODE = BEACON_CODE
642 645
643 646 self.FTP_WEI = None
644 647 self.EXP_CODE = None
645 648 self.SUB_EXP_CODE = None
646 649 self.PLOT_POS = None
647 650
648 651 self.filename_phase = None
649 652
650 653 self.figfile = None
651 654
652 655 self.xmin = None
653 656 self.xmax = None
654 657
655 658 def getSubplots(self):
656 659
657 660 ncol = 1
658 661 nrow = 1
659 662
660 663 return nrow, ncol
661 664
662 665 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
663 666
664 667 self.__showprofile = showprofile
665 668 self.nplots = nplots
666 669
667 670 ncolspan = 7
668 671 colspan = 6
669 672 self.__nsubplots = 2
670 673
671 674 self.createFigure(id = id,
672 675 wintitle = wintitle,
673 676 widthplot = self.WIDTH+self.WIDTHPROF,
674 677 heightplot = self.HEIGHT+self.HEIGHTPROF,
675 678 show=show)
676 679
677 680 nrow, ncol = self.getSubplots()
678 681
679 682 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
680 683
681 684 def save_phase(self, filename_phase):
682 685 f = open(filename_phase,'w+')
683 686 f.write('\n\n')
684 687 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
685 688 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
686 689 f.close()
687 690
688 691 def save_data(self, filename_phase, data, data_datetime):
689 692 f=open(filename_phase,'a')
690 693 timetuple_data = data_datetime.timetuple()
691 694 day = str(timetuple_data.tm_mday)
692 695 month = str(timetuple_data.tm_mon)
693 696 year = str(timetuple_data.tm_year)
694 697 hour = str(timetuple_data.tm_hour)
695 698 minute = str(timetuple_data.tm_min)
696 699 second = str(timetuple_data.tm_sec)
697 700 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
698 701 f.close()
699 702
700 703 def plot(self):
701 704 log.warning('TODO: Not yet implemented...')
702 705
703 706 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
704 707 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
705 708 timerange=None,
706 709 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
707 710 server=None, folder=None, username=None, password=None,
708 711 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
709 712
710 713 if dataOut.flagNoData:
711 714 return dataOut
712 715
713 716 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
714 717 return
715 718
716 719 if pairsList == None:
717 720 pairsIndexList = dataOut.pairsIndexList[:10]
718 721 else:
719 722 pairsIndexList = []
720 723 for pair in pairsList:
721 724 if pair not in dataOut.pairsList:
722 725 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
723 726 pairsIndexList.append(dataOut.pairsList.index(pair))
724 727
725 728 if pairsIndexList == []:
726 729 return
727 730
728 731 # if len(pairsIndexList) > 4:
729 732 # pairsIndexList = pairsIndexList[0:4]
730 733
731 734 hmin_index = None
732 735 hmax_index = None
733 736
734 737 if hmin != None and hmax != None:
735 738 indexes = numpy.arange(dataOut.nHeights)
736 739 hmin_list = indexes[dataOut.heightList >= hmin]
737 740 hmax_list = indexes[dataOut.heightList <= hmax]
738 741
739 742 if hmin_list.any():
740 743 hmin_index = hmin_list[0]
741 744
742 745 if hmax_list.any():
743 746 hmax_index = hmax_list[-1]+1
744 747
745 748 x = dataOut.getTimeRange()
746 749
747 750 thisDatetime = dataOut.datatime
748 751
749 752 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
750 753 xlabel = "Local Time"
751 754 ylabel = "Phase (degrees)"
752 755
753 756 update_figfile = False
754 757
755 758 nplots = len(pairsIndexList)
756 759 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
757 760 phase_beacon = numpy.zeros(len(pairsIndexList))
758 761 for i in range(nplots):
759 762 pair = dataOut.pairsList[pairsIndexList[i]]
760 763 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
761 764 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
762 765 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
763 766 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
764 767 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
765 768
766 769 if dataOut.beacon_heiIndexList:
767 770 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
768 771 else:
769 772 phase_beacon[i] = numpy.average(phase)
770 773
771 774 if not self.isConfig:
772 775
773 776 nplots = len(pairsIndexList)
774 777
775 778 self.setup(id=id,
776 779 nplots=nplots,
777 780 wintitle=wintitle,
778 781 showprofile=showprofile,
779 782 show=show)
780 783
781 784 if timerange != None:
782 785 self.timerange = timerange
783 786
784 787 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
785 788
786 789 if ymin == None: ymin = 0
787 790 if ymax == None: ymax = 360
788 791
789 792 self.FTP_WEI = ftp_wei
790 793 self.EXP_CODE = exp_code
791 794 self.SUB_EXP_CODE = sub_exp_code
792 795 self.PLOT_POS = plot_pos
793 796
794 797 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
795 798 self.isConfig = True
796 799 self.figfile = figfile
797 800 self.xdata = numpy.array([])
798 801 self.ydata = numpy.array([])
799 802
800 803 update_figfile = True
801 804
802 805 #open file beacon phase
803 806 path = '%s%03d' %(self.PREFIX, self.id)
804 807 beacon_file = os.path.join(path,'%s.txt'%self.name)
805 808 self.filename_phase = os.path.join(figpath,beacon_file)
806 809 #self.save_phase(self.filename_phase)
807 810
808 811
809 812 #store data beacon phase
810 813 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
811 814
812 815 self.setWinTitle(title)
813 816
814 817
815 818 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
816 819
817 820 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
818 821
819 822 axes = self.axesList[0]
820 823
821 824 self.xdata = numpy.hstack((self.xdata, x[0:1]))
822 825
823 826 if len(self.ydata)==0:
824 827 self.ydata = phase_beacon.reshape(-1,1)
825 828 else:
826 829 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
827 830
828 831
829 832 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
830 833 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
831 834 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
832 835 XAxisAsTime=True, grid='both'
833 836 )
834 837
835 838 self.draw()
836 839
837 840 if dataOut.ltctime >= self.xmax:
838 841 self.counter_imagwr = wr_period
839 842 self.isConfig = False
840 843 update_figfile = True
841 844
842 845 self.save(figpath=figpath,
843 846 figfile=figfile,
844 847 save=save,
845 848 ftp=ftp,
846 849 wr_period=wr_period,
847 850 thisDatetime=thisDatetime,
848 851 update_figfile=update_figfile)
849 852
850 853 return dataOut
851 854
852 855 class NoiselessSpectraPlot(Plot):
853 856 '''
854 857 Plot for Spectra data, subtracting
855 858 the noise in all channels, using for
856 859 amisr-14 data
857 860 '''
858 861
859 862 CODE = 'noiseless_spc'
860 863 colormap = 'jet'
861 864 plot_type = 'pcolor'
862 865 buffering = False
863 866 channelList = []
864 867 last_noise = None
865 868
866 869 def setup(self):
867 870
868 871 self.nplots = len(self.data.channels)
869 872 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
870 873 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
871 874 self.height = 3.5 * self.nrows
872 875
873 876 self.cb_label = 'dB'
874 877 if self.showprofile:
875 878 self.width = 5.8 * self.ncols
876 879 else:
877 880 self.width = 4.8* self.ncols
878 881 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.92, 'bottom': 0.12})
879 882
880 883 self.ylabel = 'Range [km]'
881 884
882 885
883 886 def update_list(self,dataOut):
884 887 if len(self.channelList) == 0:
885 888 self.channelList = dataOut.channelList
886 889
887 890 def update(self, dataOut):
888 891
889 892 self.update_list(dataOut)
890 893 data = {}
891 894 meta = {}
892 895
893 896 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter#*dataOut.nFFTPoints
894 897 n0 = 10*numpy.log10(dataOut.getNoise()/norm)
895 898 noise = numpy.repeat(n0,(dataOut.nFFTPoints*dataOut.nHeights)).reshape(dataOut.nChannels,dataOut.nFFTPoints,dataOut.nHeights)
896 899
897 900 z = []
898 901 for ch in range(dataOut.nChannels):
899 902 if hasattr(dataOut.normFactor,'shape'):
900 903 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
901 904 else:
902 905 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
903 906
904 907 z = numpy.asarray(z)
905 908 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
906 909 spc = 10*numpy.log10(z)
907 910
908 911
909 912 data['spc'] = spc - noise
910 913 #print(spc.shape)
911 914 data['rti'] = spc.mean(axis=1)
912 915 data['noise'] = noise
913 916
914 917
915 918
916 919 # data['noise'] = noise
917 920 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
918 921
919 922 return data, meta
920 923
921 924 def plot(self):
922 925 if self.xaxis == "frequency":
923 926 x = self.data.xrange[0]
924 927 self.xlabel = "Frequency (kHz)"
925 928 elif self.xaxis == "time":
926 929 x = self.data.xrange[1]
927 930 self.xlabel = "Time (ms)"
928 931 else:
929 932 x = self.data.xrange[2]
930 933 self.xlabel = "Velocity (m/s)"
931 934
932 935 self.titles = []
933 936 y = self.data.yrange
934 937 self.y = y
935 938
936 939 data = self.data[-1]
937 940 z = data['spc']
938 941
939 942 for n, ax in enumerate(self.axes):
940 943 #noise = data['noise'][n]
941 944
942 945 if ax.firsttime:
943 946 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
944 947 self.xmin = self.xmin if self.xmin else -self.xmax
945 948 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
946 949 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
947 950 ax.plt = ax.pcolormesh(x, y, z[n].T,
948 951 vmin=self.zmin,
949 952 vmax=self.zmax,
950 953 cmap=plt.get_cmap(self.colormap)
951 954 )
952 955
953 956 if self.showprofile:
954 957 ax.plt_profile = self.pf_axes[n].plot(
955 958 data['rti'][n], y)[0]
956 959
957 960
958 961 else:
959 962 ax.plt.set_array(z[n].T.ravel())
960 963 if self.showprofile:
961 964 ax.plt_profile.set_data(data['rti'][n], y)
962 965
963 966
964 967 self.titles.append('CH {}'.format(self.channelList[n]))
965 968
966 969
967 970 class NoiselessRTIPlot(Plot):
968 971 '''
969 972 Plot for RTI data
970 973 '''
971 974
972 975 CODE = 'noiseless_rti'
973 976 colormap = 'jet'
974 977 plot_type = 'pcolorbuffer'
975 978 titles = None
976 979 channelList = []
977 980 elevationList = []
978 981 azimuthList = []
979 982 last_noise = None
980 983
981 984 def setup(self):
982 985 self.xaxis = 'time'
983 986 self.ncols = 1
984 987 #print("dataChannels ",self.data.channels)
985 988 self.nrows = len(self.data.channels)
986 989 self.nplots = len(self.data.channels)
987 990 self.ylabel = 'Range [km]'
988 991 self.xlabel = 'Time'
989 992 self.cb_label = 'dB'
990 993 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
991 994 self.titles = ['{} Channel {}'.format(
992 995 self.CODE.upper(), x) for x in range(self.nplots)]
993 996
994 997 def update_list(self,dataOut):
995 998 if len(self.channelList) == 0:
996 999 self.channelList = dataOut.channelList
997 1000 if len(self.elevationList) == 0:
998 1001 self.elevationList = dataOut.elevationList
999 1002 if len(self.azimuthList) == 0:
1000 1003 self.azimuthList = dataOut.azimuthList
1001 1004
1002 1005 def update(self, dataOut):
1003 1006 if len(self.channelList) == 0:
1004 1007 self.update_list(dataOut)
1008
1005 1009 data = {}
1006 1010 meta = {}
1007 1011 #print(dataOut.max_nIncohInt, dataOut.nIncohInt)
1008 1012 #print(dataOut.windowOfFilter,dataOut.nCohInt,dataOut.nProfiles,dataOut.max_nIncohInt,dataOut.nIncohInt)
1009 1013 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
1010 1014
1011 1015
1012 1016 n0 = 10*numpy.log10(dataOut.getNoise()/norm)
1013 1017
1014 1018 data['noise'] = n0
1015 1019 noise = numpy.repeat(n0,dataOut.nHeights).reshape(dataOut.nChannels,dataOut.nHeights)
1016 1020
1017 1021 data['noiseless_rti'] = dataOut.getPower() - noise
1018 1022
1019 1023 return data, meta
1020 1024
1021 1025 def plot(self):
1022 1026
1023 1027 self.x = self.data.times
1024 1028 self.y = self.data.yrange
1025 1029 self.z = self.data['noiseless_rti']
1026 1030 self.z = numpy.array(self.z, dtype=float)
1027 1031 self.z = numpy.ma.masked_invalid(self.z)
1028 1032
1029 1033
1030 1034 try:
1031 1035 if self.channelList != None:
1032 1036 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
1033 1037 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
1034 1038 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
1035 1039 else:
1036 1040 self.titles = ['{} Channel {}'.format(
1037 1041 self.CODE.upper(), x) for x in self.channelList]
1038 1042 except:
1039 1043 if self.channelList.any() != None:
1040
1041 self.titles = ['{} Channel {}'.format(
1042 self.CODE.upper(), x) for x in self.channelList]
1044 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
1045 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
1046 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
1047 else:
1048 self.titles = ['{} Channel {}'.format(
1049 self.CODE.upper(), x) for x in self.channelList]
1043 1050
1044 1051
1045 1052 if self.decimation is None:
1046 1053 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1047 1054 else:
1048 1055 x, y, z = self.fill_gaps(*self.decimate())
1049 1056 dummy_var = self.axes #Extrañamente esto actualiza el valor axes
1050 1057 #print("plot shapes ", z.shape, x.shape, y.shape)
1051 1058 for n, ax in enumerate(self.axes):
1052 1059
1053 1060
1054 1061 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
1055 1062 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
1056 1063 data = self.data[-1]
1057 1064 if ax.firsttime:
1058 1065 ax.plt = ax.pcolormesh(x, y, z[n].T,
1059 1066 vmin=self.zmin,
1060 1067 vmax=self.zmax,
1061 1068 cmap=plt.get_cmap(self.colormap)
1062 1069 )
1063 1070 if self.showprofile:
1064 1071 ax.plot_profile = self.pf_axes[n].plot(data['noiseless_rti'][n], self.y)[0]
1065 1072
1066 1073 else:
1067 1074 ax.collections.remove(ax.collections[0])
1068 1075 ax.plt = ax.pcolormesh(x, y, z[n].T,
1069 1076 vmin=self.zmin,
1070 1077 vmax=self.zmax,
1071 1078 cmap=plt.get_cmap(self.colormap)
1072 1079 )
1073 1080 if self.showprofile:
1074 1081 ax.plot_profile.set_data(data['noiseless_rti'][n], self.y)
1075 1082 # if "noise" in self.data:
1076 1083 # #ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
1077 1084 # ax.plot_noise.set_data(data['noise'][n], self.y)
1078 1085
1079 1086
1080 1087 class OutliersRTIPlot(Plot):
1081 1088 '''
1082 1089 Plot for data_xxxx object
1083 1090 '''
1084 1091
1085 1092 CODE = 'outlier_rtc' # Range Time Counts
1086 1093 colormap = 'cool'
1087 1094 plot_type = 'pcolorbuffer'
1088 1095
1089 1096 def setup(self):
1090 1097 self.xaxis = 'time'
1091 1098 self.ncols = 1
1092 1099 self.nrows = self.data.shape('outlier_rtc')[0]
1093 1100 self.nplots = self.nrows
1094 1101 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1095 1102
1096 1103
1097 1104 if not self.xlabel:
1098 1105 self.xlabel = 'Time'
1099 1106
1100 1107 self.ylabel = 'Height [km]'
1101 1108 if not self.titles:
1102 1109 self.titles = ['Outliers Ch:{}'.format(x) for x in range(self.nrows)]
1103 1110
1104 1111 def update(self, dataOut):
1105 1112
1106 1113 data = {}
1107 1114 data['outlier_rtc'] = dataOut.data_outlier
1108 1115
1109 1116 meta = {}
1110 1117
1111 1118 return data, meta
1112 1119
1113 1120 def plot(self):
1114 1121 # self.data.normalize_heights()
1115 1122 self.x = self.data.times
1116 1123 self.y = self.data.yrange
1117 1124 self.z = self.data['outlier_rtc']
1118 1125
1119 1126 #self.z = numpy.ma.masked_invalid(self.z)
1120 1127
1121 1128 if self.decimation is None:
1122 1129 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1123 1130 else:
1124 1131 x, y, z = self.fill_gaps(*self.decimate())
1125 1132
1126 1133 for n, ax in enumerate(self.axes):
1127 1134
1128 1135 self.zmax = self.zmax if self.zmax is not None else numpy.max(
1129 1136 self.z[n])
1130 1137 self.zmin = self.zmin if self.zmin is not None else numpy.min(
1131 1138 self.z[n])
1132 1139 data = self.data[-1]
1133 1140 if ax.firsttime:
1134 1141 if self.zlimits is not None:
1135 1142 self.zmin, self.zmax = self.zlimits[n]
1136 1143
1137 1144 ax.plt = ax.pcolormesh(x, y, z[n].T,
1138 1145 vmin=self.zmin,
1139 1146 vmax=self.zmax,
1140 1147 cmap=self.cmaps[n]
1141 1148 )
1142 1149 if self.showprofile:
1143 1150 ax.plot_profile = self.pf_axes[n].plot(data['outlier_rtc'][n], self.y)[0]
1144 1151 self.pf_axes[n].set_xlabel('')
1145 1152 else:
1146 1153 if self.zlimits is not None:
1147 1154 self.zmin, self.zmax = self.zlimits[n]
1148 1155 ax.collections.remove(ax.collections[0])
1149 1156 ax.plt = ax.pcolormesh(x, y, z[n].T ,
1150 1157 vmin=self.zmin,
1151 1158 vmax=self.zmax,
1152 1159 cmap=self.cmaps[n]
1153 1160 )
1154 1161 if self.showprofile:
1155 1162 ax.plot_profile.set_data(data['outlier_rtc'][n], self.y)
1156 1163 self.pf_axes[n].set_xlabel('')
1157 1164
1158 1165 class NIncohIntRTIPlot(Plot):
1159 1166 '''
1160 1167 Plot for data_xxxx object
1161 1168 '''
1162 1169
1163 1170 CODE = 'integrations_rtc' # Range Time Counts
1164 1171 colormap = 'BuGn'
1165 1172 plot_type = 'pcolorbuffer'
1166 1173
1167 1174 def setup(self):
1168 1175 self.xaxis = 'time'
1169 1176 self.ncols = 1
1170 1177 self.nrows = self.data.shape('integrations_rtc')[0]
1171 1178 self.nplots = self.nrows
1172 1179 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1173 1180
1174 1181
1175 1182 if not self.xlabel:
1176 1183 self.xlabel = 'Time'
1177 1184
1178 1185 self.ylabel = 'Height [km]'
1179 1186 if not self.titles:
1180 1187 self.titles = ['Integration Ch:{}'.format(x) for x in range(self.nrows)]
1181 1188
1182 1189 def update(self, dataOut):
1183 1190
1184 1191 data = {}
1185 1192 data['integrations_rtc'] = dataOut.nIncohInt
1186 1193
1187 1194 meta = {}
1188 1195
1189 1196 return data, meta
1190 1197
1191 1198 def plot(self):
1192 1199 # self.data.normalize_heights()
1193 1200 self.x = self.data.times
1194 1201 self.y = self.data.yrange
1195 1202 self.z = self.data['integrations_rtc']
1196 1203
1197 1204 #self.z = numpy.ma.masked_invalid(self.z)
1198 1205
1199 1206 if self.decimation is None:
1200 1207 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1201 1208 else:
1202 1209 x, y, z = self.fill_gaps(*self.decimate())
1203 1210
1204 1211 for n, ax in enumerate(self.axes):
1205 1212
1206 1213 self.zmax = self.zmax if self.zmax is not None else numpy.max(
1207 1214 self.z[n])
1208 1215 self.zmin = self.zmin if self.zmin is not None else numpy.min(
1209 1216 self.z[n])
1210 1217 data = self.data[-1]
1211 1218 if ax.firsttime:
1212 1219 if self.zlimits is not None:
1213 1220 self.zmin, self.zmax = self.zlimits[n]
1214 1221
1215 1222 ax.plt = ax.pcolormesh(x, y, z[n].T,
1216 1223 vmin=self.zmin,
1217 1224 vmax=self.zmax,
1218 1225 cmap=self.cmaps[n]
1219 1226 )
1220 1227 if self.showprofile:
1221 1228 ax.plot_profile = self.pf_axes[n].plot(data['integrations_rtc'][n], self.y)[0]
1222 1229 self.pf_axes[n].set_xlabel('')
1223 1230 else:
1224 1231 if self.zlimits is not None:
1225 1232 self.zmin, self.zmax = self.zlimits[n]
1226 1233 ax.collections.remove(ax.collections[0])
1227 1234 ax.plt = ax.pcolormesh(x, y, z[n].T ,
1228 1235 vmin=self.zmin,
1229 1236 vmax=self.zmax,
1230 1237 cmap=self.cmaps[n]
1231 1238 )
1232 1239 if self.showprofile:
1233 1240 ax.plot_profile.set_data(data['integrations_rtc'][n], self.y)
1234 1241 self.pf_axes[n].set_xlabel('')
Show file before | Show file after | Hide comments
@@ -1,698 +1,725
1 1 ''''
2 2 Created on Set 9, 2015
3 3
4 4 @author: roj-idl71 Karim Kuyeng
5 5
6 6 @update: 2021, Joab Apaza
7 7 '''
8 8
9 9 import os
10 10 import sys
11 11 import glob
12 12 import fnmatch
13 13 import datetime
14 14 import time
15 15 import re
16 16 import h5py
17 17 import numpy
18 18
19 19 try:
20 20 from gevent import sleep
21 21 except:
22 22 from time import sleep
23 23
24 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
24 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader,ProcessingHeader
25 25 from schainpy.model.data.jrodata import Voltage
26 26 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
27 27 from numpy import imag
28 28 from schainpy.utils import log
29 29
30 30
31 31 class AMISRReader(ProcessingUnit):
32 32 '''
33 33 classdocs
34 34 '''
35 35
36 36 def __init__(self):
37 37 '''
38 38 Constructor
39 39 '''
40 40
41 41 ProcessingUnit.__init__(self)
42 42
43 43 self.set = None
44 44 self.subset = None
45 45 self.extension_file = '.h5'
46 46 self.dtc_str = 'dtc'
47 47 self.dtc_id = 0
48 48 self.status = True
49 49 self.isConfig = False
50 50 self.dirnameList = []
51 51 self.filenameList = []
52 52 self.fileIndex = None
53 53 self.flagNoMoreFiles = False
54 54 self.flagIsNewFile = 0
55 55 self.filename = ''
56 56 self.amisrFilePointer = None
57 57
58 58 self.beamCodeMap = None
59 59 self.azimuthList = []
60 60 self.elevationList = []
61 61 self.dataShape = None
62 62 self.flag_old_beams = False
63 63
64 64
65 65 self.profileIndex = 0
66 66
67 67
68 68 self.beamCodeByFrame = None
69 69 self.radacTimeByFrame = None
70 70
71 71 self.dataset = None
72 72
73 73 self.__firstFile = True
74 74
75 75 self.buffer = None
76 76
77 77 self.timezone = 'ut'
78 78
79 79 self.__waitForNewFile = 20
80 80 self.__filename_online = None
81 81 #Is really necessary create the output object in the initializer
82 82 self.dataOut = Voltage()
83 83 self.dataOut.error=False
84 84 self.margin_days = 1
85 85
86 86 def setup(self,path=None,
87 87 startDate=None,
88 88 endDate=None,
89 89 startTime=None,
90 90 endTime=None,
91 91 walk=True,
92 92 timezone='ut',
93 93 all=0,
94 94 code = None,
95 95 nCode = 1,
96 96 nBaud = 0,
97 nOsamp = None,
97 98 online=False,
98 99 old_beams=False,
99 100 margin_days=1,
100 nFFT = 1,
101 nFFT = None,
101 102 nChannels = None,
102 103 ):
103 104
104 105
105 106
106 107 self.timezone = timezone
107 108 self.all = all
108 109 self.online = online
109 110 self.flag_old_beams = old_beams
110 111 self.code = code
111 112 self.nCode = int(nCode)
112 113 self.nBaud = int(nBaud)
114 self.nOsamp = int(nOsamp)
113 115 self.margin_days = margin_days
114 116 self.__sampleRate = None
115 117
116 118 self.nFFT = nFFT
117 119 self.nChannels = nChannels
118 120
119 121 #self.findFiles()
120 122 if not(online):
121 123 #Busqueda de archivos offline
122 124 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
123 125 else:
124 126 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
125 127
126 128 if not(self.filenameList):
127 129 raise schainpy.admin.SchainWarning("There is no files into the folder: %s"%(path))
128 130 #sys.exit(0)
129 131 self.dataOut.error = True
130 132
131 133 self.fileIndex = 0
132 134
133 135 self.readNextFile(online)
134 136
135 137 '''
136 138 Add code
137 139 '''
138 140 self.isConfig = True
139 141 # print("Setup Done")
140 142 pass
141 143
142 144
143 145 def readAMISRHeader(self,fp):
144 146
145 147 if self.isConfig and (not self.flagNoMoreFiles):
146 148 newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
147 149 if self.dataShape != newShape and newShape != None:
148 150 raise schainpy.admin.SchainError("NEW FILE HAS A DIFFERENT SHAPE: ")
149 151 print(self.dataShape,newShape,"\n")
150 152 return 0
151 153 else:
152 154 self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
153 155
154 156
155 157 header = 'Raw11/Data/RadacHeader'
156 158 if self.nChannels == None:
157 expFile = fp['Setup/ExperimentFile'][()].decode()
159 expFile = fp['Setup/Experimentfile'][()].decode()
158 160 linesExp = expFile.split("\n")
159 161 a = [line for line in linesExp if "nbeamcodes" in line]
160 162 self.nChannels = int(a[0][11:])
161 163
162 164 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
163 165 if (self.startDate > datetime.date(2021, 7, 15)) or self.flag_old_beams: #Se cambió la forma de extracción de Apuntes el 17 o forzar con flag de reorganización
164 166 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
165 167 self.trueBeams = self.beamcodeFile.split("\n")
166 168 self.trueBeams.pop()#remove last
169 if self.nFFT == None:
170 log.error("FFT or number of repetitions per channels is needed",self.name)
167 171 beams_idx = [k*self.nFFT for k in range(self.nChannels)]
168 172 beams = [self.trueBeams[b] for b in beams_idx]
169 173 self.beamCode = [int(x, 16) for x in beams]
170 174
171 175 else:
172 176 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
173 177 self.beamCode = _beamCode[0,:]
174 178
175 179 if self.beamCodeMap == None:
176 180 self.beamCodeMap = fp['Setup/BeamcodeMap']
177 181 for beam in self.beamCode:
178 182 beamAziElev = numpy.where(self.beamCodeMap[:,0]==beam)
179 183 beamAziElev = beamAziElev[0].squeeze()
180 184 self.azimuthList.append(self.beamCodeMap[beamAziElev,1])
181 185 self.elevationList.append(self.beamCodeMap[beamAziElev,2])
182 186 #print("Beamssss: ",self.beamCodeMap[beamAziElev,1],self.beamCodeMap[beamAziElev,2])
183 187 #print(self.beamCode)
184 188 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
185 189 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
186 190 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
187 191 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
188 192 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
189 193 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
190 194 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
191 195 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
192 196 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
193 197 self.frequency = fp.get('Rx/Frequency')
194 txAus = fp.get('Raw11/Data/Pulsewidth')
198 txAus = fp.get('Raw11/Data/Pulsewidth') #seconds
195 199 self.baud = fp.get('Raw11/Data/TxBaud')
196 200 sampleRate = fp.get('Rx/SampleRate')
197 201 self.__sampleRate = sampleRate[()]
198 202 self.nblocks = self.pulseCount.shape[0] #nblocks
199 203 self.profPerBlockRAW = self.pulseCount.shape[1] #profiles per block in raw data
200 204 self.nprofiles = self.pulseCount.shape[1] #nprofile
201 205 self.nsa = self.nsamplesPulse[0,0] #ngates
202 206 self.nchannels = len(self.beamCode)
203 207 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
204 208 #print("IPPS secs: ",self.ippSeconds)
205 209 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
206 210 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
207 211
208 212 #filling radar controller header parameters
209 213 self.__ippKm = self.ippSeconds *.15*1e6 # in km
210 self.__txA = (txAus[()])*.15 #(ipp[us]*.15km/1us) in km
214 #self.__txA = txAus[()]*.15 #(ipp[us]*.15km/1us) in km
215 self.__txA = txAus[()] #seconds
216 self.__txAKm = self.__txA*1e6*.15
211 217 self.__txB = 0
212 218 nWindows=1
213 219 self.__nSamples = self.nsa
214 220 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
215 221 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
216 222 #print("amisr-ipp:",self.ippSeconds, self.__ippKm)
217 223 #for now until understand why the code saved is different (code included even though code not in tuf file)
218 224 #self.__codeType = 0
219 225 # self.__nCode = None
220 226 # self.__nBaud = None
221 227 self.__code = self.code
222 228 self.__codeType = 0
223 229 if self.code != None:
224 230 self.__codeType = 1
225 231 self.__nCode = self.nCode
226 232 self.__nBaud = self.nBaud
233 self.__baudTX = self.__txA/(self.nBaud)
227 234 #self.__code = 0
228 235
229 236 #filling system header parameters
230 237 self.__nSamples = self.nsa
231 238 self.newProfiles = self.nprofiles/self.nchannels
232 239 self.__channelList = [n for n in range(self.nchannels)]
233 240
234 241 self.__frequency = self.frequency[0][0]
235 242
236 243
237 244 return 1
238 245
239 246
240 247 def createBuffers(self):
241 248
242 249 pass
243 250
244 251 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
245 252 self.path = path
246 253 self.startDate = startDate
247 254 self.endDate = endDate
248 255 self.startTime = startTime
249 256 self.endTime = endTime
250 257 self.walk = walk
251 258
252 259 def __checkPath(self):
253 260 if os.path.exists(self.path):
254 261 self.status = 1
255 262 else:
256 263 self.status = 0
257 264 print('Path:%s does not exists'%self.path)
258 265
259 266 return
260 267
261 268
262 269 def __selDates(self, amisr_dirname_format):
263 270 try:
264 271 year = int(amisr_dirname_format[0:4])
265 272 month = int(amisr_dirname_format[4:6])
266 273 dom = int(amisr_dirname_format[6:8])
267 274 thisDate = datetime.date(year,month,dom)
268 275 #margen de un día extra, igual luego se filtra for fecha y hora
269 276 if (thisDate>=(self.startDate - datetime.timedelta(days=self.margin_days)) and thisDate <= (self.endDate)+ datetime.timedelta(days=1)):
270 277 return amisr_dirname_format
271 278 except:
272 279 return None
273 280
274 281
275 282 def __findDataForDates(self,online=False):
276 283
277 284 if not(self.status):
278 285 return None
279 286
280 287 pat = '\d+.\d+'
281 288 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
282 289 dirnameList = [x for x in dirnameList if x!=None]
283 290 dirnameList = [x.string for x in dirnameList]
284 291 if not(online):
285 292 dirnameList = [self.__selDates(x) for x in dirnameList]
286 293 dirnameList = [x for x in dirnameList if x!=None]
287 294 if len(dirnameList)>0:
288 295 self.status = 1
289 296 self.dirnameList = dirnameList
290 297 self.dirnameList.sort()
291 298 else:
292 299 self.status = 0
293 300 return None
294 301
295 302 def __getTimeFromData(self):
296 303 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
297 304 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
298 305
299 306 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
300 307 print('........................................')
301 308 filter_filenameList = []
302 309 self.filenameList.sort()
303 310 total_files = len(self.filenameList)
304 311 #for i in range(len(self.filenameList)-1):
305 312 for i in range(total_files):
306 313 filename = self.filenameList[i]
307 314 #print("file-> ",filename)
308 315 try:
309 316 fp = h5py.File(filename,'r')
310 317 time_str = fp.get('Time/RadacTimeString')
311 318
312 319 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
313 320 #startDateTimeStr_File = "2019-12-16 09:21:11"
314 321 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
315 322 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
316 323
317 324 #endDateTimeStr_File = "2019-12-16 11:10:11"
318 325 endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
319 326 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
320 327 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
321 328
322 329 fp.close()
323 330
324 331 #print("check time", startDateTime_File)
325 332 if self.timezone == 'lt':
326 333 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
327 334 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
328 335 if (startDateTime_File >=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
329 336 filter_filenameList.append(filename)
330 337
331 338 if (startDateTime_File>endDateTime_Reader):
332 339 break
333 340 except Exception as e:
334 341 log.warning("Error opening file {} -> {}".format(os.path.split(filename)[1],e))
335 342
336 343 filter_filenameList.sort()
337 344 self.filenameList = filter_filenameList
338 345
339 346 return 1
340 347
341 348 def __filterByGlob1(self, dirName):
342 349 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
343 350 filter_files.sort()
344 351 filterDict = {}
345 352 filterDict.setdefault(dirName)
346 353 filterDict[dirName] = filter_files
347 354 return filterDict
348 355
349 356 def __getFilenameList(self, fileListInKeys, dirList):
350 357 for value in fileListInKeys:
351 358 dirName = list(value.keys())[0]
352 359 for file in value[dirName]:
353 360 filename = os.path.join(dirName, file)
354 361 self.filenameList.append(filename)
355 362
356 363
357 364 def __selectDataForTimes(self, online=False):
358 365 #aun no esta implementado el filtro for tiempo
359 366 if not(self.status):
360 367 return None
361 368
362 369 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
363 370 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
364 371 self.__getFilenameList(fileListInKeys, dirList)
365 372 if not(online):
366 373 #filtro por tiempo
367 374 if not(self.all):
368 375 self.__getTimeFromData()
369 376
370 377 if len(self.filenameList)>0:
371 378 self.status = 1
372 379 self.filenameList.sort()
373 380 else:
374 381 self.status = 0
375 382 return None
376 383
377 384 else:
378 385 #get the last file - 1
379 386 self.filenameList = [self.filenameList[-2]]
380 387 new_dirnameList = []
381 388 for dirname in self.dirnameList:
382 389 junk = numpy.array([dirname in x for x in self.filenameList])
383 390 junk_sum = junk.sum()
384 391 if junk_sum > 0:
385 392 new_dirnameList.append(dirname)
386 393 self.dirnameList = new_dirnameList
387 394 return 1
388 395
389 396 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
390 397 endTime=datetime.time(23,59,59),walk=True):
391 398
392 399 if endDate ==None:
393 400 startDate = datetime.datetime.utcnow().date()
394 401 endDate = datetime.datetime.utcnow().date()
395 402
396 403 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
397 404
398 405 self.__checkPath()
399 406
400 407 self.__findDataForDates(online=True)
401 408
402 409 self.dirnameList = [self.dirnameList[-1]]
403 410
404 411 self.__selectDataForTimes(online=True)
405 412
406 413 return
407 414
408 415
409 416 def searchFilesOffLine(self,
410 417 path,
411 418 startDate,
412 419 endDate,
413 420 startTime=datetime.time(0,0,0),
414 421 endTime=datetime.time(23,59,59),
415 422 walk=True):
416 423
417 424 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
418 425
419 426 self.__checkPath()
420 427
421 428 self.__findDataForDates()
422 429
423 430 self.__selectDataForTimes()
424 431
425 432 for i in range(len(self.filenameList)):
426 433 print("%s" %(self.filenameList[i]))
427 434
428 435 return
429 436
430 437 def __setNextFileOffline(self):
431 438
432 439 try:
433 440 self.filename = self.filenameList[self.fileIndex]
434 441 self.amisrFilePointer = h5py.File(self.filename,'r')
435 442 self.fileIndex += 1
436 443 except:
437 444 self.flagNoMoreFiles = 1
438 445 raise schainpy.admin.SchainError('No more files to read')
439 446 return 0
440 447
441 448 self.flagIsNewFile = 1
442 449 print("Setting the file: %s"%self.filename)
443 450
444 451 return 1
445 452
446 453
447 454 def __setNextFileOnline(self):
448 455 filename = self.filenameList[0]
449 456 if self.__filename_online != None:
450 457 self.__selectDataForTimes(online=True)
451 458 filename = self.filenameList[0]
452 459 wait = 0
453 460 self.__waitForNewFile=300 ## DEBUG:
454 461 while self.__filename_online == filename:
455 462 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
456 463 if wait == 5:
457 464 self.flagNoMoreFiles = 1
458 465 return 0
459 466 sleep(self.__waitForNewFile)
460 467 self.__selectDataForTimes(online=True)
461 468 filename = self.filenameList[0]
462 469 wait += 1
463 470
464 471 self.__filename_online = filename
465 472
466 473 self.amisrFilePointer = h5py.File(filename,'r')
467 474 self.flagIsNewFile = 1
468 475 self.filename = filename
469 476 print("Setting the file: %s"%self.filename)
470 477 return 1
471 478
472 479
473 480 def readData(self):
474 481 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
475 482 re = buffer[:,:,:,0]
476 483 im = buffer[:,:,:,1]
477 484 dataset = re + im*1j
478 485
479 486 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
480 487 timeset = self.radacTime[:,0]
481 488
482 489 return dataset,timeset
483 490
484 491 def reshapeData(self):
485 492 #print(self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa)
486 493 channels = self.beamCodeByPulse[0,:]
487 494 nchan = self.nchannels
488 495 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
489 496 nblocks = self.nblocks
490 497 nsamples = self.nsa
491 498 #print("Channels: ",self.nChannels)
492 499 #Dimensions : nChannels, nProfiles, nSamples
493 500 new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
494 501 ############################################
495 502 profPerCH = int(self.profPerBlockRAW / (self.nFFT* self.nChannels))
496
503 #profPerCH = int(self.profPerBlockRAW / self.nChannels)
497 504 for thisChannel in range(nchan):
498 505
499 idx_ch = [thisChannel+(k*nchan*self.nFFT) for k in range(profPerCH)]
506
507 idx_ch = [self.nFFT*(thisChannel + nchan*k) for k in range(profPerCH)]
508 #print(idx_ch)
500 509 if self.nFFT > 1:
501 510 aux = [numpy.arange(i, i+self.nFFT) for i in idx_ch]
502 511 idx_ch = None
503 512 idx_ch =aux
504 513 idx_ch = numpy.array(idx_ch, dtype=int).flatten()
505 514 else:
506 515 idx_ch = numpy.array(idx_ch, dtype=int)
507 516
508 517 #print(thisChannel,profPerCH,idx_ch)
509 518 #print(numpy.where(channels==self.beamCode[thisChannel])[0])
510 519 #new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
511 520 new_block[:,thisChannel,:,:] = self.dataset[:,idx_ch,:]
512 521
513 522 new_block = numpy.transpose(new_block, (1,0,2,3))
514 523 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
515 524
516 525 return new_block
517 526
518 527 def updateIndexes(self):
519 528
520 529 pass
521 530
522 531 def fillJROHeader(self):
523 532
524 533 #fill radar controller header
525 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
526 txA=self.__txA,
527 txB=0,
528 nWindows=1,
529 nHeights=self.__nSamples,
530 firstHeight=self.__firstHeight,
531 deltaHeight=self.__deltaHeight,
532 codeType=self.__codeType,
533 nCode=self.__nCode, nBaud=self.__nBaud,
534 code = self.__code,
535 fClock=self.__sampleRate)
534
536 535 #fill system header
537 536 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
538 537 nProfiles=self.newProfiles,
539 538 nChannels=len(self.__channelList),
540 539 adcResolution=14,
541 540 pciDioBusWidth=32)
542 541
543 542 self.dataOut.type = "Voltage"
544 543 self.dataOut.data = None
545 544 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
546 545 # self.dataOut.nChannels = 0
547 546
548 547 # self.dataOut.nHeights = 0
549 548
550 549 self.dataOut.nProfiles = self.newProfiles*self.nblocks
551 550 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
552 551 ranges = numpy.reshape(self.rangeFromFile[()],(-1))
553 552 self.dataOut.heightList = ranges/1000.0 #km
554 553 self.dataOut.channelList = self.__channelList
554
555 555 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
556 556
557 557 # self.dataOut.channelIndexList = None
558 558
559 559
560 560 self.dataOut.azimuthList = numpy.array(self.azimuthList)
561 561 self.dataOut.elevationList = numpy.array(self.elevationList)
562 562 self.dataOut.codeList = numpy.array(self.beamCode)
563
564
565
566
563 567 #print(self.dataOut.elevationList)
564 568 self.dataOut.flagNoData = True
565 569
566 570 #Set to TRUE if the data is discontinuous
567 571 self.dataOut.flagDiscontinuousBlock = False
568 572
569 573 self.dataOut.utctime = None
570 574
571 575 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
572 576 if self.timezone == 'lt':
573 577 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
574 578 else:
575 579 self.dataOut.timeZone = 0 #by default time is UTC
576 580
577 581 self.dataOut.dstFlag = 0
578 582 self.dataOut.errorCount = 0
579 583 self.dataOut.nCohInt = 1
580 584 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
581 585 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
582 586 self.dataOut.flagShiftFFT = False
583 587 self.dataOut.ippSeconds = self.ippSeconds
584 self.dataOut.radar_ipp = self.ippSeconds
585 self.dataOut.pulseLength_TxA = self.__txA/0.15
586 self.dataOut.deltaHeight = self.__deltaHeight
587 #Time interval between profiles
588 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
588 self.dataOut.ipp = self.__ippKm
589
589 590
590 591 self.dataOut.frequency = self.__frequency
591 592 self.dataOut.realtime = self.online
593
594 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
595 txA=self.__txAKm,
596 txB=0,
597 nWindows=1,
598 nHeights=self.__nSamples,
599 firstHeight=self.__firstHeight,
600 codeType=self.__codeType,
601 nCode=self.__nCode, nBaud=self.__nBaud,
602 code = self.__code,
603 nOsamp=self.nOsamp,
604 frequency = self.__frequency,
605 sampleRate= self.__sampleRate,
606 fClock=self.__sampleRate)
607
608
609 self.dataOut.radarControllerHeaderObj.heightList = ranges/1000.0 #km
610 self.dataOut.radarControllerHeaderObj.heightResolution = self.__deltaHeight
611 self.dataOut.radarControllerHeaderObj.rangeIpp = self.__ippKm #km
612 self.dataOut.radarControllerHeaderObj.rangeTxA = self.__txA*1e6*.15 #km
613 self.dataOut.radarControllerHeaderObj.nChannels = self.nchannels
614 self.dataOut.radarControllerHeaderObj.channelList = self.__channelList
615 self.dataOut.radarControllerHeaderObj.azimuthList = self.azimuthList
616 self.dataOut.radarControllerHeaderObj.elevationList = self.elevationList
617 self.dataOut.radarControllerHeaderObj.dtype = "Voltage"
618 self.dataOut.ippSeconds = self.ippSeconds
592 619 pass
593 620
594 621 def readNextFile(self,online=False):
595 622
596 623 if not(online):
597 624 newFile = self.__setNextFileOffline()
598 625 else:
599 626 newFile = self.__setNextFileOnline()
600 627
601 628 if not(newFile):
602 629 self.dataOut.error = True
603 630 return 0
604 631
605 632 if not self.readAMISRHeader(self.amisrFilePointer):
606 633 self.dataOut.error = True
607 634 return 0
608 635
609 636 self.createBuffers()
610 637 self.fillJROHeader()
611 638
612 639 #self.__firstFile = False
613 640
614 641
615 642
616 643 self.dataset,self.timeset = self.readData()
617 644
618 645 if self.endDate!=None:
619 646 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
620 647 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
621 648 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
622 649 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
623 650 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
624 651 if self.timezone == 'lt':
625 652 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
626 653 if (startDateTime_File>endDateTime_Reader):
627 654 return 0
628 655
629 656 self.jrodataset = self.reshapeData()
630 657 #----self.updateIndexes()
631 658 self.profileIndex = 0
632 659
633 660 return 1
634 661
635 662
636 663 def __hasNotDataInBuffer(self):
637 664 if self.profileIndex >= (self.newProfiles*self.nblocks):
638 665 return 1
639 666 return 0
640 667
641 668
642 669 def getData(self):
643 670
644 671 if self.flagNoMoreFiles:
645 672 self.dataOut.flagNoData = True
646 673 return 0
647 674
648 675 if self.profileIndex >= (self.newProfiles*self.nblocks): #
649 676 #if self.__hasNotDataInBuffer():
650 677 if not (self.readNextFile(self.online)):
651 678 return 0
652 679
653 680
654 681 if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
655 682 self.dataOut.flagNoData = True
656 683 return 0
657 684
658 685 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
659 686
660 687 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
661 688
662 689 #print("R_t",self.timeset)
663 690
664 691 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
665 692 #verificar basic header de jro data y ver si es compatible con este valor
666 693 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
667 694 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
668 695 indexblock = self.profileIndex/self.newProfiles
669 696 #print (indexblock, indexprof)
670 697 diffUTC = 0
671 698 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
672 699
673 700 #print("utc :",indexblock," __ ",t_comp)
674 701 #print(numpy.shape(self.timeset))
675 702 self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
676 703 #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
677 704
678 705 self.dataOut.profileIndex = self.profileIndex
679 706 #print("N profile:",self.profileIndex,self.newProfiles,self.nblocks,self.dataOut.utctime)
680 707 self.dataOut.flagNoData = False
681 708 # if indexprof == 0:
682 709 # print("kamisr: ",self.dataOut.utctime)
683 710
684 711 self.profileIndex += 1
685 712
686 713 return self.dataOut.data #retorno necesario??
687 714
688 715
689 716 def run(self, **kwargs):
690 717 '''
691 718 This method will be called many times so here you should put all your code
692 719 '''
693 720 #print("running kamisr")
694 721 if not self.isConfig:
695 722 self.setup(**kwargs)
696 723 self.isConfig = True
697 724
698 725 self.getData()
Show file before | Show file after | Hide comments
@@ -1,687 +1,809
1 1 import os
2 2 import time
3 3 import datetime
4 4
5 5 import numpy
6 6 import h5py
7 7
8 8 import schainpy.admin
9 9 from schainpy.model.data.jrodata import *
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 11 from schainpy.model.io.jroIO_base import *
12 12 from schainpy.utils import log
13 13
14 14
15 15 class HDFReader(Reader, ProcessingUnit):
16 16 """Processing unit to read HDF5 format files
17 17
18 18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 20 attributes.
21 21 It is possible to read any HDF5 file by given the structure in the `description`
22 22 parameter, also you can add extra values to metadata with the parameter `extras`.
23 23
24 24 Parameters:
25 25 -----------
26 26 path : str
27 27 Path where files are located.
28 28 startDate : date
29 29 Start date of the files
30 30 endDate : list
31 31 End date of the files
32 32 startTime : time
33 33 Start time of the files
34 34 endTime : time
35 35 End time of the files
36 36 description : dict, optional
37 37 Dictionary with the description of the HDF5 file
38 38 extras : dict, optional
39 39 Dictionary with extra metadata to be be added to `dataOut`
40 40
41 41 Examples
42 42 --------
43 43
44 44 desc = {
45 45 'Data': {
46 46 'data_output': ['u', 'v', 'w'],
47 47 'utctime': 'timestamps',
48 48 } ,
49 49 'Metadata': {
50 50 'heightList': 'heights'
51 51 }
52 52 }
53 53
54 54 desc = {
55 55 'Data': {
56 56 'data_output': 'winds',
57 57 'utctime': 'timestamps'
58 58 },
59 59 'Metadata': {
60 60 'heightList': 'heights'
61 61 }
62 62 }
63 63
64 64 extras = {
65 65 'timeZone': 300
66 66 }
67 67
68 68 reader = project.addReadUnit(
69 69 name='HDFReader',
70 70 path='/path/to/files',
71 71 startDate='2019/01/01',
72 72 endDate='2019/01/31',
73 73 startTime='00:00:00',
74 74 endTime='23:59:59',
75 75 # description=json.dumps(desc),
76 76 # extras=json.dumps(extras),
77 77 )
78 78
79 79 """
80 80
81 81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82 82
83 83 def __init__(self):
84 84 ProcessingUnit.__init__(self)
85 85
86 86 self.ext = ".hdf5"
87 87 self.optchar = "D"
88 88 self.meta = {}
89 89 self.data = {}
90 90 self.open_file = h5py.File
91 91 self.open_mode = 'r'
92 92 self.description = {}
93 93 self.extras = {}
94 94 self.filefmt = "*%Y%j***"
95 95 self.folderfmt = "*%Y%j"
96 96 self.utcoffset = 0
97
97 self.flagUpdateDataOut = False
98 98 self.dataOut = Parameters()
99 99 self.dataOut.error=False ## NOTE: Importante definir esto antes inicio
100 100 self.dataOut.flagNoData = True
101 101
102 102 def setup(self, **kwargs):
103 103
104 104 self.set_kwargs(**kwargs)
105 105 if not self.ext.startswith('.'):
106 106 self.ext = '.{}'.format(self.ext)
107 107
108 108 if self.online:
109 109 log.log("Searching files in online mode...", self.name)
110 110
111 111 for nTries in range(self.nTries):
112 112 fullpath = self.searchFilesOnLine(self.path, self.startDate,
113 113 self.endDate, self.expLabel, self.ext, self.walk,
114 114 self.filefmt, self.folderfmt)
115 115 pathname, filename = os.path.split(fullpath)
116 116
117 117 try:
118 118 fullpath = next(fullpath)
119 119
120 120 except:
121 121 fullpath = None
122 122
123 123 if fullpath:
124 124 break
125 125
126 126 log.warning(
127 127 'Waiting {} sec for a valid file in {}: try {} ...'.format(
128 128 self.delay, self.path, nTries + 1),
129 129 self.name)
130 130 time.sleep(self.delay)
131 131
132 132 if not(fullpath):
133 133 raise schainpy.admin.SchainError(
134 134 'There isn\'t any valid file in {}'.format(self.path))
135 135
136 136 pathname, filename = os.path.split(fullpath)
137 137 self.year = int(filename[1:5])
138 138 self.doy = int(filename[5:8])
139 139 self.set = int(filename[8:11]) - 1
140 140 else:
141 141 log.log("Searching files in {}".format(self.path), self.name)
142 142 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
143 143 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
144 144
145 145 self.setNextFile()
146 146
147 147
148 148
149 149
150 # def readFirstHeader(self):
151 # '''Read metadata and data'''
152 #
153 # self.__readMetadata2()
154 # self.__readData()
155 # self.__setBlockList()
156 #
157 # for attr in self.meta:
158 # setattr(self.dataOut, attr, self.meta[attr])
159 # self.blockIndex = 0
160 #
161 # return
162
150 163 def readFirstHeader(self):
151 164 '''Read metadata and data'''
152 165
153 self.__readMetadata()
166 self.__readMetadata2()
154 167 self.__readData()
155 168 self.__setBlockList()
156
157 169 for attr in self.meta:
158 setattr(self.dataOut, attr, self.meta[attr])
170 if "processingHeaderObj" in attr:
171 self.flagUpdateDataOut=True
172 at = attr.split('.')
173 if len(at) > 1:
174 setattr(eval("self.dataOut."+at[0]),at[1], self.meta[attr])
175 else:
176 setattr(self.dataOut, attr, self.meta[attr])
159 177 self.blockIndex = 0
160 178
179 if self.flagUpdateDataOut:
180 self.updateDataOut()
181
161 182 return
162 183
184 def updateDataOut(self):
185 self.dataOut.azimuthList = self.dataOut.processingHeaderObj.azimuthList
186 self.dataOut.elevationList = self.dataOut.processingHeaderObj.elevationList
187 self.dataOut.heightList = self.dataOut.processingHeaderObj.heightList
188 self.dataOut.ippSeconds = self.dataOut.processingHeaderObj.ipp
189 self.dataOut.elevationList = self.dataOut.processingHeaderObj.elevationList
190 self.dataOut.channelList = self.dataOut.processingHeaderObj.channelList
191 self.dataOut.nCohInt = self.dataOut.processingHeaderObj.nCohInt
192 self.dataOut.nFFTPoints = self.dataOut.processingHeaderObj.nFFTPoints
193 self.flagUpdateDataOut = False
194 self.dataOut.frequency = self.dataOut.radarControllerHeaderObj.frequency
195 #self.dataOut.heightList = self.dataOut.processingHeaderObj.heightList
196
197
198 return
199
200
163 201 def __setBlockList(self):
164 202 '''
165 203 Selects the data within the times defined
166 204
167 205 self.fp
168 206 self.startTime
169 207 self.endTime
170 208 self.blockList
171 209 self.blocksPerFile
172 210
173 211 '''
174 212
175 213 startTime = self.startTime
176 214 endTime = self.endTime
177 215 thisUtcTime = self.data['utctime'] + self.utcoffset
178 216 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
179 217 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
180 218 self.startFileDatetime = thisDatetime
181 219 thisDate = thisDatetime.date()
182 220 thisTime = thisDatetime.time()
183 221
184 222 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
185 223 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
186 224
187 225 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
188 226
189 227 self.blockList = ind
190 228 self.blocksPerFile = len(ind)
191 229 self.blocksPerFile = len(thisUtcTime)
192 230 return
193 231
194 232 def __readMetadata(self):
195 233 '''
196 234 Reads Metadata
197 235 '''
198 236
199 237 meta = {}
200 238
201 239 if self.description:
202 240 for key, value in self.description['Metadata'].items():
203 241 meta[key] = self.fp[value][()]
204 242 else:
205 243 grp = self.fp['Metadata']
206 244 for name in grp:
207 245 meta[name] = grp[name][()]
208 246
209 247 if self.extras:
210 248 for key, value in self.extras.items():
211 249 meta[key] = value
212 250 self.meta = meta
213 251
214 252 return
215 253
216 254
255 def __readMetadata2(self):
256 '''
257 Reads Metadata
258 '''
259
260 meta = {}
261
262 if self.description:
263 for key, value in self.description['Metadata'].items():
264 meta[key] = self.fp[value][()]
265 else:
266 grp = self.fp['Metadata']
267 for item in grp.values():
268 name = item.name
269 if isinstance(item, h5py.Dataset):
270 name = name.split("/")[-1]
271 meta[name] = item[()]
272 else:
273 grp2 = self.fp[name]
274 Obj = name.split("/")[-1]
275
276 for item2 in grp2.values():
277 name2 = Obj+"."+item2.name.split("/")[-1]
278 meta[name2] = item2[()]
279
280 if self.extras:
281 for key, value in self.extras.items():
282 meta[key] = value
283 self.meta = meta
284 #print(meta)
285 return
286
217 287
218 288 def checkForRealPath(self, nextFile, nextDay):
219 289
220 290 # print("check FRP")
221 291 # dt = self.startFileDatetime + datetime.timedelta(1)
222 292 # filename = '{}.{}{}'.format(self.path, dt.strftime('%Y%m%d'), self.ext)
223 293 # fullfilename = os.path.join(self.path, filename)
224 294 # print("check Path ",fullfilename,filename)
225 295 # if os.path.exists(fullfilename):
226 296 # return fullfilename, filename
227 297 # return None, filename
228 298 return None,None
229 299
230 300 def __readData(self):
231 301
232 302 data = {}
233 303
234 304 if self.description:
235 305 for key, value in self.description['Data'].items():
236 306 if isinstance(value, str):
237 307 if isinstance(self.fp[value], h5py.Dataset):
238 308 data[key] = self.fp[value][()]
239 309 elif isinstance(self.fp[value], h5py.Group):
240 310 array = []
241 311 for ch in self.fp[value]:
242 312 array.append(self.fp[value][ch][()])
243 313 data[key] = numpy.array(array)
244 314 elif isinstance(value, list):
245 315 array = []
246 316 for ch in value:
247 317 array.append(self.fp[ch][()])
248 318 data[key] = numpy.array(array)
249 319 else:
250 320 grp = self.fp['Data']
251 321 for name in grp:
252 322 if isinstance(grp[name], h5py.Dataset):
253 323 array = grp[name][()]
254 324 elif isinstance(grp[name], h5py.Group):
255 325 array = []
256 326 for ch in grp[name]:
257 327 array.append(grp[name][ch][()])
258 328 array = numpy.array(array)
259 329 else:
260 330 log.warning('Unknown type: {}'.format(name))
261 331
262 332 if name in self.description:
263 333 key = self.description[name]
264 334 else:
265 335 key = name
266 336 data[key] = array
267 337
268 338 self.data = data
269 339 return
270 340
271 341 def getData(self):
272 342 if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime):
273 343 self.dataOut.flagNoData = True
274 344 self.blockIndex = self.blocksPerFile
275 345 self.dataOut.error = True # TERMINA EL PROGRAMA
276 346 return
277 347 for attr in self.data:
278 348
279 349 if self.data[attr].ndim == 1:
280 350 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
281 351 else:
282 352 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
283 353
284 354
285 355 self.blockIndex += 1
286 356
287 357 if self.blockIndex == 1:
288 358 log.log("Block No. {}/{} -> {}".format(
289 359 self.blockIndex,
290 360 self.blocksPerFile,
291 361 self.dataOut.datatime.ctime()), self.name)
292 362 else:
293 363 log.log("Block No. {}/{} ".format(
294 364 self.blockIndex,
295 365 self.blocksPerFile),self.name)
296 366
297 367 if self.blockIndex == self.blocksPerFile:
298 368 self.setNextFile()
299 369
300 370 self.dataOut.flagNoData = False
301 371
302 372
303 373 def run(self, **kwargs):
304 374
305 375 if not(self.isConfig):
306 376 self.setup(**kwargs)
307 377 self.isConfig = True
308 378
309 379 self.getData()
310 380
311 381 @MPDecorator
312 382 class HDFWriter(Operation):
313 383 """Operation to write HDF5 files.
314 384
315 385 The HDF5 file contains by default two groups Data and Metadata where
316 386 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
317 387 parameters, data attributes are normaly time dependent where the metadata
318 388 are not.
319 389 It is possible to customize the structure of the HDF5 file with the
320 390 optional description parameter see the examples.
321 391
322 392 Parameters:
323 393 -----------
324 394 path : str
325 395 Path where files will be saved.
326 396 blocksPerFile : int
327 397 Number of blocks per file
328 398 metadataList : list
329 399 List of the dataOut attributes that will be saved as metadata
330 400 dataList : int
331 401 List of the dataOut attributes that will be saved as data
332 402 setType : bool
333 403 If True the name of the files corresponds to the timestamp of the data
334 404 description : dict, optional
335 405 Dictionary with the desired description of the HDF5 file
336 406
337 407 Examples
338 408 --------
339 409
340 410 desc = {
341 411 'data_output': {'winds': ['z', 'w', 'v']},
342 412 'utctime': 'timestamps',
343 413 'heightList': 'heights'
344 414 }
345 415 desc = {
346 416 'data_output': ['z', 'w', 'v'],
347 417 'utctime': 'timestamps',
348 418 'heightList': 'heights'
349 419 }
350 420 desc = {
351 421 'Data': {
352 422 'data_output': 'winds',
353 423 'utctime': 'timestamps'
354 424 },
355 425 'Metadata': {
356 426 'heightList': 'heights'
357 427 }
358 428 }
359 429
360 430 writer = proc_unit.addOperation(name='HDFWriter')
361 431 writer.addParameter(name='path', value='/path/to/file')
362 432 writer.addParameter(name='blocksPerFile', value='32')
363 433 writer.addParameter(name='metadataList', value='heightList,timeZone')
364 434 writer.addParameter(name='dataList',value='data_output,utctime')
365 435 # writer.addParameter(name='description',value=json.dumps(desc))
366 436
367 437 """
368 438
369 439 ext = ".hdf5"
370 440 optchar = "D"
371 441 filename = None
372 442 path = None
373 443 setFile = None
374 444 fp = None
375 445 firsttime = True
376 446 #Configurations
377 447 blocksPerFile = None
378 448 blockIndex = None
379 449 dataOut = None #eval ??????
380 450 #Data Arrays
381 451 dataList = None
382 452 metadataList = None
383 453 currentDay = None
384 454 lastTime = None
385 455 timeZone = "ut"
386 456 hourLimit = 3
387 457 breakDays = True
388 458
389 459 def __init__(self):
390 460
391 461 Operation.__init__(self)
392 462
393 463
394 464 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None,
395 465 description={},timeZone = "ut",hourLimit = 3, breakDays=True):
396 466 self.path = path
397 467 self.blocksPerFile = blocksPerFile
398 468 self.metadataList = metadataList
399 469 self.dataList = [s.strip() for s in dataList]
400 470 self.setType = setType
401 471 self.description = description
402 472 self.timeZone = timeZone
403 473 self.hourLimit = hourLimit
404 474 self.breakDays = breakDays
405 475
406 476 if self.metadataList is None:
407 477 self.metadataList = self.dataOut.metadata_list
408 478
479 self.metadataList = list(set(self.metadataList))
480
409 481 tableList = []
410 482 dsList = []
411 483
412 484 for i in range(len(self.dataList)):
413 485 dsDict = {}
414 486 if hasattr(self.dataOut, self.dataList[i]):
415 487 dataAux = getattr(self.dataOut, self.dataList[i])
416 488 dsDict['variable'] = self.dataList[i]
417 489 else:
418 490 log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]),self.name)
419 491 continue
420 492
421 493 if dataAux is None:
422 494 continue
423 495 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
424 496 dsDict['nDim'] = 0
425 497 else:
426 498 dsDict['nDim'] = len(dataAux.shape)
427 499 dsDict['shape'] = dataAux.shape
428 500 dsDict['dsNumber'] = dataAux.shape[0]
429 501 dsDict['dtype'] = dataAux.dtype
430 502
431 503 dsList.append(dsDict)
432 504
433 505 self.blockIndex = 0
434 506 self.dsList = dsList
435 507 self.currentDay = self.dataOut.datatime.date()
436 508
437 509
438 510 def timeFlag(self):
439 511 currentTime = self.dataOut.utctime
440 512 timeTuple = None
441 513 if self.timeZone == "lt":
442 514 timeTuple = time.localtime(currentTime)
443 515 else :
444 516 timeTuple = time.gmtime(currentTime)
445 517
446 518 dataDay = timeTuple.tm_yday
447 519
448 520 if self.lastTime is None:
449 521 self.lastTime = currentTime
450 522 self.currentDay = dataDay
451 523 return False
452 524
453 525 timeDiff = currentTime - self.lastTime
454 526
455 527 #Si el dia es diferente o si la diferencia entre un dato y otro supera self.hourLimit
456 528 if (dataDay != self.currentDay) and self.breakDays:
457 529 self.currentDay = dataDay
458 530 return True
459 531 elif timeDiff > self.hourLimit*60*60:
460 532 self.lastTime = currentTime
461 533 return True
462 534 else:
463 535 self.lastTime = currentTime
464 536 return False
465 537
466 538 def run(self, dataOut,**kwargs):
467 539
468 540 self.dataOut = dataOut
469 541 #print(self.dataOut.radarControllerHeaderObj.toString())
470 542 if not(self.isConfig):
471 543 self.setup(**kwargs)
472 544
473 545 self.isConfig = True
474 546 self.setNextFile()
475 547
476 548 self.putData()
477 549
478 550 #return self.dataOut
479 551
480 552 def setNextFile(self):
481 553
482 554 ext = self.ext
483 555 path = self.path
484 556 setFile = self.setFile
485 557 timeTuple = None
486 558 if self.timeZone == "lt":
487 559 timeTuple = time.localtime(self.dataOut.utctime)
488 560 elif self.timeZone == "ut":
489 561 timeTuple = time.gmtime(self.dataOut.utctime)
490 562 #print("path: ",timeTuple)
491 563 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
492 564 fullpath = os.path.join(path, subfolder)
493 565
494 566 if os.path.exists(fullpath):
495 567 filesList = os.listdir(fullpath)
496 568 filesList = [k for k in filesList if k.startswith(self.optchar)]
497 569 if len( filesList ) > 0:
498 570 filesList = sorted(filesList, key=str.lower)
499 571 filen = filesList[-1]
500 572 # el filename debera tener el siguiente formato
501 573 # 0 1234 567 89A BCDE (hex)
502 574 # x YYYY DDD SSS .ext
503 575 if isNumber(filen[8:11]):
504 576 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
505 577 else:
506 578 setFile = -1
507 579 else:
508 580 setFile = -1 #inicializo mi contador de seteo
509 581 else:
510 582 os.makedirs(fullpath)
511 583 setFile = -1 #inicializo mi contador de seteo
512 584
513 585 if self.setType is None:
514 586 setFile += 1
515 587 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
516 588 timeTuple.tm_year,
517 589 timeTuple.tm_yday,
518 590 setFile,
519 591 ext )
520 592 else:
521 593 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
522 594 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
523 595 timeTuple.tm_year,
524 596 timeTuple.tm_yday,
525 597 setFile,
526 598 ext )
527 599
528 600 self.filename = os.path.join( path, subfolder, file )
529 601
530 602
531 603
532 604 def getLabel(self, name, x=None):
533 605
534 606 if x is None:
535 607 if 'Data' in self.description:
536 608 data = self.description['Data']
537 609 if 'Metadata' in self.description:
538 610 data.update(self.description['Metadata'])
539 611 else:
540 612 data = self.description
541 613 if name in data:
542 614 if isinstance(data[name], str):
543 615 return data[name]
544 616 elif isinstance(data[name], list):
545 617 return None
546 618 elif isinstance(data[name], dict):
547 619 for key, value in data[name].items():
548 620 return key
549 621 return name
550 622 else:
551 623 if 'Metadata' in self.description:
552 624 meta = self.description['Metadata']
553 625 else:
554 626 meta = self.description
555 627 if name in meta:
556 628 if isinstance(meta[name], list):
557 629 return meta[name][x]
558 630 elif isinstance(meta[name], dict):
559 631 for key, value in meta[name].items():
560 632 return value[x]
561 633 if 'cspc' in name:
562 634 return 'pair{:02d}'.format(x)
563 635 else:
564 636 return 'channel{:02d}'.format(x)
565 637
566 638 def writeMetadata(self, fp):
567 639
568 640 if self.description:
569 641 if 'Metadata' in self.description:
570 642 grp = fp.create_group('Metadata')
571 643 else:
572 644 grp = fp
573 645 else:
574 646 grp = fp.create_group('Metadata')
575 647
576 648 for i in range(len(self.metadataList)):
577 649 attribute = self.metadataList[i]
578 650 if not hasattr(self.dataOut,attribute ):
579 651 log.warning('Metadata: `{}` not found'.format(attribute), self.name)
580 652 continue
581 653 value = getattr(self.dataOut, attribute)
582 654 if isinstance(value, bool):
583 655 if value is True:
584 656 value = 1
585 657 else:
586 658 value = 0
587 659 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
588 660 return
589 661
662 def writeMetadata2(self, fp):
663
664 if self.description:
665 if 'Metadata' in self.description:
666 grp = fp.create_group('Metadata')
667 else:
668 grp = fp
669 else:
670 grp = fp.create_group('Metadata')
671
672
673 for i in range(len(self.metadataList)):
674
675 attribute = self.metadataList[i]
676 attr = attribute.split('.')
677 if len(attr) > 1:
678 if not hasattr(eval("self.dataOut."+attr[0]),attr[1]):
679 log.warning('Metadata: {}.{} not found'.format(attr[0],attr[1]), self.name)
680 continue
681 value = getattr(eval("self.dataOut."+attr[0]),attr[1])
682 if isinstance(value, bool):
683 if value is True:
684 value = 1
685 else:
686 value = 0
687 if isinstance(value,type(None)):
688 log.warning("Invalid value detected, {} is None".format(attribute), self.name)
689 value = 0
690 grp2 = None
691 if not 'Metadata/'+attr[0] in fp:
692 grp2 = fp.create_group('Metadata/'+attr[0])
693 else:
694 grp2 = fp['Metadata/'+attr[0]]
695 #print("attribute: ", attribute, value)
696 grp2.create_dataset(attr[1], data=value)
697
698 else:
699 if not hasattr(self.dataOut, attr[0] ):
700 log.warning('Metadata: `{}` not found'.format(attribute), self.name)
701 continue
702 value = getattr(self.dataOut, attr[0])
703 if isinstance(value, bool):
704 if value is True:
705 value = 1
706 else:
707 value = 0
708 grp.create_dataset(self.getLabel(attribute), data=value)
709
710 return
711
590 712 def writeData(self, fp):
591 713
592 714 if self.description:
593 715 if 'Data' in self.description:
594 716 grp = fp.create_group('Data')
595 717 else:
596 718 grp = fp
597 719 else:
598 720 grp = fp.create_group('Data')
599 721
600 722 dtsets = []
601 723 data = []
602 724
603 725 for dsInfo in self.dsList:
604 726 if dsInfo['nDim'] == 0:
605 727 ds = grp.create_dataset(
606 728 self.getLabel(dsInfo['variable']),
607 729 (self.blocksPerFile, ),
608 730 chunks=True,
609 731 dtype=numpy.float64)
610 732 dtsets.append(ds)
611 733 data.append((dsInfo['variable'], -1))
612 734 else:
613 735 label = self.getLabel(dsInfo['variable'])
614 736 if label is not None:
615 737 sgrp = grp.create_group(label)
616 738 else:
617 739 sgrp = grp
618 740 for i in range(dsInfo['dsNumber']):
619 741 ds = sgrp.create_dataset(
620 742 self.getLabel(dsInfo['variable'], i),
621 743 (self.blocksPerFile, ) + dsInfo['shape'][1:],
622 744 chunks=True,
623 745 dtype=dsInfo['dtype'])
624 746 dtsets.append(ds)
625 747 data.append((dsInfo['variable'], i))
626 748 fp.flush()
627 749
628 750 log.log('Creating file: {}'.format(fp.filename), self.name)
629 751
630 752 self.ds = dtsets
631 753 self.data = data
632 754 self.firsttime = True
633 755
634 756 return
635 757
636 758 def putData(self):
637 759
638 760 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
639 761 self.closeFile()
640 762 self.setNextFile()
641 763 self.dataOut.flagNoData = False
642 764 self.blockIndex = 0
643 765 return
644 766
645 767
646 768
647 769 if self.blockIndex == 0:
648 770 #Escribir metadata Aqui???
649 771 #Setting HDF5 File
650 772 self.fp = h5py.File(self.filename, 'w')
651 773 #write metadata
652 self.writeMetadata(self.fp)
774 self.writeMetadata2(self.fp)
653 775 #Write data
654 776 self.writeData(self.fp)
655 777 log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
656 778 elif (self.blockIndex % 10 ==0):
657 779 log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
658 780 else:
659 781
660 782 log.log('Block No. {}/{}'.format(self.blockIndex+1, self.blocksPerFile), self.name)
661 783
662 784 for i, ds in enumerate(self.ds):
663 785 attr, ch = self.data[i]
664 786 if ch == -1:
665 787 ds[self.blockIndex] = getattr(self.dataOut, attr)
666 788 else:
667 789 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
668 790
669 791 self.blockIndex += 1
670 792
671 793 self.fp.flush()
672 794 self.dataOut.flagNoData = True
673 795
674 796
675 797 def closeFile(self):
676 798
677 799 if self.blockIndex != self.blocksPerFile:
678 800 for ds in self.ds:
679 801 ds.resize(self.blockIndex, axis=0)
680 802
681 803 if self.fp:
682 804 self.fp.flush()
683 805 self.fp.close()
684 806
685 807 def close(self):
686 808
687 809 self.closeFile()
Show file before | Show file after | Hide comments
@@ -1,523 +1,661
1 1 """
2 2 Utilities for IO modules
3 3 @modified: Joab Apaza
4 4 @email: roj-op01@igp.gob.pe, joab.apaza32@gmail.com
5 5
6 6 """
7 7 ################################################################################
8 8 ################################################################################
9 9 import os
10 10 from datetime import datetime
11 11 import numpy
12 12
13 13 from schainpy.model.data.jrodata import Parameters
14 14 import itertools
15 15 import numpy
16 16 import h5py
17 17 import re
18 18 import time
19 19 ################################################################################
20 20 ################################################################################
21 21 ################################################################################
22 22 def folder_in_range(folder, start_date, end_date, pattern):
23 23 """
24 24 Check whether folder is bettwen start_date and end_date
25 25
26 26 Args:
27 27 folder (str): Folder to check
28 28 start_date (date): Initial date
29 29 end_date (date): Final date
30 30 pattern (str): Datetime format of the folder
31 31 Returns:
32 32 bool: True for success, False otherwise
33 33 """
34 34 try:
35 35 dt = datetime.strptime(folder, pattern)
36 36 except:
37 37 raise ValueError('Folder {} does not match {} format'.format(folder, pattern))
38 38 return start_date <= dt.date() <= end_date
39 39
40 40 ################################################################################
41 41 ################################################################################
42 42 ################################################################################
43 43 def getHei_index( minHei, maxHei, heightList):
44 44 if (minHei < heightList[0]):
45 45 minHei = heightList[0]
46 46
47 47 if (maxHei > heightList[-1]):
48 48 maxHei = heightList[-1]
49 49
50 50 minIndex = 0
51 51 maxIndex = 0
52 52 heights = numpy.asarray(heightList)
53 53
54 54 inda = numpy.where(heights >= minHei)
55 55 indb = numpy.where(heights <= maxHei)
56 56
57 57 try:
58 58 minIndex = inda[0][0]
59 59 except:
60 60 minIndex = 0
61 61
62 62 try:
63 63 maxIndex = indb[0][-1]
64 64 except:
65 65 maxIndex = len(heightList)
66 66 return minIndex,maxIndex
67 67
68 68 ################################################################################
69 69 ################################################################################
70 70 ################################################################################
71 71 class MergeH5(object):
72 72 """Processing unit to read HDF5 format files
73 73
74 74 This unit reads HDF5 files created with `HDFWriter` operation when channels area
75 75 processed by separated. Then merge all channels in a single files.
76 76
77 77 "example"
78 78 nChannels = 4
79 79 pathOut = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/merged"
80 80 p0 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch0"
81 81 p1 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch1"
82 82 p2 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch2"
83 83 p3 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch3"
84 84 list = ['data_spc','data_cspc','nIncohInt','utctime']
85 85 merger = MergeH5(nChannels,pathOut,list, p0, p1,p2,p3)
86 86 merger.run()
87 87
88 88 """
89 89
90 90 # #__attrs__ = ['paths', 'nChannels']
91 91 isConfig = False
92 92 inPaths = None
93 93 nChannels = None
94 94 ch_dataIn = []
95 95
96 96 channelList = []
97 97
98 98 def __init__(self,nChannels, pOut, dataList, *args):
99 99
100 100 self.inPaths = [p for p in args]
101 101 #print(self.inPaths)
102 102 if len(self.inPaths) != nChannels:
103 103 print("ERROR, number of channels different from iput paths {} != {}".format(nChannels, len(args)))
104 104 return
105 105
106 106 self.pathOut = pOut
107 107 self.dataList = dataList
108 108 self.nChannels = len(self.inPaths)
109 109 self.ch_dataIn = [Parameters() for p in args]
110 110 self.dataOut = Parameters()
111 111 self.channelList = [n for n in range(nChannels)]
112 112 self.blocksPerFile = None
113 113 self.date = None
114 114 self.ext = ".hdf5$"
115 115 self.dataList = dataList
116 116 self.optchar = "D"
117 117 self.meta = {}
118 118 self.data = {}
119 119 self.open_file = h5py.File
120 120 self.open_mode = 'r'
121 121 self.description = {}
122 122 self.extras = {}
123 123 self.filefmt = "*%Y%j***"
124 124 self.folderfmt = "*%Y%j"
125 125
126 self.flag_spc = False
127 self.flag_pow = False
128 self.flag_snr = False
129 self.flag_nIcoh = False
130 self.flagProcessingHeader = False
131 self.flagControllerHeader = False
126 132
127 133 def setup(self):
128 134
129 135
130 136 # if not self.ext.startswith('.'):
131 137 # self.ext = '.{}'.format(self.ext)
132 138
133 139 self.filenameList = self.searchFiles(self.inPaths, None)
134 140 self.nfiles = len(self.filenameList[0])
135 141
136 142
137 143
138 144 def searchFiles(self, paths, date, walk=True):
139 145 # self.paths = path
140 146 #self.date = startDate
141 147 #self.walk = walk
142 148 filenameList = [[] for n in range(self.nChannels)]
143 149 ch = 0
144 150 for path in paths:
145 151 if os.path.exists(path):
146 152 print("Searching files in {}".format(path))
147 153 filenameList[ch] = self.getH5files(path, walk)
148 154 print("Found: ")
149 155 for f in filenameList[ch]:
150 156 print(f)
151 157 else:
152 158 self.status = 0
153 159 print('Path:%s does not exists'%path)
154 160 return 0
155 161 ch+=1
156 162 return filenameList
157 163
158 164 def getH5files(self, path, walk):
159 165
160 166 dirnameList = []
161 167 pat = '(\d)+.'+self.ext
162 168 if walk:
163 169 for root, dirs, files in os.walk(path):
164 170 for dir in dirs:
165 171 #print(os.path.join(root,dir))
166 172 files = [re.search(pat,x) for x in os.listdir(os.path.join(root,dir))]
167 173 #print(files)
168 174 files = [x for x in files if x!=None]
169 175 files = [x.string for x in files]
170 176 files = [os.path.join(root,dir,x) for x in files if x!=None]
171 177 files.sort()
172 178
173 179 dirnameList += files
174 180 return dirnameList
175 181 else:
176 182
177 183 dirnameList = [re.search(pat,x) for x in os.listdir(path)]
178 184 dirnameList = [x for x in dirnameList if x!=None]
179 185 dirnameList = [x.string for x in dirnameList]
180 186 dirnameList = [x for x in dirnameList if x!=None]
181 187 dirnameList.sort()
182 188
183 189 return dirnameList
184 190
185 191
186 192 def readFile(self,fp,ch):
187 193
188 194 '''Read metadata and data'''
189 195
190 self.readMetadata(fp)
196 self.readMetadata(fp,ch)
197 #print(self.metadataList)
191 198 data = self.readData(fp)
192 199
193 200
194 201 for attr in self.meta:
195 setattr(self.ch_dataIn[ch], attr, self.meta[attr])
202
203 if "processingHeaderObj" in attr:
204 self.flagProcessingHeader=True
205
206 if "radarControllerHeaderObj" in attr:
207 self.flagControllerHeader=True
208
209 at = attr.split('.')
210 #print("AT ", at)
211 if len(at) > 1:
212 setattr(eval("self.ch_dataIn[ch]."+at[0]),at[1], self.meta[attr])
213 else:
214 setattr(self.ch_dataIn[ch], attr, self.meta[attr])
196 215
197 216 self.fill_dataIn(data, self.ch_dataIn[ch])
198 217
199 218
200 219 return
201 220
202 221
203 def readMetadata(self, fp):
222 def readMetadata(self, fp, ch):
204 223 '''
205 224 Reads Metadata
206 225 '''
207 226 meta = {}
208 227 self.metadataList = []
209 228 grp = fp['Metadata']
210 for name in grp:
211 meta[name] = grp[name][()]
212 self.metadataList.append(name)
229 for item in grp.values():
230 name = item.name
231
232 if isinstance(item, h5py.Dataset):
233 name = name.split("/")[-1]
234 if 'List' in name:
235 meta[name] = item[()].tolist()
236 else:
237 meta[name] = item[()]
238 self.metadataList.append(name)
239 else:
240 grp2 = fp[name]
241 Obj = name.split("/")[-1]
242 #print(Obj)
243 for item2 in grp2.values():
244 name2 = Obj+"."+item2.name.split("/")[-1]
245 if 'List' in name2:
246 meta[name2] = item2[()].tolist()
247 else:
248 meta[name2] = item2[()]
249 self.metadataList.append(name2)
250
251
213 252
214 for k in meta:
215 if ('List' in k):
216 meta[k] = meta[k].tolist()
217 253 if not self.meta:
218 self.meta = meta
219 self.meta["channelList"] =[n for n in range(self.nChannels)]
254 self.meta = meta.copy()
255 for key in list(self.meta.keys()):
256 if "channelList" in key:
257 self.meta["channelList"] =[n for n in range(self.nChannels)]
258 if "processingHeaderObj" in key:
259 self.meta["processingHeaderObj.channelList"] =[n for n in range(self.nChannels)]
260 if "radarControllerHeaderObj" in key:
261 self.meta["radarControllerHeaderObj.channelList"] =[n for n in range(self.nChannels)]
220 262 return 1
263
221 264 else:
222 if len(self.meta) == len(meta):
223 for k in meta:
224 if 'List' in k and 'channel' not in k and "height" not in k:
225 self.meta[k] += meta[k]
226 265
227 return 1
228 else:
229 return 0
266 for k in list(self.meta.keys()):
267 if 'List' in k and 'channel' not in k and "height" not in k and "radarControllerHeaderObj" not in k:
268 self.meta[k] += meta[k]
269
270 #print("Metadata: ",self.meta)
271 return 1
272
230 273
231 274
232 275
233 276 def fill_dataIn(self,data, dataIn):
234 277
235 278 for attr in data:
236 279 if data[attr].ndim == 1:
237 280 setattr(dataIn, attr, data[attr][:])
238 281 else:
239 282 setattr(dataIn, attr, numpy.squeeze(data[attr][:,:]))
240 print("shape in", dataIn.data_spc.shape, len(dataIn.data_spc))
241 if dataIn.data_spc.ndim > 3:
242 dataIn.data_spc = dataIn.data_spc[0]
243 #print("shape in", dataIn.data_spc.shape)
283 #print("shape in", dataIn.data_spc.shape, len(dataIn.data_spc))
284 if self.flag_spc:
285 if dataIn.data_spc.ndim > 3:
286 dataIn.data_spc = dataIn.data_spc[0]
287 #print("shape in", dataIn.data_spc.shape)
244 288
245 289
246 290
247 291 def getBlocksPerFile(self):
248 292 b = numpy.zeros(self.nChannels)
249 293 for i in range(self.nChannels):
250 b[i] = self.ch_dataIn[i].data_spc.shape[0] #number of blocks
294 if self.flag_spc:
295 b[i] = self.ch_dataIn[i].data_spc.shape[0] #number of blocks
296 elif self.flag_pow:
297 b[i] = self.ch_dataIn[i].data_pow.shape[0] #number of blocks
298 elif self.flag_snr:
299 b[i] = self.ch_dataIn[i].data_snr.shape[0] #number of blocks
251 300
252 301 self.blocksPerFile = int(b.min())
253 302 iresh_ch = numpy.where(b > self.blocksPerFile)[0]
254 303 if len(iresh_ch) > 0:
255 304 for ich in iresh_ch:
256 305 for i in range(len(self.dataList)):
257 306 if hasattr(self.ch_dataIn[ich], self.dataList[i]):
258 307 # print("reshaping ", self.dataList[i])
259 308 # print(getattr(self.ch_dataIn[ich], self.dataList[i]).shape)
260 309 dataAux = getattr(self.ch_dataIn[ich], self.dataList[i])
261 310 setattr(self.ch_dataIn[ich], self.dataList[i], None)
262 311 setattr(self.ch_dataIn[ich], self.dataList[i], dataAux[0:self.blocksPerFile])
263 312 # print(getattr(self.ch_dataIn[ich], self.dataList[i]).shape)
264 313 else:
265 314 return
266 315
267 316
268 317 def getLabel(self, name, x=None):
269 318
270 319 if x is None:
271 320 if 'Data' in self.description:
272 321 data = self.description['Data']
273 322 if 'Metadata' in self.description:
274 323 data.update(self.description['Metadata'])
275 324 else:
276 325 data = self.description
277 326 if name in data:
278 327 if isinstance(data[name], str):
279 328 return data[name]
280 329 elif isinstance(data[name], list):
281 330 return None
282 331 elif isinstance(data[name], dict):
283 332 for key, value in data[name].items():
284 333 return key
285 334 return name
286 335 else:
287 336 if 'Metadata' in self.description:
288 337 meta = self.description['Metadata']
289 338 else:
290 339 meta = self.description
291 340 if name in meta:
292 341 if isinstance(meta[name], list):
293 342 return meta[name][x]
294 343 elif isinstance(meta[name], dict):
295 344 for key, value in meta[name].items():
296 345 return value[x]
297 346
298 347 if 'cspc' in name:
299 348 return 'pair{:02d}'.format(x)
300 349 else:
301 350 return 'channel{:02d}'.format(x)
302 351
303 352 def readData(self, fp):
304 353 #print("read fp: ", fp)
305 354 data = {}
306 355
307 356 grp = fp['Data']
308 357
309 358 for name in grp:
359 if "spc" in name:
360 self.flag_spc = True
361 if "pow" in name:
362 self.flag_pow = True
363 if "snr" in name:
364 self.flag_snr = True
310 365 if isinstance(grp[name], h5py.Dataset):
311 366 array = grp[name][()]
312 367 elif isinstance(grp[name], h5py.Group):
313 368 array = []
314 369 for ch in grp[name]:
315 370 array.append(grp[name][ch][()])
316 371 array = numpy.array(array)
317 372 else:
318 373 print('Unknown type: {}'.format(name))
319 374 data[name] = array
320 375
321 376 return data
322 377
323 378 def getDataOut(self):
324 379
325 380 self.dataOut = self.ch_dataIn[0].copy() #dataIn #blocks, fft, hei for metadata
326 if self.dataOut.data_spc.ndim < 3:
327 return 0
381 if self.flagProcessingHeader:
382 self.dataOut.processingHeaderObj = self.ch_dataIn[0].processingHeaderObj.copy()
383 self.dataOut.heightList = self.dataOut.processingHeaderObj.heightList
384 self.dataOut.ippSeconds = self.dataOut.processingHeaderObj.ipp
385 self.dataOut.channelList = self.dataOut.processingHeaderObj.channelList
386 self.dataOut.nCohInt = self.dataOut.processingHeaderObj.nCohInt
387 self.dataOut.nFFTPoints = self.dataOut.processingHeaderObj.nFFTPoints
388
389 if self.flagControllerHeader:
390 self.dataOut.radarControllerHeaderObj = self.ch_dataIn[0].radarControllerHeaderObj.copy()
391 self.dataOut.frequency = self.dataOut.radarControllerHeaderObj.frequency
392 #--------------------------------------------------------------------
393
394
395 #--------------------------------------------------------------------
396 if self.flag_spc:
397 if self.dataOut.data_spc.ndim < 3:
398 print("shape spc in: ",self.dataOut.data_spc.shape )
399 return 0
400 if self.flag_pow:
401 if self.dataOut.data_pow.ndim < 2:
402 print("shape pow in: ",self.dataOut.data_pow.shape )
403 return 0
404 if self.flag_snr:
405 if self.dataOut.data_snr.ndim < 2:
406 print("shape snr in: ",self.dataOut.data_snr.shape )
407 return 0
408
328 409 self.dataOut.data_spc = None
410 self.dataOut.data_cspc = None
411 self.dataOut.data_pow = None
412 self.dataOut.data_snr = None
329 413 self.dataOut.utctim = None
330 414 self.dataOut.nIncohInt = None
331 #print(self.ch_dataIn[0].data_spc.shape)
332 spc = [data.data_spc for data in self.ch_dataIn]
333 self.dataOut.data_spc = numpy.stack(spc, axis=1) #blocks, ch, fft, hei
415 #--------------------------------------------------------------------
416 if self.flag_spc:
417 spc = [data.data_spc for data in self.ch_dataIn]
418 self.dataOut.data_spc = numpy.stack(spc, axis=1) #blocks, ch, fft, hei
419 #--------------------------------------------------------------------
420 if self.flag_pow:
421 pow = [data.data_pow for data in self.ch_dataIn]
422 self.dataOut.data_pow = numpy.stack(pow, axis=1) #blocks, ch, fft, hei
423 #--------------------------------------------------------------------
424 if self.flag_snr:
425 snr = [data.data_snr for data in self.ch_dataIn]
426 self.dataOut.data_snr = numpy.stack(snr, axis=1) #blocks, ch, fft, hei
427
428 #--------------------------------------------------------------------
334 429 time = [data.utctime for data in self.ch_dataIn]
335 430 time = numpy.asarray(time).mean(axis=0)
336 #time = numpy.reshape(time, (len(time),1))
337 431 time = numpy.squeeze(time)
338 432 self.dataOut.utctime = time
339 ints = [data.nIncohInt for data in self.ch_dataIn]
340 self.dataOut.nIncohInt = numpy.stack(ints, axis=1)
341
342
343 if self.dataOut.nIncohInt.ndim > 3:
344 aux = self.dataOut.nIncohInt
345 self.dataOut.nIncohInt = None
346 self.dataOut.nIncohInt = aux[0]
347
433 #--------------------------------------------------------------------
434 if self.flag_nIcoh:
435 ints = [data.nIncohInt for data in self.ch_dataIn]
436 self.dataOut.nIncohInt = numpy.stack(ints, axis=1)
437
438 if self.dataOut.nIncohInt.ndim > 3:
439 aux = self.dataOut.nIncohInt
440 self.dataOut.nIncohInt = None
441 self.dataOut.nIncohInt = aux[0]
442
443 if self.dataOut.nIncohInt.ndim < 3:
444 nIncohInt = numpy.repeat(self.dataOut.nIncohInt, self.dataOut.nHeights).reshape(self.blocksPerFile,self.nChannels, self.dataOut.nHeights)
445 #nIncohInt = numpy.reshape(nIncohInt, (self.blocksPerFile,self.nChannels, self.dataOut.nHeights))
446 self.dataOut.nIncohInt = None
447 self.dataOut.nIncohInt = nIncohInt
448
449 if (self.dataOut.nIncohInt.shape)[0]==self.nChannels: ## ch,blocks, hei
450 self.dataOut.nIncohInt = numpy.swapaxes(self.dataOut.nIncohInt, 0, 1) ## blocks,ch, hei
451 #--------------------------------------------------------------------
452 #print("utcTime: ", time.shape)
453 #print("data_spc ",self.dataOut.data_spc.shape)
454 if "data_cspc" in self.dataList:
455 pairsList = [pair for pair in itertools.combinations(self.channelList, 2)]
456 #print("PairsList: ", pairsList)
457 self.dataOut.pairsList = pairsList
458 cspc = []
348 459
349 if self.dataOut.nIncohInt.ndim < 3:
350 nIncohInt = numpy.repeat(self.dataOut.nIncohInt, self.dataOut.nHeights).reshape(self.blocksPerFile,self.nChannels, self.dataOut.nHeights)
351 #nIncohInt = numpy.reshape(nIncohInt, (self.blocksPerFile,self.nChannels, self.dataOut.nHeights))
352 self.dataOut.nIncohInt = None
353 self.dataOut.nIncohInt = nIncohInt
460 for i, j in pairsList:
461 cspc.append(self.ch_dataIn[i].data_spc*numpy.conjugate(self.ch_dataIn[j].data_spc)) #blocks, fft, hei
354 462
355 if (self.dataOut.nIncohInt.shape)[0]==self.nChannels: ## ch,blocks, hei
356 self.dataOut.nIncohInt = numpy.swapaxes(self.dataOut.nIncohInt, 0, 1) ## blocks,ch, hei
463 cspc = numpy.asarray(cspc) # # pairs, blocks, fft, hei
464 #print("cspc: ",cspc.shape)
465 self.dataOut.data_cspc = numpy.swapaxes(cspc, 0, 1) ## blocks, pairs, fft, hei
466 #print("dataOut.data_cspc: ",self.dataOut.data_cspc.shape)
467 #if "data_pow" in self.dataList:
357 468
358 #print("utcTime: ", time.shape)
359 #print("data_spc ",self.dataOut.data_spc.shape)
360 pairsList = [pair for pair in itertools.combinations(self.channelList, 2)]
361 #print("PairsList: ", pairsList)
362 self.dataOut.pairsList = pairsList
363 cspc = []
364
365 for i, j in pairsList:
366 cspc.append(self.ch_dataIn[i].data_spc*numpy.conjugate(self.ch_dataIn[j].data_spc)) #blocks, fft, hei
367
368 cspc = numpy.asarray(cspc) # # pairs, blocks, fft, hei
369 #print("cspc: ",cspc.shape)
370 self.dataOut.data_cspc = numpy.swapaxes(cspc, 0, 1) ## blocks, pairs, fft, hei
371 #print("dataOut.data_cspc: ",self.dataOut.data_cspc.shape)
372 469 return 1
373 470
471 # def writeMetadata(self, fp):
472 #
473 #
474 # grp = fp.create_group('Metadata')
475 #
476 # for i in range(len(self.metadataList)):
477 # if not hasattr(self.dataOut, self.metadataList[i]):
478 # print('Metadata: `{}` not found'.format(self.metadataList[i]))
479 # continue
480 # value = getattr(self.dataOut, self.metadataList[i])
481 # if isinstance(value, bool):
482 # if value is True:
483 # value = 1
484 # else:
485 # value = 0
486 # grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
487 # return
374 488 def writeMetadata(self, fp):
375 489
376 490
377 491 grp = fp.create_group('Metadata')
378 492
379 493 for i in range(len(self.metadataList)):
380 if not hasattr(self.dataOut, self.metadataList[i]):
381 print('Metadata: `{}` not found'.format(self.metadataList[i]))
382 continue
383 value = getattr(self.dataOut, self.metadataList[i])
384 if isinstance(value, bool):
385 if value is True:
386 value = 1
494 attribute = self.metadataList[i]
495 attr = attribute.split('.')
496 if '' in attr:
497 attr.remove('')
498 #print(attr)
499 if len(attr) > 1:
500 if not hasattr(eval("self.dataOut."+attr[0]),attr[1]):
501 print('Metadata: {}.{} not found'.format(attr[0],attr[1]))
502 continue
503 value = getattr(eval("self.dataOut."+attr[0]),attr[1])
504 if isinstance(value, bool):
505 if value is True:
506 value = 1
507 else:
508 value = 0
509 grp2 = None
510 if not 'Metadata/'+attr[0] in fp:
511 grp2 = fp.create_group('Metadata/'+attr[0])
387 512 else:
388 value = 0
389 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
513 grp2 = fp['Metadata/'+attr[0]]
514
515 grp2.create_dataset(attr[1], data=value)
516
517 else:
518 if not hasattr(self.dataOut, attr[0] ):
519 print('Metadata: `{}` not found'.format(attribute))
520 continue
521 value = getattr(self.dataOut, attr[0])
522 if isinstance(value, bool):
523 if value is True:
524 value = 1
525 else:
526 value = 0
527 grp.create_dataset(self.getLabel(attribute), data=value)
390 528 return
391 529
392 530 def getDsList(self):
393 531
394 532 dsList =[]
395 533 for i in range(len(self.dataList)):
396 534 dsDict = {}
397 535 if hasattr(self.dataOut, self.dataList[i]):
398 536 dataAux = getattr(self.dataOut, self.dataList[i])
399 537 dsDict['variable'] = self.dataList[i]
400 538 else:
401 539 print('Attribute {} not found in dataOut'.format(self.dataList[i]))
402 540 continue
403 541
404 542 if dataAux is None:
405 543 continue
406 544 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
407 545 dsDict['nDim'] = 0
408 546 else:
409 547
410 548 dsDict['nDim'] = len(dataAux.shape) -1
411 549 dsDict['shape'] = dataAux.shape
412 550
413 551 if len(dsDict['shape'])>=2:
414 552 dsDict['dsNumber'] = dataAux.shape[1]
415 553 else:
416 554 dsDict['dsNumber'] = 1
417 555 dsDict['dtype'] = dataAux.dtype
418 556 # if len(dataAux.shape) == 4:
419 557 # dsDict['nDim'] = len(dataAux.shape) -1
420 558 # dsDict['shape'] = dataAux.shape
421 559 # dsDict['dsNumber'] = dataAux.shape[1]
422 560 # dsDict['dtype'] = dataAux.dtype
423 561 # else:
424 562 # dsDict['nDim'] = len(dataAux.shape)
425 563 # dsDict['shape'] = dataAux.shape
426 564 # dsDict['dsNumber'] = dataAux.shape[0]
427 565 # dsDict['dtype'] = dataAux.dtype
428 566
429 567 dsList.append(dsDict)
430 568 #print(dsList)
431 569 self.dsList = dsList
432 570
433 571 def clean_dataIn(self):
434 572 for ch in range(self.nChannels):
435 573 self.ch_dataIn[ch].data_spc = None
436 574 self.ch_dataIn[ch].utctime = None
437 575 self.ch_dataIn[ch].nIncohInt = None
438 576 self.meta ={}
439 577 self.blocksPerFile = None
440 578
441 579 def writeData(self, outFilename):
442 580
443 581 self.getDsList()
444 582
445 583 fp = h5py.File(outFilename, 'w')
446 584 self.writeMetadata(fp)
447 585 grp = fp.create_group('Data')
448 586
449 587 dtsets = []
450 588 data = []
451 589 for dsInfo in self.dsList:
452 590 if dsInfo['nDim'] == 0:
453 591 ds = grp.create_dataset(
454 592 self.getLabel(dsInfo['variable']),(self.blocksPerFile, ),chunks=True,dtype=numpy.float64)
455 593 dtsets.append(ds)
456 594 data.append((dsInfo['variable'], -1))
457 595 else:
458 596 label = self.getLabel(dsInfo['variable'])
459 597 if label is not None:
460 598 sgrp = grp.create_group(label)
461 599 else:
462 600 sgrp = grp
463 601 k = -1*(dsInfo['nDim'] - 1)
464 602 #print(k, dsInfo['shape'], dsInfo['shape'][k:])
465 603 for i in range(dsInfo['dsNumber']):
466 604 ds = sgrp.create_dataset(
467 605 self.getLabel(dsInfo['variable'], i),(self.blocksPerFile, ) + dsInfo['shape'][k:],
468 606 chunks=True,
469 607 dtype=dsInfo['dtype'])
470 608 dtsets.append(ds)
471 609 data.append((dsInfo['variable'], i))
472 610
473 611 #print("\n",dtsets)
474 612
475 613 print('Creating merged file: {}'.format(fp.filename))
476 614
477 615 for i, ds in enumerate(dtsets):
478 616 attr, ch = data[i]
479 617 if ch == -1:
480 618 ds[:] = getattr(self.dataOut, attr)
481 619 else:
482 620 #print(ds, getattr(self.dataOut, attr)[ch].shape)
483 621 aux = getattr(self.dataOut, attr)# block, ch, ...
484 622 aux = numpy.swapaxes(aux,0,1) # ch, blocks, ...
485 623 #print(ds.shape, aux.shape)
486 624 #ds[:] = getattr(self.dataOut, attr)[ch]
487 625 ds[:] = aux[ch]
488 626
489 627 fp.flush()
490 628 fp.close()
491 629 self.clean_dataIn()
492 630 return
493 631
494 632
495 633
496 634 def run(self):
497 635
498 636 if not(self.isConfig):
499 637 self.setup()
500 638 self.isConfig = True
501 639
502 640 for nf in range(self.nfiles):
503 641 name = None
504 642 for ch in range(self.nChannels):
505 643 name = self.filenameList[ch][nf]
506 644 filename = os.path.join(self.inPaths[ch], name)
507 645 fp = h5py.File(filename, 'r')
508 646 #print("Opening file: ",filename)
509 647 self.readFile(fp,ch)
510 648 fp.close()
511 649
512 650 if self.blocksPerFile == None:
513 651 self.getBlocksPerFile()
514 652 print("blocks per file: ", self.blocksPerFile)
515 653
516 654 if not self.getDataOut():
517 655 print("Error getting DataOut invalid number of blocks")
518 656 return
519 657 name = name[-16:]
520 658 #print("Final name out: ", name)
521 659 outFile = os.path.join(self.pathOut, name)
522 660 #print("Outfile: ", outFile)
523 661 self.writeData(outFile)
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,2164 +1,2191
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Spectra processing Unit and operations
6 6
7 7 Here you will find the processing unit `SpectraProc` and several operations
8 8 to work with Spectra data type
9 9 """
10 10
11 11 import time
12 12 import itertools
13 13
14 14 import numpy
15 15 import math
16 16
17 17 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
18 18 from schainpy.model.data.jrodata import Spectra
19 19 from schainpy.model.data.jrodata import hildebrand_sekhon
20 20 from schainpy.model.data import _noise
21 21
22 22 from schainpy.utils import log
23 23 import matplotlib.pyplot as plt
24 24 #from scipy.optimize import curve_fit
25 25 from schainpy.model.io.utilsIO import getHei_index
26 26
27 27 class SpectraProc(ProcessingUnit):
28 28
29 29 def __init__(self):
30 30
31 31 ProcessingUnit.__init__(self)
32 32
33 33 self.buffer = None
34 34 self.firstdatatime = None
35 35 self.profIndex = 0
36 36 self.dataOut = Spectra()
37 37 self.id_min = None
38 38 self.id_max = None
39 39 self.setupReq = False #Agregar a todas las unidades de proc
40 self.nsamplesFFT = 0
40 41
41 42 def __updateSpecFromVoltage(self):
42 43
43 44
44 45
45 46 self.dataOut.timeZone = self.dataIn.timeZone
46 47 self.dataOut.dstFlag = self.dataIn.dstFlag
47 48 self.dataOut.errorCount = self.dataIn.errorCount
48 49 self.dataOut.useLocalTime = self.dataIn.useLocalTime
49 try:
50 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
51 except:
52 pass
50
51 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
53 52 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
53 self.dataOut.ippSeconds = self.dataIn.ippSeconds
54 self.dataOut.ipp = self.dataIn.ipp
54 55 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
55 56 self.dataOut.channelList = self.dataIn.channelList
56 57 self.dataOut.heightList = self.dataIn.heightList
57 58 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
58 59 self.dataOut.nProfiles = self.dataOut.nFFTPoints
59 60 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
60 61 self.dataOut.utctime = self.firstdatatime
61 62 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
62 63 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
63 64 self.dataOut.flagShiftFFT = False
64 65 self.dataOut.nCohInt = self.dataIn.nCohInt
65 66 self.dataOut.nIncohInt = 1
66 self.dataOut.radar_ipp = self.dataIn.radar_ipp
67 self.dataOut.sampled_heightsFFT = self.dataIn.sampled_heightsFFT
68 self.dataOut.pulseLength_TxA = self.dataIn.pulseLength_TxA
69 67 self.dataOut.deltaHeight = self.dataIn.deltaHeight
70 68 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
71 69 self.dataOut.frequency = self.dataIn.frequency
72 70 self.dataOut.realtime = self.dataIn.realtime
73 71 self.dataOut.azimuth = self.dataIn.azimuth
74 72 self.dataOut.zenith = self.dataIn.zenith
75 73 self.dataOut.codeList = self.dataIn.codeList
76 74 self.dataOut.azimuthList = self.dataIn.azimuthList
77 75 self.dataOut.elevationList = self.dataIn.elevationList
78 76
79 77
80 78 def __getFft(self):
81 79 # print("fft donw")
82 80 """
83 81 Convierte valores de Voltaje a Spectra
84 82
85 83 Affected:
86 84 self.dataOut.data_spc
87 85 self.dataOut.data_cspc
88 86 self.dataOut.data_dc
89 87 self.dataOut.heightList
90 88 self.profIndex
91 89 self.buffer
92 90 self.dataOut.flagNoData
93 91 """
94 92 fft_volt = numpy.fft.fft(
95 93 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
96 94 fft_volt = fft_volt.astype(numpy.dtype('complex'))
97 95 dc = fft_volt[:, 0, :]
98 96
99 97 # calculo de self-spectra
100 98 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
101 99 spc = fft_volt * numpy.conjugate(fft_volt)
102 100 spc = spc.real
103 101
104 102 blocksize = 0
105 103 blocksize += dc.size
106 104 blocksize += spc.size
107 105
108 106 cspc = None
109 107 pairIndex = 0
110 108 if self.dataOut.pairsList != None:
111 109 # calculo de cross-spectra
112 110 cspc = numpy.zeros(
113 111 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
114 112 for pair in self.dataOut.pairsList:
115 113 if pair[0] not in self.dataOut.channelList:
116 114 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
117 115 str(pair), str(self.dataOut.channelList)))
118 116 if pair[1] not in self.dataOut.channelList:
119 117 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
120 118 str(pair), str(self.dataOut.channelList)))
121 119
122 120 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
123 121 numpy.conjugate(fft_volt[pair[1], :, :])
124 122 pairIndex += 1
125 123 blocksize += cspc.size
126 124
127 125 self.dataOut.data_spc = spc
128 126 self.dataOut.data_cspc = cspc
129 127 self.dataOut.data_dc = dc
130 128 self.dataOut.blockSize = blocksize
131 129 self.dataOut.flagShiftFFT = False
132 130
133 131 def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False, zeroPad=False):
134 #print("run spc proc")
132
135 133
136 134 try:
137 135 type = self.dataIn.type.decode("utf-8")
138 136 self.dataIn.type = type
139 137 except:
140 138 pass
141 139 if self.dataIn.type == "Spectra":
142 140
143 141 try:
144 142 self.dataOut.copy(self.dataIn)
143 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
144 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
145 145 self.dataOut.nProfiles = self.dataOut.nFFTPoints
146 146 #self.dataOut.nHeights = len(self.dataOut.heightList)
147 147 except Exception as e:
148 148 print("Error dataIn ",e)
149 149
150
151
150 152 if shift_fft:
151 153 #desplaza a la derecha en el eje 2 determinadas posiciones
152 154 shift = int(self.dataOut.nFFTPoints/2)
153 155 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
154 156
155 157 if self.dataOut.data_cspc is not None:
156 158 #desplaza a la derecha en el eje 2 determinadas posiciones
157 159 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
158 160 if pairsList:
159 161 self.__selectPairs(pairsList)
160 162
161 163
162 164 elif self.dataIn.type == "Voltage":
163 165
164 166 self.dataOut.flagNoData = True
165
167 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
168 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
166 169 if nFFTPoints == None:
167 170 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
168 171
169 172 if nProfiles == None:
170 173 nProfiles = nFFTPoints
171 174
172 175 if ippFactor == None:
173 176 self.dataOut.ippFactor = 1
174 177
175 178 self.dataOut.nFFTPoints = nFFTPoints
176 179 #print(" volts ch,prof, h: ", self.dataIn.data.shape)
177 180 if self.buffer is None:
178 181 if not zeroPad:
179 182 self.buffer = numpy.zeros((self.dataIn.nChannels,
180 183 nProfiles,
181 184 self.dataIn.nHeights),
182 185 dtype='complex')
183 186 else:
184 187 self.buffer = numpy.zeros((self.dataIn.nChannels,
185 188 nFFTPoints,
186 189 self.dataIn.nHeights),
187 190 dtype='complex')
188 191
189 192 if self.dataIn.flagDataAsBlock:
190 193 nVoltProfiles = self.dataIn.data.shape[1]
191 194
192 195 if nVoltProfiles == nProfiles or zeroPad:
193 196 self.buffer = self.dataIn.data.copy()
194 197 self.profIndex = nVoltProfiles
195 198
196 199 elif nVoltProfiles < nProfiles:
197 200
198 201 if self.profIndex == 0:
199 202 self.id_min = 0
200 203 self.id_max = nVoltProfiles
201 204
202 205 self.buffer[:, self.id_min:self.id_max,
203 206 :] = self.dataIn.data
204 207 self.profIndex += nVoltProfiles
205 208 self.id_min += nVoltProfiles
206 209 self.id_max += nVoltProfiles
207 210 else:
208 211 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
209 212 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
210 213 self.dataOut.flagNoData = True
211 214 else:
212 215 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
213 216 self.profIndex += 1
214 217
215 218 if self.firstdatatime == None:
216 219 self.firstdatatime = self.dataIn.utctime
217 220
218 221 if self.profIndex == nProfiles or zeroPad:
219 222
220 223 self.__updateSpecFromVoltage()
221 224
222 225 if pairsList == None:
223 226 self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)]
224 227 else:
225 228 self.dataOut.pairsList = pairsList
226 229 self.__getFft()
227 230 self.dataOut.flagNoData = False
228 231 self.firstdatatime = None
232 self.nsamplesFFT = self.profIndex
229 233 self.profIndex = 0
230 234
235 #update Processing Header:
236 self.dataOut.processingHeaderObj.dtype = "Spectra"
237 self.dataOut.processingHeaderObj.nFFTPoints = self.dataOut.nFFTPoints
238 self.dataOut.processingHeaderObj.nSamplesFFT = self.nsamplesFFT
239 self.dataOut.processingHeaderObj.nIncohInt = 1
240
241
231 242 elif self.dataIn.type == "Parameters":
232 243
233 244 self.dataOut.data_spc = self.dataIn.data_spc
234 245 self.dataOut.data_cspc = self.dataIn.data_cspc
235 246 self.dataOut.data_outlier = self.dataIn.data_outlier
236 247 self.dataOut.nProfiles = self.dataIn.nProfiles
237 248 self.dataOut.nIncohInt = self.dataIn.nIncohInt
238 249 self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
239 250 self.dataOut.ippFactor = self.dataIn.ippFactor
240 251 self.dataOut.max_nIncohInt = self.dataIn.max_nIncohInt
241 252 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
253 self.dataOut.ProcessingHeader = self.dataIn.ProcessingHeader.copy()
254 self.dataOut.ippSeconds = self.dataIn.ippSeconds
242 255 self.dataOut.ipp = self.dataIn.ipp
243 256 #self.dataOut.abscissaList = self.dataIn.getVelRange(1)
244 257 #self.dataOut.spc_noise = self.dataIn.getNoise()
245 258 #self.dataOut.spc_range = (self.dataIn.getFreqRange(1) , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1))
246 259 # self.dataOut.normFactor = self.dataIn.normFactor
247 260 if hasattr(self.dataIn, 'channelList'):
248 261 self.dataOut.channelList = self.dataIn.channelList
249 262 if hasattr(self.dataIn, 'pairsList'):
250 263 self.dataOut.pairsList = self.dataIn.pairsList
251 264 self.dataOut.groupList = self.dataIn.pairsList
252 265
253 266 self.dataOut.flagNoData = False
254 267
255 268 if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels
256 269 self.dataOut.ChanDist = self.dataIn.ChanDist
257 270 else: self.dataOut.ChanDist = None
258 271
259 272 #if hasattr(self.dataIn, 'VelRange'): #Velocities range
260 273 # self.dataOut.VelRange = self.dataIn.VelRange
261 274 #else: self.dataOut.VelRange = None
262 275
263 276
264 277
265 278 else:
266 279 raise ValueError("The type of input object {} is not valid".format(
267 280 self.dataIn.type))
281
282
283
284
268 285 #print("spc proc Done", self.dataOut.data_spc.shape)
286 #print(self.dataOut.data_spc)
287 return
269 288
270 289 def __selectPairs(self, pairsList):
271 290
272 291 if not pairsList:
273 292 return
274 293
275 294 pairs = []
276 295 pairsIndex = []
277 296
278 297 for pair in pairsList:
279 298 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
280 299 continue
281 300 pairs.append(pair)
282 301 pairsIndex.append(pairs.index(pair))
283 302
284 303 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
285 304 self.dataOut.pairsList = pairs
286 305
287 306 return
288 307
289 308 def selectFFTs(self, minFFT, maxFFT ):
290 309 """
291 310 Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
292 311 minFFT<= FFT <= maxFFT
293 312 """
294 313
295 314 if (minFFT > maxFFT):
296 315 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
297 316
298 317 if (minFFT < self.dataOut.getFreqRange()[0]):
299 318 minFFT = self.dataOut.getFreqRange()[0]
300 319
301 320 if (maxFFT > self.dataOut.getFreqRange()[-1]):
302 321 maxFFT = self.dataOut.getFreqRange()[-1]
303 322
304 323 minIndex = 0
305 324 maxIndex = 0
306 325 FFTs = self.dataOut.getFreqRange()
307 326
308 327 inda = numpy.where(FFTs >= minFFT)
309 328 indb = numpy.where(FFTs <= maxFFT)
310 329
311 330 try:
312 331 minIndex = inda[0][0]
313 332 except:
314 333 minIndex = 0
315 334
316 335 try:
317 336 maxIndex = indb[0][-1]
318 337 except:
319 338 maxIndex = len(FFTs)
320 339
321 340 self.selectFFTsByIndex(minIndex, maxIndex)
322 341
323 342 return 1
324 343
325 344 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
326 345 newheis = numpy.where(
327 346 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
328 347
329 348 if hei_ref != None:
330 349 newheis = numpy.where(self.dataOut.heightList > hei_ref)
331 350
332 351 minIndex = min(newheis[0])
333 352 maxIndex = max(newheis[0])
334 353 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
335 354 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
336 355
337 356 # determina indices
338 357 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
339 358 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
340 359 avg_dB = 10 * \
341 360 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
342 361 beacon_dB = numpy.sort(avg_dB)[-nheis:]
343 362 beacon_heiIndexList = []
344 363 for val in avg_dB.tolist():
345 364 if val >= beacon_dB[0]:
346 365 beacon_heiIndexList.append(avg_dB.tolist().index(val))
347 366
348 367 #data_spc = data_spc[:,:,beacon_heiIndexList]
349 368 data_cspc = None
350 369 if self.dataOut.data_cspc is not None:
351 370 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
352 371 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
353 372
354 373 data_dc = None
355 374 if self.dataOut.data_dc is not None:
356 375 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
357 376 #data_dc = data_dc[:,beacon_heiIndexList]
358 377
359 378 self.dataOut.data_spc = data_spc
360 379 self.dataOut.data_cspc = data_cspc
361 380 self.dataOut.data_dc = data_dc
362 381 self.dataOut.heightList = heightList
363 382 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
364 383
365 384 return 1
366 385
367 386 def selectFFTsByIndex(self, minIndex, maxIndex):
368 387 """
369 388
370 389 """
371 390
372 391 if (minIndex < 0) or (minIndex > maxIndex):
373 392 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
374 393
375 394 if (maxIndex >= self.dataOut.nProfiles):
376 395 maxIndex = self.dataOut.nProfiles-1
377 396
378 397 #Spectra
379 398 data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
380 399
381 400 data_cspc = None
382 401 if self.dataOut.data_cspc is not None:
383 402 data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
384 403
385 404 data_dc = None
386 405 if self.dataOut.data_dc is not None:
387 406 data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
388 407
389 408 self.dataOut.data_spc = data_spc
390 409 self.dataOut.data_cspc = data_cspc
391 410 self.dataOut.data_dc = data_dc
392 411
393 412 self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
394 413 self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
395 414 self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
396 415
397 416 return 1
398 417
399 418 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
400 419 # validacion de rango
401 420 if minHei == None:
402 421 minHei = self.dataOut.heightList[0]
403 422
404 423 if maxHei == None:
405 424 maxHei = self.dataOut.heightList[-1]
406 425
407 426 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
408 427 print('minHei: %.2f is out of the heights range' % (minHei))
409 428 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
410 429 minHei = self.dataOut.heightList[0]
411 430
412 431 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
413 432 print('maxHei: %.2f is out of the heights range' % (maxHei))
414 433 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
415 434 maxHei = self.dataOut.heightList[-1]
416 435
417 436 # validacion de velocidades
418 437 velrange = self.dataOut.getVelRange(1)
419 438
420 439 if minVel == None:
421 440 minVel = velrange[0]
422 441
423 442 if maxVel == None:
424 443 maxVel = velrange[-1]
425 444
426 445 if (minVel < velrange[0]) or (minVel > maxVel):
427 446 print('minVel: %.2f is out of the velocity range' % (minVel))
428 447 print('minVel is setting to %.2f' % (velrange[0]))
429 448 minVel = velrange[0]
430 449
431 450 if (maxVel > velrange[-1]) or (maxVel < minVel):
432 451 print('maxVel: %.2f is out of the velocity range' % (maxVel))
433 452 print('maxVel is setting to %.2f' % (velrange[-1]))
434 453 maxVel = velrange[-1]
435 454
436 455 # seleccion de indices para rango
437 456 minIndex = 0
438 457 maxIndex = 0
439 458 heights = self.dataOut.heightList
440 459
441 460 inda = numpy.where(heights >= minHei)
442 461 indb = numpy.where(heights <= maxHei)
443 462
444 463 try:
445 464 minIndex = inda[0][0]
446 465 except:
447 466 minIndex = 0
448 467
449 468 try:
450 469 maxIndex = indb[0][-1]
451 470 except:
452 471 maxIndex = len(heights)
453 472
454 473 if (minIndex < 0) or (minIndex > maxIndex):
455 474 raise ValueError("some value in (%d,%d) is not valid" % (
456 475 minIndex, maxIndex))
457 476
458 477 if (maxIndex >= self.dataOut.nHeights):
459 478 maxIndex = self.dataOut.nHeights - 1
460 479
461 480 # seleccion de indices para velocidades
462 481 indminvel = numpy.where(velrange >= minVel)
463 482 indmaxvel = numpy.where(velrange <= maxVel)
464 483 try:
465 484 minIndexVel = indminvel[0][0]
466 485 except:
467 486 minIndexVel = 0
468 487
469 488 try:
470 489 maxIndexVel = indmaxvel[0][-1]
471 490 except:
472 491 maxIndexVel = len(velrange)
473 492
474 493 # seleccion del espectro
475 494 data_spc = self.dataOut.data_spc[:,
476 495 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
477 496 # estimacion de ruido
478 497 noise = numpy.zeros(self.dataOut.nChannels)
479 498
480 499 for channel in range(self.dataOut.nChannels):
481 500 daux = data_spc[channel, :, :]
482 501 sortdata = numpy.sort(daux, axis=None)
483 502 noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt)
484 503
485 504 self.dataOut.noise_estimation = noise.copy()
486 505
487 506 return 1
488 507
489 508 class removeDC(Operation):
490 509
491 510 def run(self, dataOut, mode=2):
492 511 self.dataOut = dataOut
493 512 jspectra = self.dataOut.data_spc
494 513 jcspectra = self.dataOut.data_cspc
495 514
496 515 num_chan = jspectra.shape[0]
497 516 num_hei = jspectra.shape[2]
498 517
499 518 if jcspectra is not None:
500 519 jcspectraExist = True
501 520 num_pairs = jcspectra.shape[0]
502 521 else:
503 522 jcspectraExist = False
504 523
505 524 freq_dc = int(jspectra.shape[1] / 2)
506 525 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
507 526 ind_vel = ind_vel.astype(int)
508 527
509 528 if ind_vel[0] < 0:
510 529 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
511 530
512 531 if mode == 1:
513 532 jspectra[:, freq_dc, :] = (
514 533 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
515 534
516 535 if jcspectraExist:
517 536 jcspectra[:, freq_dc, :] = (
518 537 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
519 538
520 539 if mode == 2:
521 540
522 541 vel = numpy.array([-2, -1, 1, 2])
523 542 xx = numpy.zeros([4, 4])
524 543
525 544 for fil in range(4):
526 545 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
527 546
528 547 xx_inv = numpy.linalg.inv(xx)
529 548 xx_aux = xx_inv[0, :]
530 549
531 550 for ich in range(num_chan):
532 551 yy = jspectra[ich, ind_vel, :]
533 552 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
534 553
535 554 junkid = jspectra[ich, freq_dc, :] <= 0
536 555 cjunkid = sum(junkid)
537 556
538 557 if cjunkid.any():
539 558 jspectra[ich, freq_dc, junkid.nonzero()] = (
540 559 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
541 560
542 561 if jcspectraExist:
543 562 for ip in range(num_pairs):
544 563 yy = jcspectra[ip, ind_vel, :]
545 564 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
546 565
547 566 self.dataOut.data_spc = jspectra
548 567 self.dataOut.data_cspc = jcspectra
549 568
550 569 return self.dataOut
551 570
552 571 class getNoiseB(Operation):
553 572
554 573 __slots__ =('offset','warnings', 'isConfig', 'minIndex','maxIndex','minIndexFFT','maxIndexFFT')
555 574 def __init__(self):
556 575
557 576 Operation.__init__(self)
558 577 self.isConfig = False
559 578
560 579 def setup(self, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
561 580
562 581 self.warnings = warnings
563 582 if minHei == None:
564 583 minHei = self.dataOut.heightList[0]
565 584
566 585 if maxHei == None:
567 586 maxHei = self.dataOut.heightList[-1]
568 587
569 588 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
570 589 if self.warnings:
571 590 print('minHei: %.2f is out of the heights range' % (minHei))
572 591 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
573 592 minHei = self.dataOut.heightList[0]
574 593
575 594 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
576 595 if self.warnings:
577 596 print('maxHei: %.2f is out of the heights range' % (maxHei))
578 597 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
579 598 maxHei = self.dataOut.heightList[-1]
580 599
581 600
582 601 #indices relativos a los puntos de fft, puede ser de acuerdo a velocidad o frecuencia
583 602 minIndexFFT = 0
584 603 maxIndexFFT = 0
585 604 # validacion de velocidades
586 605 indminPoint = None
587 606 indmaxPoint = None
588 607 if self.dataOut.type == 'Spectra':
589 608 if minVel == None and maxVel == None :
590 609
591 610 freqrange = self.dataOut.getFreqRange(1)
592 611
593 612 if minFreq == None:
594 613 minFreq = freqrange[0]
595 614
596 615 if maxFreq == None:
597 616 maxFreq = freqrange[-1]
598 617
599 618 if (minFreq < freqrange[0]) or (minFreq > maxFreq):
600 619 if self.warnings:
601 620 print('minFreq: %.2f is out of the frequency range' % (minFreq))
602 621 print('minFreq is setting to %.2f' % (freqrange[0]))
603 622 minFreq = freqrange[0]
604 623
605 624 if (maxFreq > freqrange[-1]) or (maxFreq < minFreq):
606 625 if self.warnings:
607 626 print('maxFreq: %.2f is out of the frequency range' % (maxFreq))
608 627 print('maxFreq is setting to %.2f' % (freqrange[-1]))
609 628 maxFreq = freqrange[-1]
610 629
611 630 indminPoint = numpy.where(freqrange >= minFreq)
612 631 indmaxPoint = numpy.where(freqrange <= maxFreq)
613 632
614 633 else:
615 634
616 635 velrange = self.dataOut.getVelRange(1)
617 636
618 637 if minVel == None:
619 638 minVel = velrange[0]
620 639
621 640 if maxVel == None:
622 641 maxVel = velrange[-1]
623 642
624 643 if (minVel < velrange[0]) or (minVel > maxVel):
625 644 if self.warnings:
626 645 print('minVel: %.2f is out of the velocity range' % (minVel))
627 646 print('minVel is setting to %.2f' % (velrange[0]))
628 647 minVel = velrange[0]
629 648
630 649 if (maxVel > velrange[-1]) or (maxVel < minVel):
631 650 if self.warnings:
632 651 print('maxVel: %.2f is out of the velocity range' % (maxVel))
633 652 print('maxVel is setting to %.2f' % (velrange[-1]))
634 653 maxVel = velrange[-1]
635 654
636 655 indminPoint = numpy.where(velrange >= minVel)
637 656 indmaxPoint = numpy.where(velrange <= maxVel)
638 657
639 658
640 659 # seleccion de indices para rango
641 660 minIndex = 0
642 661 maxIndex = 0
643 662 heights = self.dataOut.heightList
644 663
645 664 inda = numpy.where(heights >= minHei)
646 665 indb = numpy.where(heights <= maxHei)
647 666
648 667 try:
649 668 minIndex = inda[0][0]
650 669 except:
651 670 minIndex = 0
652 671
653 672 try:
654 673 maxIndex = indb[0][-1]
655 674 except:
656 675 maxIndex = len(heights)
657 676
658 677 if (minIndex < 0) or (minIndex > maxIndex):
659 678 raise ValueError("some value in (%d,%d) is not valid" % (
660 679 minIndex, maxIndex))
661 680
662 681 if (maxIndex >= self.dataOut.nHeights):
663 682 maxIndex = self.dataOut.nHeights - 1
664 683 #############################################################3
665 684 # seleccion de indices para velocidades
666 685 if self.dataOut.type == 'Spectra':
667 686 try:
668 687 minIndexFFT = indminPoint[0][0]
669 688 except:
670 689 minIndexFFT = 0
671 690
672 691 try:
673 692 maxIndexFFT = indmaxPoint[0][-1]
674 693 except:
675 694 maxIndexFFT = len( self.dataOut.getFreqRange(1))
676 695
677 696 self.minIndex, self.maxIndex, self.minIndexFFT, self.maxIndexFFT = minIndex, maxIndex, minIndexFFT, maxIndexFFT
678 697 self.isConfig = True
679 698 self.offset = 1
680 699 if offset!=None:
681 700 self.offset = 10**(offset/10)
682 701 #print("config getNoiseB Done")
683 702
684 703 def run(self, dataOut, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
685 704 self.dataOut = dataOut
686 705
687 706 if not self.isConfig:
688 707 self.setup(offset, minHei, maxHei,minVel, maxVel, minFreq, maxFreq, warnings)
689 708
690 709 self.dataOut.noise_estimation = None
691 710 noise = None
692 711 #print("data type: ",self.dataOut.type, self.dataOut.nIncohInt, self.dataOut.max_nIncohInt)
693 712 if self.dataOut.type == 'Voltage':
694 713 noise = self.dataOut.getNoise(ymin_index=self.minIndex, ymax_index=self.maxIndex)
695 714 #print(minIndex, maxIndex,minIndexVel, maxIndexVel)
696 715 elif self.dataOut.type == 'Spectra':
697 716 #print(self.dataOut.nChannels, self.minIndex, self.maxIndex,self.minIndexFFT, self.maxIndexFFT, self.dataOut.max_nIncohInt, self.dataOut.nIncohInt)
698 717 noise = numpy.zeros( self.dataOut.nChannels)
699 718 norm = 1
700 719
701 720 for channel in range( self.dataOut.nChannels):
702 721 if not hasattr(self.dataOut.nIncohInt,'__len__'):
703 722 norm = 1
704 723 else:
705 724 norm = self.dataOut.max_nIncohInt[channel]/self.dataOut.nIncohInt[channel, self.minIndex:self.maxIndex]
706 725 #print("norm nIncoh: ", norm ,self.dataOut.data_spc.shape, self.dataOut.max_nIncohInt)
707 726 daux = self.dataOut.data_spc[channel,self.minIndexFFT:self.maxIndexFFT, self.minIndex:self.maxIndex]
708 727 daux = numpy.multiply(daux, norm)
709 728 #print("offset: ", self.offset, 10*numpy.log10(self.offset))
710 729 # noise[channel] = self.getNoiseByMean(daux)/self.offset
711 730 #print(daux.shape, daux)
712 731 #noise[channel] = self.getNoiseByHS(daux, self.dataOut.max_nIncohInt)/self.offset
713 732 sortdata = numpy.sort(daux, axis=None)
714 733
715 734 noise[channel] = _noise.hildebrand_sekhon(sortdata, self.dataOut.max_nIncohInt[channel])/self.offset
716 735
717 736
718 737 #noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex)
719 738 else:
720 739 noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex)
721 740 self.dataOut.noise_estimation = noise.copy() # dataOut.noise
722 741 #print("2: ",10*numpy.log10(self.dataOut.noise_estimation/64))
723 742 #print("2: ",self.dataOut.noise_estimation)
724 743 #print(self.dataOut.flagNoData)
725 744 #print("getNoise Done", noise, self.dataOut.nProfiles ,self.dataOut.ippFactor)
726 745 return self.dataOut
727 746
728 747 def getNoiseByMean(self,data):
729 748 #data debe estar ordenado
730 749 data = numpy.mean(data,axis=1)
731 750 sortdata = numpy.sort(data, axis=None)
732 751 #sortID=data.argsort()
733 752 #print(data.shape)
734 753
735 754 pnoise = None
736 755 j = 0
737 756
738 757 mean = numpy.mean(sortdata)
739 758 min = numpy.min(sortdata)
740 759 delta = mean - min
741 760 indexes = numpy.where(sortdata > (mean+delta))[0] #only array of indexes
742 761 #print(len(indexes))
743 762 if len(indexes)==0:
744 763 pnoise = numpy.mean(sortdata)
745 764 else:
746 765 j = indexes[0]
747 766 pnoise = numpy.mean(sortdata[0:j])
748 767
749 768 # from matplotlib import pyplot as plt
750 769 # plt.plot(sortdata)
751 770 # plt.vlines(j,(pnoise-delta),(pnoise+delta), color='r')
752 771 # plt.show()
753 772 #print("noise: ", 10*numpy.log10(pnoise))
754 773 return pnoise
755 774
756 775 def getNoiseByHS(self,data, navg):
757 776 #data debe estar ordenado
758 777 #data = numpy.mean(data,axis=1)
759 778 sortdata = numpy.sort(data, axis=None)
760 779
761 780 lenOfData = len(sortdata)
762 781 nums_min = lenOfData*0.2
763 782
764 783 if nums_min <= 5:
765 784
766 785 nums_min = 5
767 786
768 787 sump = 0.
769 788 sumq = 0.
770 789
771 790 j = 0
772 791 cont = 1
773 792
774 793 while((cont == 1)and(j < lenOfData)):
775 794
776 795 sump += sortdata[j]
777 796 sumq += sortdata[j]**2
778 797 #sumq -= sump**2
779 798 if j > nums_min:
780 799 rtest = float(j)/(j-1) + 1.0/navg
781 800 #if ((sumq*j) > (sump**2)):
782 801 if ((sumq*j) > (rtest*sump**2)):
783 802 j = j - 1
784 803 sump = sump - sortdata[j]
785 804 sumq = sumq - sortdata[j]**2
786 805 cont = 0
787 806
788 807 j += 1
789 808
790 809 lnoise = sump / j
791 810
792 811 return lnoise
793 812
794 813
795 814
796 815 def fit_func( x, a0, a1, a2): #, a3, a4, a5):
797 816 z = (x - a1) / a2
798 817 y = a0 * numpy.exp(-z**2 / a2) #+ a3 + a4 * x + a5 * x**2
799 818 return y
800 819
801 820
802 821 # class CleanRayleigh(Operation):
803 822 #
804 823 # def __init__(self):
805 824 #
806 825 # Operation.__init__(self)
807 826 # self.i=0
808 827 # self.isConfig = False
809 828 # self.__dataReady = False
810 829 # self.__profIndex = 0
811 830 # self.byTime = False
812 831 # self.byProfiles = False
813 832 #
814 833 # self.bloques = None
815 834 # self.bloque0 = None
816 835 #
817 836 # self.index = 0
818 837 #
819 838 # self.buffer = 0
820 839 # self.buffer2 = 0
821 840 # self.buffer3 = 0
822 841 #
823 842 #
824 843 # def setup(self,dataOut,min_hei,max_hei,n, timeInterval,factor_stdv):
825 844 #
826 845 # self.nChannels = dataOut.nChannels
827 846 # self.nProf = dataOut.nProfiles
828 847 # self.nPairs = dataOut.data_cspc.shape[0]
829 848 # self.pairsArray = numpy.array(dataOut.pairsList)
830 849 # self.spectra = dataOut.data_spc
831 850 # self.cspectra = dataOut.data_cspc
832 851 # self.heights = dataOut.heightList #alturas totales
833 852 # self.nHeights = len(self.heights)
834 853 # self.min_hei = min_hei
835 854 # self.max_hei = max_hei
836 855 # if (self.min_hei == None):
837 856 # self.min_hei = 0
838 857 # if (self.max_hei == None):
839 858 # self.max_hei = dataOut.heightList[-1]
840 859 # self.hval = ((self.max_hei>=self.heights) & (self.heights >= self.min_hei)).nonzero()
841 860 # self.heightsClean = self.heights[self.hval] #alturas filtradas
842 861 # self.hval = self.hval[0] # forma (N,), an solo N elementos -> Indices de alturas
843 862 # self.nHeightsClean = len(self.heightsClean)
844 863 # self.channels = dataOut.channelList
845 864 # self.nChan = len(self.channels)
846 865 # self.nIncohInt = dataOut.nIncohInt
847 866 # self.__initime = dataOut.utctime
848 867 # self.maxAltInd = self.hval[-1]+1
849 868 # self.minAltInd = self.hval[0]
850 869 #
851 870 # self.crosspairs = dataOut.pairsList
852 871 # self.nPairs = len(self.crosspairs)
853 872 # self.normFactor = dataOut.normFactor
854 873 # self.nFFTPoints = dataOut.nFFTPoints
855 874 # self.ippSeconds = dataOut.ippSeconds
856 875 # self.currentTime = self.__initime
857 876 # self.pairsArray = numpy.array(dataOut.pairsList)
858 877 # self.factor_stdv = factor_stdv
859 878 #
860 879 # if n != None :
861 880 # self.byProfiles = True
862 881 # self.nIntProfiles = n
863 882 # else:
864 883 # self.__integrationtime = timeInterval
865 884 #
866 885 # self.__dataReady = False
867 886 # self.isConfig = True
868 887 #
869 888 #
870 889 #
871 890 # def run(self, dataOut,min_hei=None,max_hei=None, n=None, timeInterval=10,factor_stdv=2.5):
872 891 # #print("runing cleanRayleigh")
873 892 # if not self.isConfig :
874 893 #
875 894 # self.setup(dataOut, min_hei,max_hei,n,timeInterval,factor_stdv)
876 895 #
877 896 # tini=dataOut.utctime
878 897 #
879 898 # if self.byProfiles:
880 899 # if self.__profIndex == self.nIntProfiles:
881 900 # self.__dataReady = True
882 901 # else:
883 902 # if (tini - self.__initime) >= self.__integrationtime:
884 903 #
885 904 # self.__dataReady = True
886 905 # self.__initime = tini
887 906 #
888 907 # #if (tini.tm_min % 2) == 0 and (tini.tm_sec < 5 and self.fint==0):
889 908 #
890 909 # if self.__dataReady:
891 910 #
892 911 # self.__profIndex = 0
893 912 # jspc = self.buffer
894 913 # jcspc = self.buffer2
895 914 # #jnoise = self.buffer3
896 915 # self.buffer = dataOut.data_spc
897 916 # self.buffer2 = dataOut.data_cspc
898 917 # #self.buffer3 = dataOut.noise
899 918 # self.currentTime = dataOut.utctime
900 919 # if numpy.any(jspc) :
901 920 # #print( jspc.shape, jcspc.shape)
902 921 # jspc = numpy.reshape(jspc,(int(len(jspc)/self.nChannels),self.nChannels,self.nFFTPoints,self.nHeights))
903 922 # try:
904 923 # jcspc= numpy.reshape(jcspc,(int(len(jcspc)/self.nPairs),self.nPairs,self.nFFTPoints,self.nHeights))
905 924 # except:
906 925 # print("no cspc")
907 926 # self.__dataReady = False
908 927 # #print( jspc.shape, jcspc.shape)
909 928 # dataOut.flagNoData = False
910 929 # else:
911 930 # dataOut.flagNoData = True
912 931 # self.__dataReady = False
913 932 # return dataOut
914 933 # else:
915 934 # #print( len(self.buffer))
916 935 # if numpy.any(self.buffer):
917 936 # self.buffer = numpy.concatenate((self.buffer,dataOut.data_spc), axis=0)
918 937 # try:
919 938 # self.buffer2 = numpy.concatenate((self.buffer2,dataOut.data_cspc), axis=0)
920 939 # self.buffer3 += dataOut.data_dc
921 940 # except:
922 941 # pass
923 942 # else:
924 943 # self.buffer = dataOut.data_spc
925 944 # self.buffer2 = dataOut.data_cspc
926 945 # self.buffer3 = dataOut.data_dc
927 946 # #print self.index, self.fint
928 947 # #print self.buffer2.shape
929 948 # dataOut.flagNoData = True ## NOTE: ?? revisar LUEGO
930 949 # self.__profIndex += 1
931 950 # return dataOut ## NOTE: REV
932 951 #
933 952 #
934 953 # #index = tini.tm_hour*12+tini.tm_min/5
935 954 # '''
936 955 # #REVISAR
937 956 # '''
938 957 # # jspc = jspc/self.nFFTPoints/self.normFactor
939 958 # # jcspc = jcspc/self.nFFTPoints/self.normFactor
940 959 #
941 960 #
942 961 #
943 962 # tmp_spectra,tmp_cspectra = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
944 963 # dataOut.data_spc = tmp_spectra
945 964 # dataOut.data_cspc = tmp_cspectra
946 965 #
947 966 # #dataOut.data_spc,dataOut.data_cspc = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
948 967 #
949 968 # dataOut.data_dc = self.buffer3
950 969 # dataOut.nIncohInt *= self.nIntProfiles
951 970 # dataOut.max_nIncohInt = self.nIntProfiles
952 971 # dataOut.utctime = self.currentTime #tiempo promediado
953 972 # #print("Time: ",time.localtime(dataOut.utctime))
954 973 # # dataOut.data_spc = sat_spectra
955 974 # # dataOut.data_cspc = sat_cspectra
956 975 # self.buffer = 0
957 976 # self.buffer2 = 0
958 977 # self.buffer3 = 0
959 978 #
960 979 # return dataOut
961 980 #
962 981 # def cleanRayleigh(self,dataOut,spectra,cspectra,factor_stdv):
963 982 # print("OP cleanRayleigh")
964 983 # #import matplotlib.pyplot as plt
965 984 # #for k in range(149):
966 985 # #channelsProcssd = []
967 986 # #channelA_ok = False
968 987 # #rfunc = cspectra.copy() #self.bloques
969 988 # rfunc = spectra.copy()
970 989 # #rfunc = cspectra
971 990 # #val_spc = spectra*0.0 #self.bloque0*0.0
972 991 # #val_cspc = cspectra*0.0 #self.bloques*0.0
973 992 # #in_sat_spectra = spectra.copy() #self.bloque0
974 993 # #in_sat_cspectra = cspectra.copy() #self.bloques
975 994 #
976 995 #
977 996 # ###ONLY FOR TEST:
978 997 # raxs = math.ceil(math.sqrt(self.nPairs))
979 998 # if raxs == 0:
980 999 # raxs = 1
981 1000 # caxs = math.ceil(self.nPairs/raxs)
982 1001 # if self.nPairs <4:
983 1002 # raxs = 2
984 1003 # caxs = 2
985 1004 # #print(raxs, caxs)
986 1005 # fft_rev = 14 #nFFT to plot
987 1006 # hei_rev = ((self.heights >= 550) & (self.heights <= 551)).nonzero() #hei to plot
988 1007 # hei_rev = hei_rev[0]
989 1008 # #print(hei_rev)
990 1009 #
991 1010 # #print numpy.absolute(rfunc[:,0,0,14])
992 1011 #
993 1012 # gauss_fit, covariance = None, None
994 1013 # for ih in range(self.minAltInd,self.maxAltInd):
995 1014 # for ifreq in range(self.nFFTPoints):
996 1015 # '''
997 1016 # ###ONLY FOR TEST:
998 1017 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
999 1018 # fig, axs = plt.subplots(raxs, caxs)
1000 1019 # fig2, axs2 = plt.subplots(raxs, caxs)
1001 1020 # col_ax = 0
1002 1021 # row_ax = 0
1003 1022 # '''
1004 1023 # #print(self.nPairs)
1005 1024 # for ii in range(self.nChan): #PARES DE CANALES SELF y CROSS
1006 1025 # # if self.crosspairs[ii][1]-self.crosspairs[ii][0] > 1: # APLICAR SOLO EN PARES CONTIGUOS
1007 1026 # # continue
1008 1027 # # if not self.crosspairs[ii][0] in channelsProcssd:
1009 1028 # # channelA_ok = True
1010 1029 # #print("pair: ",self.crosspairs[ii])
1011 1030 # '''
1012 1031 # ###ONLY FOR TEST:
1013 1032 # if (col_ax%caxs==0 and col_ax!=0 and self.nPairs !=1):
1014 1033 # col_ax = 0
1015 1034 # row_ax += 1
1016 1035 # '''
1017 1036 # func2clean = 10*numpy.log10(numpy.absolute(rfunc[:,ii,ifreq,ih])) #Potencia?
1018 1037 # #print(func2clean.shape)
1019 1038 # val = (numpy.isfinite(func2clean)==True).nonzero()
1020 1039 #
1021 1040 # if len(val)>0: #limitador
1022 1041 # min_val = numpy.around(numpy.amin(func2clean)-2) #> (-40)
1023 1042 # if min_val <= -40 :
1024 1043 # min_val = -40
1025 1044 # max_val = numpy.around(numpy.amax(func2clean)+2) #< 200
1026 1045 # if max_val >= 200 :
1027 1046 # max_val = 200
1028 1047 # #print min_val, max_val
1029 1048 # step = 1
1030 1049 # #print("Getting bins and the histogram")
1031 1050 # x_dist = min_val + numpy.arange(1 + ((max_val-(min_val))/step))*step
1032 1051 # y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
1033 1052 # #print(len(y_dist),len(binstep[:-1]))
1034 1053 # #print(row_ax,col_ax, " ..")
1035 1054 # #print(self.pairsArray[ii][0],self.pairsArray[ii][1])
1036 1055 # mean = numpy.sum(x_dist * y_dist) / numpy.sum(y_dist)
1037 1056 # sigma = numpy.sqrt(numpy.sum(y_dist * (x_dist - mean)**2) / numpy.sum(y_dist))
1038 1057 # parg = [numpy.amax(y_dist),mean,sigma]
1039 1058 #
1040 1059 # newY = None
1041 1060 #
1042 1061 # try :
1043 1062 # gauss_fit, covariance = curve_fit(fit_func, x_dist, y_dist,p0=parg)
1044 1063 # mode = gauss_fit[1]
1045 1064 # stdv = gauss_fit[2]
1046 1065 # #print(" FIT OK",gauss_fit)
1047 1066 # '''
1048 1067 # ###ONLY FOR TEST:
1049 1068 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1050 1069 # newY = fit_func(x_dist,gauss_fit[0],gauss_fit[1],gauss_fit[2])
1051 1070 # axs[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
1052 1071 # axs[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
1053 1072 # axs[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
1054 1073 # '''
1055 1074 # except:
1056 1075 # mode = mean
1057 1076 # stdv = sigma
1058 1077 # #print("FIT FAIL")
1059 1078 # #continue
1060 1079 #
1061 1080 #
1062 1081 # #print(mode,stdv)
1063 1082 # #Removing echoes greater than mode + std_factor*stdv
1064 1083 # noval = (abs(func2clean - mode)>=(factor_stdv*stdv)).nonzero()
1065 1084 # #noval tiene los indices que se van a remover
1066 1085 # #print("Chan ",ii," novals: ",len(noval[0]))
1067 1086 # if len(noval[0]) > 0: #forma de array (N,) es igual a longitud (N)
1068 1087 # novall = ((func2clean - mode) >= (factor_stdv*stdv)).nonzero()
1069 1088 # #print(novall)
1070 1089 # #print(" ",self.pairsArray[ii])
1071 1090 # #cross_pairs = self.pairsArray[ii]
1072 1091 # #Getting coherent echoes which are removed.
1073 1092 # # if len(novall[0]) > 0:
1074 1093 # #
1075 1094 # # val_spc[novall[0],cross_pairs[0],ifreq,ih] = 1
1076 1095 # # val_spc[novall[0],cross_pairs[1],ifreq,ih] = 1
1077 1096 # # val_cspc[novall[0],ii,ifreq,ih] = 1
1078 1097 # #print("OUT NOVALL 1")
1079 1098 # try:
1080 1099 # pair = (self.channels[ii],self.channels[ii + 1])
1081 1100 # except:
1082 1101 # pair = (99,99)
1083 1102 # #print("par ", pair)
1084 1103 # if ( pair in self.crosspairs):
1085 1104 # q = self.crosspairs.index(pair)
1086 1105 # #print("está aqui: ", q, (ii,ii + 1))
1087 1106 # new_a = numpy.delete(cspectra[:,q,ifreq,ih], noval[0])
1088 1107 # cspectra[noval,q,ifreq,ih] = numpy.mean(new_a) #mean CrossSpectra
1089 1108 #
1090 1109 # #if channelA_ok:
1091 1110 # #chA = self.channels.index(cross_pairs[0])
1092 1111 # new_b = numpy.delete(spectra[:,ii,ifreq,ih], noval[0])
1093 1112 # spectra[noval,ii,ifreq,ih] = numpy.mean(new_b) #mean Spectra Pair A
1094 1113 # #channelA_ok = False
1095 1114 #
1096 1115 # # chB = self.channels.index(cross_pairs[1])
1097 1116 # # new_c = numpy.delete(spectra[:,chB,ifreq,ih], noval[0])
1098 1117 # # spectra[noval,chB,ifreq,ih] = numpy.mean(new_c) #mean Spectra Pair B
1099 1118 # #
1100 1119 # # channelsProcssd.append(self.crosspairs[ii][0]) # save channel A
1101 1120 # # channelsProcssd.append(self.crosspairs[ii][1]) # save channel B
1102 1121 # '''
1103 1122 # ###ONLY FOR TEST:
1104 1123 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1105 1124 # func2clean = 10*numpy.log10(numpy.absolute(spectra[:,ii,ifreq,ih]))
1106 1125 # y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
1107 1126 # axs2[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
1108 1127 # axs2[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
1109 1128 # axs2[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
1110 1129 # '''
1111 1130 # '''
1112 1131 # ###ONLY FOR TEST:
1113 1132 # col_ax += 1 #contador de ploteo columnas
1114 1133 # ##print(col_ax)
1115 1134 # ###ONLY FOR TEST:
1116 1135 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1117 1136 # title = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km"
1118 1137 # title2 = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km CLEANED"
1119 1138 # fig.suptitle(title)
1120 1139 # fig2.suptitle(title2)
1121 1140 # plt.show()
1122 1141 # '''
1123 1142 # ##################################################################################################
1124 1143 #
1125 1144 # #print("Getting average of the spectra and cross-spectra from incoherent echoes.")
1126 1145 # out_spectra = numpy.zeros([self.nChan,self.nFFTPoints,self.nHeights], dtype=float) #+numpy.nan
1127 1146 # out_cspectra = numpy.zeros([self.nPairs,self.nFFTPoints,self.nHeights], dtype=complex) #+numpy.nan
1128 1147 # for ih in range(self.nHeights):
1129 1148 # for ifreq in range(self.nFFTPoints):
1130 1149 # for ich in range(self.nChan):
1131 1150 # tmp = spectra[:,ich,ifreq,ih]
1132 1151 # valid = (numpy.isfinite(tmp[:])==True).nonzero()
1133 1152 #
1134 1153 # if len(valid[0]) >0 :
1135 1154 # out_spectra[ich,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
1136 1155 #
1137 1156 # for icr in range(self.nPairs):
1138 1157 # tmp = numpy.squeeze(cspectra[:,icr,ifreq,ih])
1139 1158 # valid = (numpy.isfinite(tmp)==True).nonzero()
1140 1159 # if len(valid[0]) > 0:
1141 1160 # out_cspectra[icr,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
1142 1161 #
1143 1162 # return out_spectra, out_cspectra
1144 1163 #
1145 1164 # def REM_ISOLATED_POINTS(self,array,rth):
1146 1165 # # import matplotlib.pyplot as plt
1147 1166 # if rth == None :
1148 1167 # rth = 4
1149 1168 # #print("REM ISO")
1150 1169 # num_prof = len(array[0,:,0])
1151 1170 # num_hei = len(array[0,0,:])
1152 1171 # n2d = len(array[:,0,0])
1153 1172 #
1154 1173 # for ii in range(n2d) :
1155 1174 # #print ii,n2d
1156 1175 # tmp = array[ii,:,:]
1157 1176 # #print tmp.shape, array[ii,101,:],array[ii,102,:]
1158 1177 #
1159 1178 # # fig = plt.figure(figsize=(6,5))
1160 1179 # # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1161 1180 # # ax = fig.add_axes([left, bottom, width, height])
1162 1181 # # x = range(num_prof)
1163 1182 # # y = range(num_hei)
1164 1183 # # cp = ax.contour(y,x,tmp)
1165 1184 # # ax.clabel(cp, inline=True,fontsize=10)
1166 1185 # # plt.show()
1167 1186 #
1168 1187 # #indxs = WHERE(FINITE(tmp) AND tmp GT 0,cindxs)
1169 1188 # tmp = numpy.reshape(tmp,num_prof*num_hei)
1170 1189 # indxs1 = (numpy.isfinite(tmp)==True).nonzero()
1171 1190 # indxs2 = (tmp > 0).nonzero()
1172 1191 #
1173 1192 # indxs1 = (indxs1[0])
1174 1193 # indxs2 = indxs2[0]
1175 1194 # #indxs1 = numpy.array(indxs1[0])
1176 1195 # #indxs2 = numpy.array(indxs2[0])
1177 1196 # indxs = None
1178 1197 # #print indxs1 , indxs2
1179 1198 # for iv in range(len(indxs2)):
1180 1199 # indv = numpy.array((indxs1 == indxs2[iv]).nonzero())
1181 1200 # #print len(indxs2), indv
1182 1201 # if len(indv[0]) > 0 :
1183 1202 # indxs = numpy.concatenate((indxs,indxs2[iv]), axis=None)
1184 1203 # # print indxs
1185 1204 # indxs = indxs[1:]
1186 1205 # #print(indxs, len(indxs))
1187 1206 # if len(indxs) < 4 :
1188 1207 # array[ii,:,:] = 0.
1189 1208 # return
1190 1209 #
1191 1210 # xpos = numpy.mod(indxs ,num_hei)
1192 1211 # ypos = (indxs / num_hei)
1193 1212 # sx = numpy.argsort(xpos) # Ordering respect to "x" (time)
1194 1213 # #print sx
1195 1214 # xpos = xpos[sx]
1196 1215 # ypos = ypos[sx]
1197 1216 #
1198 1217 # # *********************************** Cleaning isolated points **********************************
1199 1218 # ic = 0
1200 1219 # while True :
1201 1220 # r = numpy.sqrt(list(numpy.power((xpos[ic]-xpos),2)+ numpy.power((ypos[ic]-ypos),2)))
1202 1221 # #no_coh = WHERE(FINITE(r) AND (r LE rth),cno_coh)
1203 1222 # #plt.plot(r)
1204 1223 # #plt.show()
1205 1224 # no_coh1 = (numpy.isfinite(r)==True).nonzero()
1206 1225 # no_coh2 = (r <= rth).nonzero()
1207 1226 # #print r, no_coh1, no_coh2
1208 1227 # no_coh1 = numpy.array(no_coh1[0])
1209 1228 # no_coh2 = numpy.array(no_coh2[0])
1210 1229 # no_coh = None
1211 1230 # #print valid1 , valid2
1212 1231 # for iv in range(len(no_coh2)):
1213 1232 # indv = numpy.array((no_coh1 == no_coh2[iv]).nonzero())
1214 1233 # if len(indv[0]) > 0 :
1215 1234 # no_coh = numpy.concatenate((no_coh,no_coh2[iv]), axis=None)
1216 1235 # no_coh = no_coh[1:]
1217 1236 # #print len(no_coh), no_coh
1218 1237 # if len(no_coh) < 4 :
1219 1238 # #print xpos[ic], ypos[ic], ic
1220 1239 # # plt.plot(r)
1221 1240 # # plt.show()
1222 1241 # xpos[ic] = numpy.nan
1223 1242 # ypos[ic] = numpy.nan
1224 1243 #
1225 1244 # ic = ic + 1
1226 1245 # if (ic == len(indxs)) :
1227 1246 # break
1228 1247 # #print( xpos, ypos)
1229 1248 #
1230 1249 # indxs = (numpy.isfinite(list(xpos))==True).nonzero()
1231 1250 # #print indxs[0]
1232 1251 # if len(indxs[0]) < 4 :
1233 1252 # array[ii,:,:] = 0.
1234 1253 # return
1235 1254 #
1236 1255 # xpos = xpos[indxs[0]]
1237 1256 # ypos = ypos[indxs[0]]
1238 1257 # for i in range(0,len(ypos)):
1239 1258 # ypos[i]=int(ypos[i])
1240 1259 # junk = tmp
1241 1260 # tmp = junk*0.0
1242 1261 #
1243 1262 # tmp[list(xpos + (ypos*num_hei))] = junk[list(xpos + (ypos*num_hei))]
1244 1263 # array[ii,:,:] = numpy.reshape(tmp,(num_prof,num_hei))
1245 1264 #
1246 1265 # #print array.shape
1247 1266 # #tmp = numpy.reshape(tmp,(num_prof,num_hei))
1248 1267 # #print tmp.shape
1249 1268 #
1250 1269 # # fig = plt.figure(figsize=(6,5))
1251 1270 # # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1252 1271 # # ax = fig.add_axes([left, bottom, width, height])
1253 1272 # # x = range(num_prof)
1254 1273 # # y = range(num_hei)
1255 1274 # # cp = ax.contour(y,x,array[ii,:,:])
1256 1275 # # ax.clabel(cp, inline=True,fontsize=10)
1257 1276 # # plt.show()
1258 1277 # return array
1259 1278 #
1260 1279
1261 1280 class IntegrationFaradaySpectra(Operation):
1262 1281
1263 1282 __profIndex = 0
1264 1283 __withOverapping = False
1265 1284
1266 1285 __byTime = False
1267 1286 __initime = None
1268 1287 __lastdatatime = None
1269 1288 __integrationtime = None
1270 1289
1271 1290 __buffer_spc = None
1272 1291 __buffer_cspc = None
1273 1292 __buffer_dc = None
1274 1293
1275 1294 __dataReady = False
1276 1295
1277 1296 __timeInterval = None
1278 1297 n_ints = None #matriz de numero de integracions (CH,HEI)
1279 1298 n = None
1280 1299 minHei_ind = None
1281 1300 maxHei_ind = None
1282 1301 navg = 1.0
1283 1302 factor = 0.0
1284 1303 dataoutliers = None # (CHANNELS, HEIGHTS)
1285 1304
1286 1305 def __init__(self):
1287 1306
1288 1307 Operation.__init__(self)
1289 1308
1290 1309 def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None, minHei=None, maxHei=None, avg=1,factor=0.75):
1291 1310 """
1292 1311 Set the parameters of the integration class.
1293 1312
1294 1313 Inputs:
1295 1314
1296 1315 n : Number of coherent integrations
1297 1316 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1298 1317 overlapping :
1299 1318
1300 1319 """
1301 1320
1302 1321 self.__initime = None
1303 1322 self.__lastdatatime = 0
1304 1323
1305 1324 self.__buffer_spc = []
1306 1325 self.__buffer_cspc = []
1307 1326 self.__buffer_dc = 0
1308 1327
1309 1328 self.__profIndex = 0
1310 1329 self.__dataReady = False
1311 1330 self.__byTime = False
1312 1331
1313 1332 self.factor = factor
1314 1333 self.navg = avg
1315 1334 #self.ByLags = dataOut.ByLags ###REDEFINIR
1316 1335 self.ByLags = False
1317 1336 self.maxProfilesInt = 0
1318 1337 self.__nChannels = dataOut.nChannels
1319 1338 if DPL != None:
1320 1339 self.DPL=DPL
1321 1340 else:
1322 1341 #self.DPL=dataOut.DPL ###REDEFINIR
1323 1342 self.DPL=0
1324 1343
1325 1344 if n is None and timeInterval is None:
1326 1345 raise ValueError("n or timeInterval should be specified ...")
1327 1346
1328 1347 if n is not None:
1329 1348 self.n = int(n)
1330 1349 else:
1331 1350 self.__integrationtime = int(timeInterval)
1332 1351 self.n = None
1333 1352 self.__byTime = True
1334 1353
1354
1335 1355 if minHei == None:
1336 1356 minHei = self.dataOut.heightList[0]
1337 1357
1338 1358 if maxHei == None:
1339 1359 maxHei = self.dataOut.heightList[-1]
1340 1360
1341 1361 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
1342 1362 print('minHei: %.2f is out of the heights range' % (minHei))
1343 1363 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
1344 1364 minHei = self.dataOut.heightList[0]
1345 1365
1346 1366 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
1347 1367 print('maxHei: %.2f is out of the heights range' % (maxHei))
1348 1368 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
1349 1369 maxHei = self.dataOut.heightList[-1]
1350 1370
1351 1371 ind_list1 = numpy.where(self.dataOut.heightList >= minHei)
1352 1372 ind_list2 = numpy.where(self.dataOut.heightList <= maxHei)
1353 1373 self.minHei_ind = ind_list1[0][0]
1354 1374 self.maxHei_ind = ind_list2[0][-1]
1355 #print("setup rem sats done")
1375
1356 1376
1357 1377 def putData(self, data_spc, data_cspc, data_dc):
1358 1378 """
1359 1379 Add a profile to the __buffer_spc and increase in one the __profileIndex
1360 1380
1361 1381 """
1362 1382
1363 1383 self.__buffer_spc.append(data_spc)
1364 1384
1365 1385 if self.__nChannels < 2:
1366 1386 self.__buffer_cspc = None
1367 1387 else:
1368 1388 self.__buffer_cspc.append(data_cspc)
1369 1389
1370 1390 if data_dc is None:
1371 1391 self.__buffer_dc = None
1372 1392 else:
1373 1393 self.__buffer_dc += data_dc
1374 1394
1375 1395 self.__profIndex += 1
1376 1396
1377 1397 return
1378 1398
1379 1399 def hildebrand_sekhon_Integration(self,sortdata,navg, factor):
1380 1400 #data debe estar ordenado
1381 1401 #sortdata = numpy.sort(data, axis=None)
1382 1402 #sortID=data.argsort()
1383 1403 lenOfData = len(sortdata)
1384 1404 nums_min = lenOfData*factor
1385 1405 if nums_min <= 5:
1386 1406 nums_min = 5
1387 1407 sump = 0.
1388 1408 sumq = 0.
1389 1409 j = 0
1390 1410 cont = 1
1391 1411 while((cont == 1)and(j < lenOfData)):
1392 1412 sump += sortdata[j]
1393 1413 sumq += sortdata[j]**2
1394 1414 if j > nums_min:
1395 1415 rtest = float(j)/(j-1) + 1.0/navg
1396 1416 if ((sumq*j) > (rtest*sump**2)):
1397 1417 j = j - 1
1398 1418 sump = sump - sortdata[j]
1399 1419 sumq = sumq - sortdata[j]**2
1400 1420 cont = 0
1401 1421 j += 1
1402 1422 #lnoise = sump / j
1403 1423 #print("H S done")
1404 1424 #return j,sortID
1405 1425 return j
1406 1426
1407 1427
1408 1428 def pushData(self):
1409 1429 """
1410 1430 Return the sum of the last profiles and the profiles used in the sum.
1411 1431
1412 1432 Affected:
1413 1433
1414 1434 self.__profileIndex
1415 1435
1416 1436 """
1417 1437 bufferH=None
1418 1438 buffer=None
1419 1439 buffer1=None
1420 1440 buffer_cspc=None
1421 1441 #print("aes: ", self.__buffer_cspc)
1422 1442 self.__buffer_spc=numpy.array(self.__buffer_spc)
1423 1443 if self.__nChannels > 1 :
1424 1444 self.__buffer_cspc=numpy.array(self.__buffer_cspc)
1425 1445
1426 1446 #print("FREQ_DC",self.__buffer_spc.shape,self.__buffer_cspc.shape)
1427 1447
1428 1448 freq_dc = int(self.__buffer_spc.shape[2] / 2)
1429 1449 #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights)
1430 1450
1431 1451 self.dataOutliers = numpy.zeros((self.nChannels,self.nHeights)) # --> almacen de outliers
1432 1452
1433 1453 for k in range(self.minHei_ind,self.maxHei_ind):
1434 1454 if self.__nChannels > 1:
1435 1455 buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k])
1436 1456
1437 1457 outliers_IDs_cspc=[]
1438 1458 cspc_outliers_exist=False
1439 1459 for i in range(self.nChannels):#dataOut.nChannels):
1440 1460
1441 1461 buffer1=numpy.copy(self.__buffer_spc[:,i,:,k])
1442 1462 indexes=[]
1443 1463 #sortIDs=[]
1444 1464 outliers_IDs=[]
1445 1465
1446 1466 for j in range(self.nProfiles): #frecuencias en el tiempo
1447 1467 # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0
1448 1468 # continue
1449 1469 # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1
1450 1470 # continue
1451 1471 buffer=buffer1[:,j]
1452 1472 sortdata = numpy.sort(buffer, axis=None)
1453 1473
1454 1474 sortID=buffer.argsort()
1455 1475 index = _noise.hildebrand_sekhon2(sortdata,self.navg)
1456 1476
1457 1477 #index,sortID=self.hildebrand_sekhon_Integration(buffer,1,self.factor)
1458 1478
1459 1479 # fig,ax = plt.subplots()
1460 1480 # ax.set_title(str(k)+" "+str(j))
1461 1481 # x=range(len(sortdata))
1462 1482 # ax.scatter(x,sortdata)
1463 1483 # ax.axvline(index)
1464 1484 # plt.show()
1465 1485
1466 1486 indexes.append(index)
1467 1487 #sortIDs.append(sortID)
1468 1488 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
1469 1489
1470 1490 #print("Outliers: ",outliers_IDs)
1471 1491 outliers_IDs=numpy.array(outliers_IDs)
1472 1492 outliers_IDs=outliers_IDs.ravel()
1473 1493 outliers_IDs=numpy.unique(outliers_IDs)
1474 1494 outliers_IDs=outliers_IDs.astype(numpy.dtype('int64'))
1475 1495 indexes=numpy.array(indexes)
1476 1496 indexmin=numpy.min(indexes)
1477 1497
1478 1498
1479 1499 #print(indexmin,buffer1.shape[0], k)
1480 1500
1481 1501 # fig,ax = plt.subplots()
1482 1502 # ax.plot(sortdata)
1483 1503 # ax2 = ax.twinx()
1484 1504 # x=range(len(indexes))
1485 1505 # #plt.scatter(x,indexes)
1486 1506 # ax2.scatter(x,indexes)
1487 1507 # plt.show()
1488 1508
1489 1509 if indexmin != buffer1.shape[0]:
1490 1510 if self.__nChannels > 1:
1491 1511 cspc_outliers_exist= True
1492 1512
1493 1513 lt=outliers_IDs
1494 1514 #avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0)
1495 1515
1496 1516 for p in list(outliers_IDs):
1497 1517 #buffer1[p,:]=avg
1498 1518 buffer1[p,:] = numpy.NaN
1499 1519
1500 1520 self.dataOutliers[i,k] = len(outliers_IDs)
1501 1521
1502 1522
1503 1523 self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1)
1504 1524
1505 1525
1506 1526 if self.__nChannels > 1:
1507 1527 outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs)
1508 1528
1509 1529
1510 1530 if self.__nChannels > 1:
1511 1531 outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64'))
1512 1532 if cspc_outliers_exist:
1513 1533
1514 1534 lt=outliers_IDs_cspc
1515 1535
1516 1536 #avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0)
1517 1537 for p in list(outliers_IDs_cspc):
1518 1538 #buffer_cspc[p,:]=avg
1519 1539 buffer_cspc[p,:] = numpy.NaN
1520 1540
1521 1541 if self.__nChannels > 1:
1522 1542 self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc)
1523 1543
1524 1544
1525 1545
1526 1546
1527 1547 nOutliers = len(outliers_IDs)
1528 1548 #print("Outliers n: ",self.dataOutliers,nOutliers)
1529 1549 buffer=None
1530 1550 bufferH=None
1531 1551 buffer1=None
1532 1552 buffer_cspc=None
1533 1553
1534 1554
1535 1555 buffer=None
1536 1556
1537 1557 #data_spc = numpy.sum(self.__buffer_spc,axis=0)
1538 1558 data_spc = numpy.nansum(self.__buffer_spc,axis=0)
1539 1559 if self.__nChannels > 1:
1540 1560 #data_cspc = numpy.sum(self.__buffer_cspc,axis=0)
1541 1561 data_cspc = numpy.nansum(self.__buffer_cspc,axis=0)
1542 1562 else:
1543 1563 data_cspc = None
1544 1564 data_dc = self.__buffer_dc
1545 1565 #(CH, HEIGH)
1546 1566 self.maxProfilesInt = self.__profIndex - 1
1547 1567 n = self.__profIndex - self.dataOutliers # n becomes a matrix
1548 1568
1549 1569 self.__buffer_spc = []
1550 1570 self.__buffer_cspc = []
1551 1571 self.__buffer_dc = 0
1552 1572 self.__profIndex = 0
1553 1573 #print("cleaned ",data_cspc)
1554 1574 return data_spc, data_cspc, data_dc, n
1555 1575
1556 1576 def byProfiles(self, *args):
1557 1577
1558 1578 self.__dataReady = False
1559 1579 avgdata_spc = None
1560 1580 avgdata_cspc = None
1561 1581 avgdata_dc = None
1562 1582
1563 1583 self.putData(*args)
1564 1584
1565 1585 if self.__profIndex == self.n:
1566 1586
1567 1587 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1568 1588 self.n_ints = n
1569 1589 self.__dataReady = True
1570 1590
1571 1591 return avgdata_spc, avgdata_cspc, avgdata_dc
1572 1592
1573 1593 def byTime(self, datatime, *args):
1574 1594
1575 1595 self.__dataReady = False
1576 1596 avgdata_spc = None
1577 1597 avgdata_cspc = None
1578 1598 avgdata_dc = None
1579 1599
1580 1600 self.putData(*args)
1581 1601
1582 1602 if (datatime - self.__initime) >= self.__integrationtime:
1583 1603 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1584 1604 self.n_ints = n
1585 1605 self.__dataReady = True
1586 1606
1587 1607 return avgdata_spc, avgdata_cspc, avgdata_dc
1588 1608
1589 1609 def integrate(self, datatime, *args):
1590 1610
1591 1611 if self.__profIndex == 0:
1592 1612 self.__initime = datatime
1593 1613
1594 1614 if self.__byTime:
1595 1615 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1596 1616 datatime, *args)
1597 1617 else:
1598 1618 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1599 1619
1600 1620 if not self.__dataReady:
1601 1621 return None, None, None, None
1602 1622
1603 1623 #print("integrate", avgdata_cspc)
1604 1624 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1605 1625
1606 1626 def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False, minHei=None, maxHei=None, avg=1, factor=0.75):
1607 1627 self.dataOut = dataOut
1608 1628 if n == 1:
1609 1629 return self.dataOut
1610
1630 self.dataOut.processingHeaderObj.timeIncohInt = timeInterval
1611 1631 #print("nchannels", self.dataOut.nChannels)
1612 1632 if self.dataOut.nChannels == 1:
1613 1633 self.dataOut.data_cspc = None #si es un solo canal no vale la pena acumular DATOS
1614 1634 #print("IN spc:", self.dataOut.data_spc.shape, self.dataOut.data_cspc)
1615 1635 if not self.isConfig:
1616 1636 self.setup(self.dataOut, n, timeInterval, overlapping,DPL ,minHei, maxHei, avg, factor)
1617 1637 self.isConfig = True
1618 1638
1619 1639 if not self.ByLags:
1620 1640 self.nProfiles=self.dataOut.nProfiles
1621 1641 self.nChannels=self.dataOut.nChannels
1622 1642 self.nHeights=self.dataOut.nHeights
1623 1643 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1624 1644 self.dataOut.data_spc,
1625 1645 self.dataOut.data_cspc,
1626 1646 self.dataOut.data_dc)
1627 1647 else:
1628 1648 self.nProfiles=self.dataOut.nProfiles
1629 1649 self.nChannels=self.dataOut.nChannels
1630 1650 self.nHeights=self.dataOut.nHeights
1631 1651 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1632 1652 self.dataOut.dataLag_spc,
1633 1653 self.dataOut.dataLag_cspc,
1634 1654 self.dataOut.dataLag_dc)
1635 1655 self.dataOut.flagNoData = True
1636 1656 if self.__dataReady:
1637 1657
1638 1658 if not self.ByLags:
1639 1659 if self.nChannels == 1:
1640 1660 #print("f int", avgdata_spc.shape)
1641 1661 self.dataOut.data_spc = avgdata_spc
1642 1662 self.dataOut.data_cspc = None
1643 1663 else:
1644 1664 self.dataOut.data_spc = numpy.squeeze(avgdata_spc)
1645 1665 self.dataOut.data_cspc = numpy.squeeze(avgdata_cspc)
1646 1666 self.dataOut.data_dc = avgdata_dc
1647 1667 self.dataOut.data_outlier = self.dataOutliers
1648 1668
1649 1669 else:
1650 1670 self.dataOut.dataLag_spc = avgdata_spc
1651 1671 self.dataOut.dataLag_cspc = avgdata_cspc
1652 1672 self.dataOut.dataLag_dc = avgdata_dc
1653 1673
1654 1674 self.dataOut.data_spc=self.dataOut.dataLag_spc[:,:,:,self.dataOut.LagPlot]
1655 1675 self.dataOut.data_cspc=self.dataOut.dataLag_cspc[:,:,:,self.dataOut.LagPlot]
1656 1676 self.dataOut.data_dc=self.dataOut.dataLag_dc[:,:,self.dataOut.LagPlot]
1657 1677
1658 1678
1659 1679 self.dataOut.nIncohInt *= self.n_ints
1660 1680 #print("maxProfilesInt: ",self.maxProfilesInt)
1661 1681
1662 1682 self.dataOut.utctime = avgdatatime
1663 1683 self.dataOut.flagNoData = False
1684
1685 # #update Processing Header:
1686 # self.dataOut.processingHeaderObj.nIncohInt =
1687 # self.dataOut.processingHeaderObj.nFFTPoints = self.dataOut.nFFTPoints
1688
1664 1689 #print("Faraday Integration DONE...", self.dataOut.data_cspc)
1665 1690 #print(self.dataOut.flagNoData)
1666 1691 return self.dataOut
1667 1692
1668 1693
1669 1694
1670 1695 class removeInterference(Operation):
1671 1696
1672 1697 def removeInterference3(self, min_hei = None, max_hei = None):
1673 1698
1674 1699 jspectra = self.dataOut.data_spc
1675 1700 #jcspectra = self.dataOut.data_cspc
1676 1701 jnoise = self.dataOut.getNoise()
1677 1702 num_incoh = self.dataOut.max_nIncohInt
1678 1703 #print(jspectra.shape)
1679 1704 num_channel, num_prof, num_hei = jspectra.shape
1680 1705 minHei = min_hei
1681 1706 maxHei = max_hei
1682 1707 ########################################################################
1683 1708 if minHei == None or (minHei < self.dataOut.heightList[0]):
1684 1709 minHei = self.dataOut.heightList[0]
1685 1710
1686 1711 if maxHei == None or (maxHei > self.dataOut.heightList[-1]):
1687 1712 maxHei = self.dataOut.heightList[-1]
1688 1713 minIndex = 0
1689 1714 maxIndex = 0
1690 1715 heights = self.dataOut.heightList
1691 1716
1692 1717 inda = numpy.where(heights >= minHei)
1693 1718 indb = numpy.where(heights <= maxHei)
1694 1719
1695 1720 try:
1696 1721 minIndex = inda[0][0]
1697 1722 except:
1698 1723 minIndex = 0
1699 1724 try:
1700 1725 maxIndex = indb[0][-1]
1701 1726 except:
1702 1727 maxIndex = len(heights)
1703 1728
1704 1729 if (minIndex < 0) or (minIndex > maxIndex):
1705 1730 raise ValueError("some value in (%d,%d) is not valid" % (
1706 1731 minIndex, maxIndex))
1707 1732 if (maxIndex >= self.dataOut.nHeights):
1708 1733 maxIndex = self.dataOut.nHeights - 1
1709 1734
1710 1735 ########################################################################
1711 1736
1712 1737
1713 1738 #dataOut.max_nIncohInt * dataOut.nCohInt
1714 1739 norm = self.dataOut.nIncohInt /self.dataOut.max_nIncohInt
1715 1740 #print(norm.shape)
1716 1741 # Subrutina de Remocion de la Interferencia
1717 1742 for ich in range(num_channel):
1718 1743 # Se ordena los espectros segun su potencia (menor a mayor)
1719 1744 #power = jspectra[ich, mask_prof, :]
1720 1745 interf = jspectra[ich, :, minIndex:maxIndex]
1721 1746 #print(interf.shape)
1722 1747 inttef = interf.mean(axis=1)
1723 1748
1724 1749 for hei in range(num_hei):
1725 1750 temp = jspectra[ich,:, hei]
1726 1751 temp -= inttef
1727 1752 temp += jnoise[ich]*norm[ich,hei]
1728 1753 jspectra[ich,:, hei] = temp
1729 1754
1730 1755 # Guardar Resultados
1731 1756 self.dataOut.data_spc = jspectra
1732 1757 #self.dataOut.data_cspc = jcspectra
1733 1758
1734 1759 return 1
1735 1760
1736 1761 def removeInterference2(self):
1737 1762
1738 1763 cspc = self.dataOut.data_cspc
1739 1764 spc = self.dataOut.data_spc
1740 1765 Heights = numpy.arange(cspc.shape[2])
1741 1766 realCspc = numpy.abs(cspc)
1742 1767
1743 1768 for i in range(cspc.shape[0]):
1744 1769 LinePower= numpy.sum(realCspc[i], axis=0)
1745 1770 Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
1746 1771 SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
1747 1772 InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
1748 1773 InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
1749 1774 InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
1750 1775
1751 1776
1752 1777 InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
1753 1778 #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
1754 1779 if len(InterferenceRange)<int(cspc.shape[1]*0.3):
1755 1780 cspc[i,InterferenceRange,:] = numpy.NaN
1756 1781
1757 1782 self.dataOut.data_cspc = cspc
1758 1783
1759 1784 def removeInterference(self, interf = 2, hei_interf = None, nhei_interf = None, offhei_interf = None):
1760 1785
1761 1786 jspectra = self.dataOut.data_spc
1762 1787 jcspectra = self.dataOut.data_cspc
1763 1788 jnoise = self.dataOut.getNoise()
1764 1789 #num_incoh = self.dataOut.nIncohInt
1765 1790 num_incoh = self.dataOut.max_nIncohInt
1766 1791 #print("spc: ", jspectra.shape, jcspectra)
1767 1792 num_channel = jspectra.shape[0]
1768 1793 num_prof = jspectra.shape[1]
1769 1794 num_hei = jspectra.shape[2]
1770 1795
1771 1796 # hei_interf
1772 1797 if hei_interf is None:
1773 1798 count_hei = int(num_hei / 2) # a half of total ranges
1774 1799 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
1775 1800 hei_interf = numpy.asarray(hei_interf)[0]
1776 1801 #print(hei_interf)
1777 1802 # nhei_interf
1778 1803 if (nhei_interf == None):
1779 1804 nhei_interf = 5
1780 1805 if (nhei_interf < 1):
1781 1806 nhei_interf = 1
1782 1807 if (nhei_interf > count_hei):
1783 1808 nhei_interf = count_hei
1784 1809 if (offhei_interf == None):
1785 1810 offhei_interf = 0
1786 1811
1787 1812 ind_hei = list(range(num_hei))
1788 1813 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
1789 1814 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
1790 1815 mask_prof = numpy.asarray(list(range(num_prof)))
1791 1816 num_mask_prof = mask_prof.size
1792 1817 comp_mask_prof = [0, num_prof / 2]
1793 1818
1794 1819 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
1795 1820 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
1796 1821 jnoise = numpy.nan
1797 1822 noise_exist = jnoise[0] < numpy.Inf
1798 1823
1799 1824 # Subrutina de Remocion de la Interferencia
1800 1825 for ich in range(num_channel):
1801 1826 # Se ordena los espectros segun su potencia (menor a mayor)
1802 1827 power = jspectra[ich, mask_prof, :]
1803 1828 power = power[:, hei_interf]
1804 1829 power = power.sum(axis=0)
1805 1830 psort = power.ravel().argsort()
1806 1831 print(hei_interf[psort[list(range(offhei_interf, nhei_interf + offhei_interf))]])
1807 1832 # Se estima la interferencia promedio en los Espectros de Potencia empleando
1808 1833 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
1809 1834 offhei_interf, nhei_interf + offhei_interf))]]]
1810 1835
1811 1836 if noise_exist:
1812 1837 # tmp_noise = jnoise[ich] / num_prof
1813 1838 tmp_noise = jnoise[ich]
1814 1839 junkspc_interf = junkspc_interf - tmp_noise
1815 1840 #junkspc_interf[:,comp_mask_prof] = 0
1816 1841 print(junkspc_interf.shape)
1817 1842 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
1818 1843 jspc_interf = jspc_interf.transpose()
1819 1844 # Calculando el espectro de interferencia promedio
1820 1845 noiseid = numpy.where(jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
1821 1846 noiseid = noiseid[0]
1822 1847 cnoiseid = noiseid.size
1823 1848 interfid = numpy.where(jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
1824 1849 interfid = interfid[0]
1825 1850 cinterfid = interfid.size
1826 1851
1827 1852 if (cnoiseid > 0):
1828 1853 jspc_interf[noiseid] = 0
1829 1854 # Expandiendo los perfiles a limpiar
1830 1855 if (cinterfid > 0):
1831 1856 new_interfid = (
1832 1857 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
1833 1858 new_interfid = numpy.asarray(new_interfid)
1834 1859 new_interfid = {x for x in new_interfid}
1835 1860 new_interfid = numpy.array(list(new_interfid))
1836 1861 new_cinterfid = new_interfid.size
1837 1862 else:
1838 1863 new_cinterfid = 0
1839 1864
1840 1865 for ip in range(new_cinterfid):
1841 1866 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
1842 1867 jspc_interf[new_interfid[ip]] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
1843 1868
1844 1869 jspectra[ich, :, ind_hei] = jspectra[ich, :,ind_hei] - jspc_interf # Corregir indices
1845 1870
1846 1871 # Removiendo la interferencia del punto de mayor interferencia
1847 1872 ListAux = jspc_interf[mask_prof].tolist()
1848 1873 maxid = ListAux.index(max(ListAux))
1849 1874 print(cinterfid)
1850 1875 if cinterfid > 0:
1851 1876 for ip in range(cinterfid * (interf == 2) - 1):
1852 1877 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
1853 1878 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
1854 1879 cind = len(ind)
1855 1880
1856 1881 if (cind > 0):
1857 1882 jspectra[ich, interfid[ip], ind] = tmp_noise * \
1858 1883 (1 + (numpy.random.uniform(cind) - 0.5) /
1859 1884 numpy.sqrt(num_incoh))
1860 1885
1861 1886 ind = numpy.array([-2, -1, 1, 2])
1862 1887 xx = numpy.zeros([4, 4])
1863 1888
1864 1889 for id1 in range(4):
1865 1890 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1866 1891 xx_inv = numpy.linalg.inv(xx)
1867 1892 xx = xx_inv[:, 0]
1868 1893 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1869 1894 yy = jspectra[ich, mask_prof[ind], :]
1870 1895 jspectra[ich, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1871 1896
1872 1897 indAux = (jspectra[ich, :, :] < tmp_noise *
1873 1898 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
1874 1899 print(indAux)
1875 1900 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
1876 1901 (1 - 1 / numpy.sqrt(num_incoh))
1877 1902
1878 1903 # Remocion de Interferencia en el Cross Spectra
1879 1904 if jcspectra is None:
1880 1905 return jspectra, jcspectra
1881 1906 num_pairs = int(jcspectra.size / (num_prof * num_hei))
1882 1907 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
1883 1908
1884 1909 for ip in range(num_pairs):
1885 1910
1886 1911 #-------------------------------------------
1887 1912
1888 1913 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
1889 1914 cspower = cspower[:, hei_interf]
1890 1915 cspower = cspower.sum(axis=0)
1891 1916
1892 1917 cspsort = cspower.ravel().argsort()
1893 1918 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
1894 1919 offhei_interf, nhei_interf + offhei_interf))]]]
1895 1920 junkcspc_interf = junkcspc_interf.transpose()
1896 1921 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
1897 1922
1898 1923 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
1899 1924
1900 1925 median_real = int(numpy.median(numpy.real(
1901 1926 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1902 1927 median_imag = int(numpy.median(numpy.imag(
1903 1928 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1904 1929 comp_mask_prof = [int(e) for e in comp_mask_prof]
1905 1930 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
1906 1931 median_real, median_imag)
1907 1932
1908 1933 for iprof in range(num_prof):
1909 1934 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
1910 1935 jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
1911 1936
1912 1937 # Removiendo la Interferencia
1913 1938 jcspectra[ip, :, ind_hei] = jcspectra[ip,
1914 1939 :, ind_hei] - jcspc_interf
1915 1940
1916 1941 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
1917 1942 maxid = ListAux.index(max(ListAux))
1918 1943
1919 1944 ind = numpy.array([-2, -1, 1, 2])
1920 1945 xx = numpy.zeros([4, 4])
1921 1946
1922 1947 for id1 in range(4):
1923 1948 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1924 1949
1925 1950 xx_inv = numpy.linalg.inv(xx)
1926 1951 xx = xx_inv[:, 0]
1927 1952
1928 1953 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1929 1954 yy = jcspectra[ip, mask_prof[ind], :]
1930 1955 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1931 1956
1932 1957 # Guardar Resultados
1933 1958 self.dataOut.data_spc = jspectra
1934 1959 self.dataOut.data_cspc = jcspectra
1935 1960
1936 1961 return 1
1937 1962
1938 1963 def run(self, dataOut, interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None, mode=1, minHei=None, maxHei=None):
1939 1964
1940 1965 self.dataOut = dataOut
1941 1966
1942 1967 if mode == 1:
1943 1968 self.removeInterference(interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None)
1944 1969 elif mode == 2:
1945 1970 self.removeInterference2()
1946 1971 elif mode == 3:
1947 1972 self.removeInterference3(min_hei=minHei, max_hei=maxHei)
1948 1973 return self.dataOut
1949 1974
1950 1975
1951 1976 class IncohInt(Operation):
1952 1977
1953 1978 __profIndex = 0
1954 1979 __withOverapping = False
1955 1980
1956 1981 __byTime = False
1957 1982 __initime = None
1958 1983 __lastdatatime = None
1959 1984 __integrationtime = None
1960 1985
1961 1986 __buffer_spc = None
1962 1987 __buffer_cspc = None
1963 1988 __buffer_dc = None
1964 1989
1965 1990 __dataReady = False
1966 1991
1967 1992 __timeInterval = None
1968 1993 incohInt = 0
1969 1994 nOutliers = 0
1970 1995 n = None
1971 1996
1972 1997 def __init__(self):
1973 1998
1974 1999 Operation.__init__(self)
1975 2000
1976 2001 def setup(self, n=None, timeInterval=None, overlapping=False):
1977 2002 """
1978 2003 Set the parameters of the integration class.
1979 2004
1980 2005 Inputs:
1981 2006
1982 2007 n : Number of coherent integrations
1983 2008 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1984 2009 overlapping :
1985 2010
1986 2011 """
1987 2012
1988 2013 self.__initime = None
1989 2014 self.__lastdatatime = 0
1990 2015
1991 2016 self.__buffer_spc = 0
1992 2017 self.__buffer_cspc = 0
1993 2018 self.__buffer_dc = 0
1994 2019
1995 2020 self.__profIndex = 0
1996 2021 self.__dataReady = False
1997 2022 self.__byTime = False
1998 2023 self.incohInt = 0
1999 2024 self.nOutliers = 0
2000 2025 if n is None and timeInterval is None:
2001 2026 raise ValueError("n or timeInterval should be specified ...")
2002 2027
2003 2028 if n is not None:
2004 2029 self.n = int(n)
2005 2030 else:
2006 2031
2007 2032 self.__integrationtime = int(timeInterval)
2008 2033 self.n = None
2009 2034 self.__byTime = True
2010 2035
2036
2011 2037 def putData(self, data_spc, data_cspc, data_dc):
2012 2038 """
2013 2039 Add a profile to the __buffer_spc and increase in one the __profileIndex
2014 2040
2015 2041 """
2016 2042 if data_spc.all() == numpy.nan :
2017 2043 print("nan ")
2018 2044 return
2019 2045 self.__buffer_spc += data_spc
2020 2046
2021 2047 if data_cspc is None:
2022 2048 self.__buffer_cspc = None
2023 2049 else:
2024 2050 self.__buffer_cspc += data_cspc
2025 2051
2026 2052 if data_dc is None:
2027 2053 self.__buffer_dc = None
2028 2054 else:
2029 2055 self.__buffer_dc += data_dc
2030 2056
2031 2057 self.__profIndex += 1
2032 2058
2033 2059 return
2034 2060
2035 2061 def pushData(self):
2036 2062 """
2037 2063 Return the sum of the last profiles and the profiles used in the sum.
2038 2064
2039 2065 Affected:
2040 2066
2041 2067 self.__profileIndex
2042 2068
2043 2069 """
2044 2070
2045 2071 data_spc = self.__buffer_spc
2046 2072 data_cspc = self.__buffer_cspc
2047 2073 data_dc = self.__buffer_dc
2048 2074 n = self.__profIndex
2049 2075
2050 2076 self.__buffer_spc = 0
2051 2077 self.__buffer_cspc = 0
2052 2078 self.__buffer_dc = 0
2053 2079
2054 2080
2055 2081 return data_spc, data_cspc, data_dc, n
2056 2082
2057 2083 def byProfiles(self, *args):
2058 2084
2059 2085 self.__dataReady = False
2060 2086 avgdata_spc = None
2061 2087 avgdata_cspc = None
2062 2088 avgdata_dc = None
2063 2089
2064 2090 self.putData(*args)
2065 2091
2066 2092 if self.__profIndex == self.n:
2067 2093
2068 2094 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
2069 2095 self.n = n
2070 2096 self.__dataReady = True
2071 2097
2072 2098 return avgdata_spc, avgdata_cspc, avgdata_dc
2073 2099
2074 2100 def byTime(self, datatime, *args):
2075 2101
2076 2102 self.__dataReady = False
2077 2103 avgdata_spc = None
2078 2104 avgdata_cspc = None
2079 2105 avgdata_dc = None
2080 2106
2081 2107 self.putData(*args)
2082 2108
2083 2109 if (datatime - self.__initime) >= self.__integrationtime:
2084 2110 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
2085 2111 self.n = n
2086 2112 self.__dataReady = True
2087 2113
2088 2114 return avgdata_spc, avgdata_cspc, avgdata_dc
2089 2115
2090 2116 def integrate(self, datatime, *args):
2091 2117
2092 2118 if self.__profIndex == 0:
2093 2119 self.__initime = datatime
2094 2120
2095 2121 if self.__byTime:
2096 2122 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
2097 2123 datatime, *args)
2098 2124 else:
2099 2125 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
2100 2126
2101 2127 if not self.__dataReady:
2102 2128 return None, None, None, None
2103 2129
2104 2130 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
2105 2131
2106 2132 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
2107 2133 if n == 1:
2108 2134 return dataOut
2109 2135
2110 2136 if dataOut.flagNoData == True:
2111 2137 return dataOut
2112 2138
2113 2139 dataOut.flagNoData = True
2114
2140 dataOut.processingHeaderObj.timeIncohInt = timeInterval
2115 2141 if not self.isConfig:
2116 2142 self.setup(n, timeInterval, overlapping)
2117 2143 self.isConfig = True
2118 2144
2145
2119 2146 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
2120 2147 dataOut.data_spc,
2121 2148 dataOut.data_cspc,
2122 2149 dataOut.data_dc)
2123 2150
2124 2151 self.incohInt += dataOut.nIncohInt
2125 2152
2126 2153 if isinstance(dataOut.data_outlier,numpy.ndarray) or isinstance(dataOut.data_outlier,int) or isinstance(dataOut.data_outlier, float):
2127 2154 self.nOutliers += dataOut.data_outlier
2128 2155
2129 2156 if self.__dataReady:
2130 2157 #print("prof: ",dataOut.max_nIncohInt,self.__profIndex)
2131 2158 dataOut.data_spc = avgdata_spc
2132 2159 dataOut.data_cspc = avgdata_cspc
2133 2160 dataOut.data_dc = avgdata_dc
2134 2161 dataOut.nIncohInt = self.incohInt
2135 2162 dataOut.data_outlier = self.nOutliers
2136 2163 dataOut.utctime = avgdatatime
2137 2164 dataOut.flagNoData = False
2138 2165 self.incohInt = 0
2139 2166 self.nOutliers = 0
2140 2167 self.__profIndex = 0
2141 2168 #print("IncohInt Done")
2142 2169 return dataOut
2143 2170
2144 2171 class dopplerFlip(Operation):
2145 2172
2146 2173 def run(self, dataOut):
2147 2174 # arreglo 1: (num_chan, num_profiles, num_heights)
2148 2175 self.dataOut = dataOut
2149 2176 # JULIA-oblicua, indice 2
2150 2177 # arreglo 2: (num_profiles, num_heights)
2151 2178 jspectra = self.dataOut.data_spc[2]
2152 2179 jspectra_tmp = numpy.zeros(jspectra.shape)
2153 2180 num_profiles = jspectra.shape[0]
2154 2181 freq_dc = int(num_profiles / 2)
2155 2182 # Flip con for
2156 2183 for j in range(num_profiles):
2157 2184 jspectra_tmp[num_profiles-j-1]= jspectra[j]
2158 2185 # Intercambio perfil de DC con perfil inmediato anterior
2159 2186 jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1]
2160 2187 jspectra_tmp[freq_dc]= jspectra[freq_dc]
2161 2188 # canal modificado es re-escrito en el arreglo de canales
2162 2189 self.dataOut.data_spc[2] = jspectra_tmp
2163 2190
2164 2191 return self.dataOut
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now