##// END OF EJS Templates
schain
RSS
ParameterReader unit and ParameterWriter operation added
Juan C. Espinoza -
r1232:0f436481ae30
parent child
Show More
Side by Side Unified
Context file:
r1232:0f436481ae30 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1372 +1,1372
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
5 5 '''
6 6
7 7 import copy
8 8 import numpy
9 9 import datetime
10 10 import json
11 11
12 12 from schainpy.utils import log
13 13 from .jroheaderIO import SystemHeader, RadarControllerHeader
14 14
15 15
16 16 def getNumpyDtype(dataTypeCode):
17 17
18 18 if dataTypeCode == 0:
19 19 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
20 20 elif dataTypeCode == 1:
21 21 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
22 22 elif dataTypeCode == 2:
23 23 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
24 24 elif dataTypeCode == 3:
25 25 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
26 26 elif dataTypeCode == 4:
27 27 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
28 28 elif dataTypeCode == 5:
29 29 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
30 30 else:
31 31 raise ValueError('dataTypeCode was not defined')
32 32
33 33 return numpyDtype
34 34
35 35
36 36 def getDataTypeCode(numpyDtype):
37 37
38 38 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
39 39 datatype = 0
40 40 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
41 41 datatype = 1
42 42 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
43 43 datatype = 2
44 44 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
45 45 datatype = 3
46 46 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
47 47 datatype = 4
48 48 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
49 49 datatype = 5
50 50 else:
51 51 datatype = None
52 52
53 53 return datatype
54 54
55 55
56 56 def hildebrand_sekhon(data, navg):
57 57 """
58 58 This method is for the objective determination of the noise level in Doppler spectra. This
59 59 implementation technique is based on the fact that the standard deviation of the spectral
60 60 densities is equal to the mean spectral density for white Gaussian noise
61 61
62 62 Inputs:
63 63 Data : heights
64 64 navg : numbers of averages
65 65
66 66 Return:
67 67 mean : noise's level
68 68 """
69 69
70 70 sortdata = numpy.sort(data, axis=None)
71 71 lenOfData = len(sortdata)
72 72 nums_min = lenOfData*0.2
73 73
74 74 if nums_min <= 5:
75 75
76 76 nums_min = 5
77 77
78 78 sump = 0.
79 79 sumq = 0.
80 80
81 81 j = 0
82 82 cont = 1
83 83
84 84 while((cont == 1)and(j < lenOfData)):
85 85
86 86 sump += sortdata[j]
87 87 sumq += sortdata[j]**2
88 88
89 89 if j > nums_min:
90 90 rtest = float(j)/(j-1) + 1.0/navg
91 91 if ((sumq*j) > (rtest*sump**2)):
92 92 j = j - 1
93 93 sump = sump - sortdata[j]
94 94 sumq = sumq - sortdata[j]**2
95 95 cont = 0
96 96
97 97 j += 1
98 98
99 99 lnoise = sump / j
100 100
101 101 return lnoise
102 102
103 103
104 104 class Beam:
105 105
106 106 def __init__(self):
107 107 self.codeList = []
108 108 self.azimuthList = []
109 109 self.zenithList = []
110 110
111 111
112 112 class GenericData(object):
113 113
114 114 flagNoData = True
115 115
116 116 def copy(self, inputObj=None):
117 117
118 118 if inputObj == None:
119 119 return copy.deepcopy(self)
120 120
121 121 for key in list(inputObj.__dict__.keys()):
122 122
123 123 attribute = inputObj.__dict__[key]
124 124
125 125 # If this attribute is a tuple or list
126 126 if type(inputObj.__dict__[key]) in (tuple, list):
127 127 self.__dict__[key] = attribute[:]
128 128 continue
129 129
130 130 # If this attribute is another object or instance
131 131 if hasattr(attribute, '__dict__'):
132 132 self.__dict__[key] = attribute.copy()
133 133 continue
134 134
135 135 self.__dict__[key] = inputObj.__dict__[key]
136 136
137 137 def deepcopy(self):
138 138
139 139 return copy.deepcopy(self)
140 140
141 141 def isEmpty(self):
142 142
143 143 return self.flagNoData
144 144
145 145
146 146 class JROData(GenericData):
147 147
148 148 # m_BasicHeader = BasicHeader()
149 149 # m_ProcessingHeader = ProcessingHeader()
150 150
151 151 systemHeaderObj = SystemHeader()
152 152 radarControllerHeaderObj = RadarControllerHeader()
153 153 # data = None
154 154 type = None
155 155 datatype = None # dtype but in string
156 156 # dtype = None
157 157 # nChannels = None
158 158 # nHeights = None
159 159 nProfiles = None
160 160 heightList = None
161 161 channelList = None
162 162 flagDiscontinuousBlock = False
163 163 useLocalTime = False
164 164 utctime = None
165 165 timeZone = None
166 166 dstFlag = None
167 167 errorCount = None
168 168 blocksize = None
169 169 # nCode = None
170 170 # nBaud = None
171 171 # code = None
172 172 flagDecodeData = False # asumo q la data no esta decodificada
173 173 flagDeflipData = False # asumo q la data no esta sin flip
174 174 flagShiftFFT = False
175 175 # ippSeconds = None
176 176 # timeInterval = None
177 177 nCohInt = None
178 178 # noise = None
179 179 windowOfFilter = 1
180 180 # Speed of ligth
181 181 C = 3e8
182 182 frequency = 49.92e6
183 183 realtime = False
184 184 beacon_heiIndexList = None
185 185 last_block = None
186 186 blocknow = None
187 187 azimuth = None
188 188 zenith = None
189 189 beam = Beam()
190 190 profileIndex = None
191 191 error = None
192 192 data = None
193 193 nmodes = None
194 194
195 195 def __str__(self):
196 196
197 197 return '{} - {}'.format(self.type, self.getDatatime())
198 198
199 199 def getNoise(self):
200 200
201 201 raise NotImplementedError
202 202
203 203 def getNChannels(self):
204 204
205 205 return len(self.channelList)
206 206
207 207 def getChannelIndexList(self):
208 208
209 209 return list(range(self.nChannels))
210 210
211 211 def getNHeights(self):
212 212
213 213 return len(self.heightList)
214 214
215 215 def getHeiRange(self, extrapoints=0):
216 216
217 217 heis = self.heightList
218 218 # deltah = self.heightList[1] - self.heightList[0]
219 219 #
220 220 # heis.append(self.heightList[-1])
221 221
222 222 return heis
223 223
224 224 def getDeltaH(self):
225 225
226 226 delta = self.heightList[1] - self.heightList[0]
227 227
228 228 return delta
229 229
230 230 def getltctime(self):
231 231
232 232 if self.useLocalTime:
233 233 return self.utctime - self.timeZone * 60
234 234
235 235 return self.utctime
236 236
237 237 def getDatatime(self):
238 238
239 239 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
240 240 return datatimeValue
241 241
242 242 def getTimeRange(self):
243 243
244 244 datatime = []
245 245
246 246 datatime.append(self.ltctime)
247 247 datatime.append(self.ltctime + self.timeInterval + 1)
248 248
249 249 datatime = numpy.array(datatime)
250 250
251 251 return datatime
252 252
253 253 def getFmaxTimeResponse(self):
254 254
255 255 period = (10**-6) * self.getDeltaH() / (0.15)
256 256
257 257 PRF = 1. / (period * self.nCohInt)
258 258
259 259 fmax = PRF
260 260
261 261 return fmax
262 262
263 263 def getFmax(self):
264 264 PRF = 1. / (self.ippSeconds * self.nCohInt)
265 265
266 266 fmax = PRF
267 267 return fmax
268 268
269 269 def getVmax(self):
270 270
271 271 _lambda = self.C / self.frequency
272 272
273 273 vmax = self.getFmax() * _lambda / 2
274 274
275 275 return vmax
276 276
277 277 def get_ippSeconds(self):
278 278 '''
279 279 '''
280 280 return self.radarControllerHeaderObj.ippSeconds
281 281
282 282 def set_ippSeconds(self, ippSeconds):
283 283 '''
284 284 '''
285 285
286 286 self.radarControllerHeaderObj.ippSeconds = ippSeconds
287 287
288 288 return
289 289
290 290 def get_dtype(self):
291 291 '''
292 292 '''
293 293 return getNumpyDtype(self.datatype)
294 294
295 295 def set_dtype(self, numpyDtype):
296 296 '''
297 297 '''
298 298
299 299 self.datatype = getDataTypeCode(numpyDtype)
300 300
301 301 def get_code(self):
302 302 '''
303 303 '''
304 304 return self.radarControllerHeaderObj.code
305 305
306 306 def set_code(self, code):
307 307 '''
308 308 '''
309 309 self.radarControllerHeaderObj.code = code
310 310
311 311 return
312 312
313 313 def get_ncode(self):
314 314 '''
315 315 '''
316 316 return self.radarControllerHeaderObj.nCode
317 317
318 318 def set_ncode(self, nCode):
319 319 '''
320 320 '''
321 321 self.radarControllerHeaderObj.nCode = nCode
322 322
323 323 return
324 324
325 325 def get_nbaud(self):
326 326 '''
327 327 '''
328 328 return self.radarControllerHeaderObj.nBaud
329 329
330 330 def set_nbaud(self, nBaud):
331 331 '''
332 332 '''
333 333 self.radarControllerHeaderObj.nBaud = nBaud
334 334
335 335 return
336 336
337 337 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
338 338 channelIndexList = property(
339 339 getChannelIndexList, "I'm the 'channelIndexList' property.")
340 340 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
341 341 #noise = property(getNoise, "I'm the 'nHeights' property.")
342 342 datatime = property(getDatatime, "I'm the 'datatime' property")
343 343 ltctime = property(getltctime, "I'm the 'ltctime' property")
344 344 ippSeconds = property(get_ippSeconds, set_ippSeconds)
345 345 dtype = property(get_dtype, set_dtype)
346 346 # timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
347 347 code = property(get_code, set_code)
348 348 nCode = property(get_ncode, set_ncode)
349 349 nBaud = property(get_nbaud, set_nbaud)
350 350
351 351
352 352 class Voltage(JROData):
353 353
354 354 # data es un numpy array de 2 dmensiones (canales, alturas)
355 355 data = None
356 356
357 357 def __init__(self):
358 358 '''
359 359 Constructor
360 360 '''
361 361
362 362 self.useLocalTime = True
363 363 self.radarControllerHeaderObj = RadarControllerHeader()
364 364 self.systemHeaderObj = SystemHeader()
365 365 self.type = "Voltage"
366 366 self.data = None
367 367 # self.dtype = None
368 368 # self.nChannels = 0
369 369 # self.nHeights = 0
370 370 self.nProfiles = None
371 371 self.heightList = None
372 372 self.channelList = None
373 373 # self.channelIndexList = None
374 374 self.flagNoData = True
375 375 self.flagDiscontinuousBlock = False
376 376 self.utctime = None
377 377 self.timeZone = None
378 378 self.dstFlag = None
379 379 self.errorCount = None
380 380 self.nCohInt = None
381 381 self.blocksize = None
382 382 self.flagDecodeData = False # asumo q la data no esta decodificada
383 383 self.flagDeflipData = False # asumo q la data no esta sin flip
384 384 self.flagShiftFFT = False
385 385 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
386 386 self.profileIndex = 0
387 387
388 388 def getNoisebyHildebrand(self, channel=None):
389 389 """
390 390 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
391 391
392 392 Return:
393 393 noiselevel
394 394 """
395 395
396 396 if channel != None:
397 397 data = self.data[channel]
398 398 nChannels = 1
399 399 else:
400 400 data = self.data
401 401 nChannels = self.nChannels
402 402
403 403 noise = numpy.zeros(nChannels)
404 404 power = data * numpy.conjugate(data)
405 405
406 406 for thisChannel in range(nChannels):
407 407 if nChannels == 1:
408 408 daux = power[:].real
409 409 else:
410 410 daux = power[thisChannel, :].real
411 411 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
412 412
413 413 return noise
414 414
415 415 def getNoise(self, type=1, channel=None):
416 416
417 417 if type == 1:
418 418 noise = self.getNoisebyHildebrand(channel)
419 419
420 420 return noise
421 421
422 422 def getPower(self, channel=None):
423 423
424 424 if channel != None:
425 425 data = self.data[channel]
426 426 else:
427 427 data = self.data
428 428
429 429 power = data * numpy.conjugate(data)
430 430 powerdB = 10 * numpy.log10(power.real)
431 431 powerdB = numpy.squeeze(powerdB)
432 432
433 433 return powerdB
434 434
435 435 def getTimeInterval(self):
436 436
437 437 timeInterval = self.ippSeconds * self.nCohInt
438 438
439 439 return timeInterval
440 440
441 441 noise = property(getNoise, "I'm the 'nHeights' property.")
442 442 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
443 443
444 444
445 445 class Spectra(JROData):
446 446
447 447 # data spc es un numpy array de 2 dmensiones (canales, perfiles, alturas)
448 448 data_spc = None
449 449 # data cspc es un numpy array de 2 dmensiones (canales, pares, alturas)
450 450 data_cspc = None
451 451 # data dc es un numpy array de 2 dmensiones (canales, alturas)
452 452 data_dc = None
453 453 # data power
454 454 data_pwr = None
455 455 nFFTPoints = None
456 456 # nPairs = None
457 457 pairsList = None
458 458 nIncohInt = None
459 459 wavelength = None # Necesario para cacular el rango de velocidad desde la frecuencia
460 460 nCohInt = None # se requiere para determinar el valor de timeInterval
461 461 ippFactor = None
462 462 profileIndex = 0
463 463 plotting = "spectra"
464 464
465 465 def __init__(self):
466 466 '''
467 467 Constructor
468 468 '''
469 469
470 470 self.useLocalTime = True
471 471 self.radarControllerHeaderObj = RadarControllerHeader()
472 472 self.systemHeaderObj = SystemHeader()
473 473 self.type = "Spectra"
474 474 # self.data = None
475 475 # self.dtype = None
476 476 # self.nChannels = 0
477 477 # self.nHeights = 0
478 478 self.nProfiles = None
479 479 self.heightList = None
480 480 self.channelList = None
481 481 # self.channelIndexList = None
482 482 self.pairsList = None
483 483 self.flagNoData = True
484 484 self.flagDiscontinuousBlock = False
485 485 self.utctime = None
486 486 self.nCohInt = None
487 487 self.nIncohInt = None
488 488 self.blocksize = None
489 489 self.nFFTPoints = None
490 490 self.wavelength = None
491 491 self.flagDecodeData = False # asumo q la data no esta decodificada
492 492 self.flagDeflipData = False # asumo q la data no esta sin flip
493 493 self.flagShiftFFT = False
494 494 self.ippFactor = 1
495 495 #self.noise = None
496 496 self.beacon_heiIndexList = []
497 497 self.noise_estimation = None
498 498
499 499 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
500 500 """
501 501 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
502 502
503 503 Return:
504 504 noiselevel
505 505 """
506 506
507 507 noise = numpy.zeros(self.nChannels)
508 508
509 509 for channel in range(self.nChannels):
510 510 daux = self.data_spc[channel,
511 511 xmin_index:xmax_index, ymin_index:ymax_index]
512 512 noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
513 513
514 514 return noise
515 515
516 516 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
517 517
518 518 if self.noise_estimation is not None:
519 519 # this was estimated by getNoise Operation defined in jroproc_spectra.py
520 520 return self.noise_estimation
521 521 else:
522 522 noise = self.getNoisebyHildebrand(
523 523 xmin_index, xmax_index, ymin_index, ymax_index)
524 524 return noise
525 525
526 526 def getFreqRangeTimeResponse(self, extrapoints=0):
527 527
528 528 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
529 529 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
530 530
531 531 return freqrange
532 532
533 533 def getAcfRange(self, extrapoints=0):
534 534
535 535 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
536 536 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
537 537
538 538 return freqrange
539 539
540 540 def getFreqRange(self, extrapoints=0):
541 541
542 542 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
543 543 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
544 544
545 545 return freqrange
546 546
547 547 def getVelRange(self, extrapoints=0):
548 548
549 549 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
550 550 velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
551 551
552 552 if self.nmodes:
553 553 return velrange/self.nmodes
554 554 else:
555 555 return velrange
556 556
557 557 def getNPairs(self):
558 558
559 559 return len(self.pairsList)
560 560
561 561 def getPairsIndexList(self):
562 562
563 563 return list(range(self.nPairs))
564 564
565 565 def getNormFactor(self):
566 566
567 567 pwcode = 1
568 568
569 569 if self.flagDecodeData:
570 570 pwcode = numpy.sum(self.code[0]**2)
571 571 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
572 572 normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
573 573
574 574 return normFactor
575 575
576 576 def getFlagCspc(self):
577 577
578 578 if self.data_cspc is None:
579 579 return True
580 580
581 581 return False
582 582
583 583 def getFlagDc(self):
584 584
585 585 if self.data_dc is None:
586 586 return True
587 587
588 588 return False
589 589
590 590 def getTimeInterval(self):
591 591
592 592 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
593 593 if self.nmodes:
594 594 return self.nmodes*timeInterval
595 595 else:
596 596 return timeInterval
597 597
598 598 def getPower(self):
599 599
600 600 factor = self.normFactor
601 601 z = self.data_spc / factor
602 602 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
603 603 avg = numpy.average(z, axis=1)
604 604
605 605 return 10 * numpy.log10(avg)
606 606
607 607 def getCoherence(self, pairsList=None, phase=False):
608 608
609 609 z = []
610 610 if pairsList is None:
611 611 pairsIndexList = self.pairsIndexList
612 612 else:
613 613 pairsIndexList = []
614 614 for pair in pairsList:
615 615 if pair not in self.pairsList:
616 616 raise ValueError("Pair %s is not in dataOut.pairsList" % (
617 617 pair))
618 618 pairsIndexList.append(self.pairsList.index(pair))
619 619 for i in range(len(pairsIndexList)):
620 620 pair = self.pairsList[pairsIndexList[i]]
621 621 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
622 622 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
623 623 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
624 624 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
625 625 if phase:
626 626 data = numpy.arctan2(avgcoherenceComplex.imag,
627 627 avgcoherenceComplex.real) * 180 / numpy.pi
628 628 else:
629 629 data = numpy.abs(avgcoherenceComplex)
630 630
631 631 z.append(data)
632 632
633 633 return numpy.array(z)
634 634
635 635 def setValue(self, value):
636 636
637 637 print("This property should not be initialized")
638 638
639 639 return
640 640
641 641 nPairs = property(getNPairs, setValue, "I'm the 'nPairs' property.")
642 642 pairsIndexList = property(
643 643 getPairsIndexList, setValue, "I'm the 'pairsIndexList' property.")
644 644 normFactor = property(getNormFactor, setValue,
645 645 "I'm the 'getNormFactor' property.")
646 646 flag_cspc = property(getFlagCspc, setValue)
647 647 flag_dc = property(getFlagDc, setValue)
648 648 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
649 649 timeInterval = property(getTimeInterval, setValue,
650 650 "I'm the 'timeInterval' property")
651 651
652 652
653 653 class SpectraHeis(Spectra):
654 654
655 655 data_spc = None
656 656 data_cspc = None
657 657 data_dc = None
658 658 nFFTPoints = None
659 659 # nPairs = None
660 660 pairsList = None
661 661 nCohInt = None
662 662 nIncohInt = None
663 663
664 664 def __init__(self):
665 665
666 666 self.radarControllerHeaderObj = RadarControllerHeader()
667 667
668 668 self.systemHeaderObj = SystemHeader()
669 669
670 670 self.type = "SpectraHeis"
671 671
672 672 # self.dtype = None
673 673
674 674 # self.nChannels = 0
675 675
676 676 # self.nHeights = 0
677 677
678 678 self.nProfiles = None
679 679
680 680 self.heightList = None
681 681
682 682 self.channelList = None
683 683
684 684 # self.channelIndexList = None
685 685
686 686 self.flagNoData = True
687 687
688 688 self.flagDiscontinuousBlock = False
689 689
690 690 # self.nPairs = 0
691 691
692 692 self.utctime = None
693 693
694 694 self.blocksize = None
695 695
696 696 self.profileIndex = 0
697 697
698 698 self.nCohInt = 1
699 699
700 700 self.nIncohInt = 1
701 701
702 702 def getNormFactor(self):
703 703 pwcode = 1
704 704 if self.flagDecodeData:
705 705 pwcode = numpy.sum(self.code[0]**2)
706 706
707 707 normFactor = self.nIncohInt * self.nCohInt * pwcode
708 708
709 709 return normFactor
710 710
711 711 def getTimeInterval(self):
712 712
713 713 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
714 714
715 715 return timeInterval
716 716
717 717 normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
718 718 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
719 719
720 720
721 721 class Fits(JROData):
722 722
723 723 heightList = None
724 724 channelList = None
725 725 flagNoData = True
726 726 flagDiscontinuousBlock = False
727 727 useLocalTime = False
728 728 utctime = None
729 729 timeZone = None
730 730 # ippSeconds = None
731 731 # timeInterval = None
732 732 nCohInt = None
733 733 nIncohInt = None
734 734 noise = None
735 735 windowOfFilter = 1
736 736 # Speed of ligth
737 737 C = 3e8
738 738 frequency = 49.92e6
739 739 realtime = False
740 740
741 741 def __init__(self):
742 742
743 743 self.type = "Fits"
744 744
745 745 self.nProfiles = None
746 746
747 747 self.heightList = None
748 748
749 749 self.channelList = None
750 750
751 751 # self.channelIndexList = None
752 752
753 753 self.flagNoData = True
754 754
755 755 self.utctime = None
756 756
757 757 self.nCohInt = 1
758 758
759 759 self.nIncohInt = 1
760 760
761 761 self.useLocalTime = True
762 762
763 763 self.profileIndex = 0
764 764
765 765 # self.utctime = None
766 766 # self.timeZone = None
767 767 # self.ltctime = None
768 768 # self.timeInterval = None
769 769 # self.header = None
770 770 # self.data_header = None
771 771 # self.data = None
772 772 # self.datatime = None
773 773 # self.flagNoData = False
774 774 # self.expName = ''
775 775 # self.nChannels = None
776 776 # self.nSamples = None
777 777 # self.dataBlocksPerFile = None
778 778 # self.comments = ''
779 779 #
780 780
781 781 def getltctime(self):
782 782
783 783 if self.useLocalTime:
784 784 return self.utctime - self.timeZone * 60
785 785
786 786 return self.utctime
787 787
788 788 def getDatatime(self):
789 789
790 790 datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
791 791 return datatime
792 792
793 793 def getTimeRange(self):
794 794
795 795 datatime = []
796 796
797 797 datatime.append(self.ltctime)
798 798 datatime.append(self.ltctime + self.timeInterval)
799 799
800 800 datatime = numpy.array(datatime)
801 801
802 802 return datatime
803 803
804 804 def getHeiRange(self):
805 805
806 806 heis = self.heightList
807 807
808 808 return heis
809 809
810 810 def getNHeights(self):
811 811
812 812 return len(self.heightList)
813 813
814 814 def getNChannels(self):
815 815
816 816 return len(self.channelList)
817 817
818 818 def getChannelIndexList(self):
819 819
820 820 return list(range(self.nChannels))
821 821
822 822 def getNoise(self, type=1):
823 823
824 824 #noise = numpy.zeros(self.nChannels)
825 825
826 826 if type == 1:
827 827 noise = self.getNoisebyHildebrand()
828 828
829 829 if type == 2:
830 830 noise = self.getNoisebySort()
831 831
832 832 if type == 3:
833 833 noise = self.getNoisebyWindow()
834 834
835 835 return noise
836 836
837 837 def getTimeInterval(self):
838 838
839 839 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
840 840
841 841 return timeInterval
842 842
843 843 def get_ippSeconds(self):
844 844 '''
845 845 '''
846 846 return self.ipp_sec
847 847
848 848
849 849 datatime = property(getDatatime, "I'm the 'datatime' property")
850 850 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
851 851 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
852 852 channelIndexList = property(
853 853 getChannelIndexList, "I'm the 'channelIndexList' property.")
854 854 noise = property(getNoise, "I'm the 'nHeights' property.")
855 855
856 856 ltctime = property(getltctime, "I'm the 'ltctime' property")
857 857 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
858 858 ippSeconds = property(get_ippSeconds, '')
859 859
860 860 class Correlation(JROData):
861 861
862 862 noise = None
863 863 SNR = None
864 864 #--------------------------------------------------
865 865 mode = None
866 866 split = False
867 867 data_cf = None
868 868 lags = None
869 869 lagRange = None
870 870 pairsList = None
871 871 normFactor = None
872 872 #--------------------------------------------------
873 873 # calculateVelocity = None
874 874 nLags = None
875 875 nPairs = None
876 876 nAvg = None
877 877
878 878 def __init__(self):
879 879 '''
880 880 Constructor
881 881 '''
882 882 self.radarControllerHeaderObj = RadarControllerHeader()
883 883
884 884 self.systemHeaderObj = SystemHeader()
885 885
886 886 self.type = "Correlation"
887 887
888 888 self.data = None
889 889
890 890 self.dtype = None
891 891
892 892 self.nProfiles = None
893 893
894 894 self.heightList = None
895 895
896 896 self.channelList = None
897 897
898 898 self.flagNoData = True
899 899
900 900 self.flagDiscontinuousBlock = False
901 901
902 902 self.utctime = None
903 903
904 904 self.timeZone = None
905 905
906 906 self.dstFlag = None
907 907
908 908 self.errorCount = None
909 909
910 910 self.blocksize = None
911 911
912 912 self.flagDecodeData = False # asumo q la data no esta decodificada
913 913
914 914 self.flagDeflipData = False # asumo q la data no esta sin flip
915 915
916 916 self.pairsList = None
917 917
918 918 self.nPoints = None
919 919
920 920 def getPairsList(self):
921 921
922 922 return self.pairsList
923 923
924 924 def getNoise(self, mode=2):
925 925
926 926 indR = numpy.where(self.lagR == 0)[0][0]
927 927 indT = numpy.where(self.lagT == 0)[0][0]
928 928
929 929 jspectra0 = self.data_corr[:, :, indR, :]
930 930 jspectra = copy.copy(jspectra0)
931 931
932 932 num_chan = jspectra.shape[0]
933 933 num_hei = jspectra.shape[2]
934 934
935 935 freq_dc = jspectra.shape[1] / 2
936 936 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
937 937
938 938 if ind_vel[0] < 0:
939 939 ind_vel[list(range(0, 1))] = ind_vel[list(
940 940 range(0, 1))] + self.num_prof
941 941
942 942 if mode == 1:
943 943 jspectra[:, freq_dc, :] = (
944 944 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
945 945
946 946 if mode == 2:
947 947
948 948 vel = numpy.array([-2, -1, 1, 2])
949 949 xx = numpy.zeros([4, 4])
950 950
951 951 for fil in range(4):
952 952 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
953 953
954 954 xx_inv = numpy.linalg.inv(xx)
955 955 xx_aux = xx_inv[0, :]
956 956
957 957 for ich in range(num_chan):
958 958 yy = jspectra[ich, ind_vel, :]
959 959 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
960 960
961 961 junkid = jspectra[ich, freq_dc, :] <= 0
962 962 cjunkid = sum(junkid)
963 963
964 964 if cjunkid.any():
965 965 jspectra[ich, freq_dc, junkid.nonzero()] = (
966 966 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
967 967
968 968 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
969 969
970 970 return noise
971 971
972 972 def getTimeInterval(self):
973 973
974 974 timeInterval = self.ippSeconds * self.nCohInt * self.nProfiles
975 975
976 976 return timeInterval
977 977
978 978 def splitFunctions(self):
979 979
980 980 pairsList = self.pairsList
981 981 ccf_pairs = []
982 982 acf_pairs = []
983 983 ccf_ind = []
984 984 acf_ind = []
985 985 for l in range(len(pairsList)):
986 986 chan0 = pairsList[l][0]
987 987 chan1 = pairsList[l][1]
988 988
989 989 # Obteniendo pares de Autocorrelacion
990 990 if chan0 == chan1:
991 991 acf_pairs.append(chan0)
992 992 acf_ind.append(l)
993 993 else:
994 994 ccf_pairs.append(pairsList[l])
995 995 ccf_ind.append(l)
996 996
997 997 data_acf = self.data_cf[acf_ind]
998 998 data_ccf = self.data_cf[ccf_ind]
999 999
1000 1000 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
1001 1001
1002 1002 def getNormFactor(self):
1003 1003 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
1004 1004 acf_pairs = numpy.array(acf_pairs)
1005 1005 normFactor = numpy.zeros((self.nPairs, self.nHeights))
1006 1006
1007 1007 for p in range(self.nPairs):
1008 1008 pair = self.pairsList[p]
1009 1009
1010 1010 ch0 = pair[0]
1011 1011 ch1 = pair[1]
1012 1012
1013 1013 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
1014 1014 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
1015 1015 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
1016 1016
1017 1017 return normFactor
1018 1018
1019 1019 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
1020 1020 normFactor = property(getNormFactor, "I'm the 'normFactor property'")
1021 1021
1022 1022
1023 1023 class Parameters(Spectra):
1024 1024
1025 1025 experimentInfo = None # Information about the experiment
1026 1026 # Information from previous data
1027 1027 inputUnit = None # Type of data to be processed
1028 1028 operation = None # Type of operation to parametrize
1029 1029 # normFactor = None #Normalization Factor
1030 1030 groupList = None # List of Pairs, Groups, etc
1031 1031 # Parameters
1032 1032 data_param = None # Parameters obtained
1033 1033 data_pre = None # Data Pre Parametrization
1034 1034 data_SNR = None # Signal to Noise Ratio
1035 1035 # heightRange = None #Heights
1036 1036 abscissaList = None # Abscissa, can be velocities, lags or time
1037 1037 # noise = None #Noise Potency
1038 1038 utctimeInit = None # Initial UTC time
1039 1039 paramInterval = None # Time interval to calculate Parameters in seconds
1040 1040 useLocalTime = True
1041 1041 # Fitting
1042 1042 data_error = None # Error of the estimation
1043 1043 constants = None
1044 1044 library = None
1045 1045 # Output signal
1046 1046 outputInterval = None # Time interval to calculate output signal in seconds
1047 1047 data_output = None # Out signal
1048 1048 nAvg = None
1049 1049 noise_estimation = None
1050 1050 GauSPC = None # Fit gaussian SPC
1051 1051
1052 1052 def __init__(self):
1053 1053 '''
1054 1054 Constructor
1055 1055 '''
1056 1056 self.radarControllerHeaderObj = RadarControllerHeader()
1057 1057
1058 1058 self.systemHeaderObj = SystemHeader()
1059 1059
1060 1060 self.type = "Parameters"
1061 1061
1062 1062 def getTimeRange1(self, interval):
1063 1063
1064 1064 datatime = []
1065 1065
1066 1066 if self.useLocalTime:
1067 1067 time1 = self.utctimeInit - self.timeZone * 60
1068 1068 else:
1069 1069 time1 = self.utctimeInit
1070 1070
1071 1071 datatime.append(time1)
1072 1072 datatime.append(time1 + interval)
1073 1073 datatime = numpy.array(datatime)
1074 1074
1075 1075 return datatime
1076 1076
1077 1077 def getTimeInterval(self):
1078 1078
1079 1079 if hasattr(self, 'timeInterval1'):
1080 1080 return self.timeInterval1
1081 1081 else:
1082 1082 return self.paramInterval
1083 1083
1084 1084 def setValue(self, value):
1085 1085
1086 1086 print("This property should not be initialized")
1087 1087
1088 1088 return
1089 1089
1090 1090 def getNoise(self):
1091 1091
1092 1092 return self.spc_noise
1093 1093
1094 1094 timeInterval = property(getTimeInterval)
1095 1095 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
1096 1096
1097 1097
1098 1098 class PlotterData(object):
1099 1099 '''
1100 1100 Object to hold data to be plotted
1101 1101 '''
1102 1102
1103 1103 MAXNUMX = 100
1104 1104 MAXNUMY = 100
1105 1105
1106 1106 def __init__(self, code, throttle_value, exp_code, buffering=True, snr=False):
1107 1107
1108 1108 self.key = code
1109 1109 self.throttle = throttle_value
1110 1110 self.exp_code = exp_code
1111 1111 self.buffering = buffering
1112 1112 self.ready = False
1113 1113 self.localtime = False
1114 1114 self.data = {}
1115 1115 self.meta = {}
1116 1116 self.__times = []
1117 1117 self.__heights = []
1118 1118
1119 1119 if 'snr' in code:
1120 1120 self.plottypes = ['snr']
1121 1121 elif code == 'spc':
1122 1122 self.plottypes = ['spc', 'noise', 'rti']
1123 1123 elif code == 'rti':
1124 1124 self.plottypes = ['noise', 'rti']
1125 1125 else:
1126 1126 self.plottypes = [code]
1127 1127
1128 1128 if 'snr' not in self.plottypes and snr:
1129 1129 self.plottypes.append('snr')
1130 1130
1131 1131 for plot in self.plottypes:
1132 1132 self.data[plot] = {}
1133 1133
1134 1134 def __str__(self):
1135 1135 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
1136 1136 return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times))
1137 1137
1138 1138 def __len__(self):
1139 1139 return len(self.__times)
1140 1140
1141 1141 def __getitem__(self, key):
1142 1142
1143 1143 if key not in self.data:
1144 1144 raise KeyError(log.error('Missing key: {}'.format(key)))
1145 1145 if 'spc' in key or not self.buffering:
1146 1146 ret = self.data[key]
1147 1147 elif 'scope' in key:
1148 1148 ret = numpy.array(self.data[key][float(self.tm)])
1149 1149 else:
1150 1150 ret = numpy.array([self.data[key][x] for x in self.times])
1151 1151 if ret.ndim > 1:
1152 1152 ret = numpy.swapaxes(ret, 0, 1)
1153 1153 return ret
1154 1154
1155 1155 def __contains__(self, key):
1156 1156 return key in self.data
1157 1157
1158 1158 def setup(self):
1159 1159 '''
1160 1160 Configure object
1161 1161 '''
1162 1162
1163 1163 self.type = ''
1164 1164 self.ready = False
1165 1165 self.data = {}
1166 1166 self.__times = []
1167 1167 self.__heights = []
1168 1168 self.__all_heights = set()
1169 1169 for plot in self.plottypes:
1170 1170 if 'snr' in plot:
1171 1171 plot = 'snr'
1172 1172 elif 'spc_moments' == plot:
1173 1173 plot = 'moments'
1174 1174 self.data[plot] = {}
1175 1175
1176 1176 if 'spc' in self.data or 'rti' in self.data or 'cspc' in self.data or 'moments' in self.data:
1177 1177 self.data['noise'] = {}
1178 1178 self.data['rti'] = {}
1179 1179 if 'noise' not in self.plottypes:
1180 1180 self.plottypes.append('noise')
1181 1181 if 'rti' not in self.plottypes:
1182 1182 self.plottypes.append('rti')
1183 1183
1184 1184 def shape(self, key):
1185 1185 '''
1186 1186 Get the shape of the one-element data for the given key
1187 1187 '''
1188 1188
1189 1189 if len(self.data[key]):
1190 1190 if 'spc' in key or not self.buffering:
1191 1191 return self.data[key].shape
1192 1192 return self.data[key][self.__times[0]].shape
1193 1193 return (0,)
1194 1194
1195 1195 def update(self, dataOut, tm):
1196 1196 '''
1197 1197 Update data object with new dataOut
1198 1198 '''
1199 1199
1200 1200 if tm in self.__times:
1201 1201 return
1202 1202 self.profileIndex = dataOut.profileIndex
1203 1203 self.tm = tm
1204 1204 self.type = dataOut.type
1205 1205 self.parameters = getattr(dataOut, 'parameters', [])
1206 1206
1207 1207 if hasattr(dataOut, 'meta'):
1208 1208 self.meta.update(dataOut.meta)
1209 1209
1210 1210 self.pairs = dataOut.pairsList
1211 1211 self.interval = dataOut.getTimeInterval()
1212 1212 self.localtime = dataOut.useLocalTime
1213 1213 if 'spc' in self.plottypes or 'cspc' in self.plottypes or 'spc_moments' in self.plottypes:
1214 1214 self.xrange = (dataOut.getFreqRange(1)/1000.,
1215 1215 dataOut.getAcfRange(1), dataOut.getVelRange(1))
1216 1216 self.factor = dataOut.normFactor
1217 1217 self.__heights.append(dataOut.heightList)
1218 1218 self.__all_heights.update(dataOut.heightList)
1219 1219 self.__times.append(tm)
1220 1220
1221 1221 for plot in self.plottypes:
1222 1222 if plot in ('spc', 'spc_moments'):
1223 1223 z = dataOut.data_spc/dataOut.normFactor
1224 1224 buffer = 10*numpy.log10(z)
1225 1225 if plot == 'cspc':
1226 1226 z = dataOut.data_spc/dataOut.normFactor
1227 1227 buffer = (dataOut.data_spc, dataOut.data_cspc)
1228 1228 if plot == 'noise':
1229 1229 buffer = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
1230 1230 if plot == 'rti':
1231 1231 buffer = dataOut.getPower()
1232 1232 if plot == 'snr_db':
1233 1233 buffer = dataOut.data_SNR
1234 1234 if plot == 'snr':
1235 1235 buffer = 10*numpy.log10(dataOut.data_SNR)
1236 1236 if plot == 'dop':
1237 buffer = 10*numpy.log10(dataOut.data_DOP)
1238 if plot == 'mean':
1239 buffer = dataOut.data_MEAN
1240 if plot == 'std':
1241 buffer = dataOut.data_STD
1237 buffer = dataOut.data_DOP
1238 if plot == 'pow':
1239 buffer = 10*numpy.log10(dataOut.data_POW)
1240 if plot == 'width':
1241 buffer = dataOut.data_WIDTH
1242 1242 if plot == 'coh':
1243 1243 buffer = dataOut.getCoherence()
1244 1244 if plot == 'phase':
1245 1245 buffer = dataOut.getCoherence(phase=True)
1246 1246 if plot == 'output':
1247 1247 buffer = dataOut.data_output
1248 1248 if plot == 'param':
1249 1249 buffer = dataOut.data_param
1250 1250 if plot == 'scope':
1251 1251 buffer = dataOut.data
1252 1252 self.flagDataAsBlock = dataOut.flagDataAsBlock
1253 1253 self.nProfiles = dataOut.nProfiles
1254 1254
1255 1255 if plot == 'spc':
1256 1256 self.data['spc'] = buffer
1257 1257 elif plot == 'cspc':
1258 1258 self.data['spc'] = buffer[0]
1259 1259 self.data['cspc'] = buffer[1]
1260 1260 elif plot == 'spc_moments':
1261 1261 self.data['spc'] = buffer
1262 1262 self.data['moments'][tm] = dataOut.moments
1263 1263 else:
1264 1264 if self.buffering:
1265 1265 self.data[plot][tm] = buffer
1266 1266 else:
1267 1267 self.data[plot] = buffer
1268 1268
1269 1269 if dataOut.channelList is None:
1270 1270 self.channels = range(buffer.shape[0])
1271 1271 else:
1272 1272 self.channels = dataOut.channelList
1273 1273
1274 1274 def normalize_heights(self):
1275 1275 '''
1276 1276 Ensure same-dimension of the data for different heighList
1277 1277 '''
1278 1278
1279 1279 H = numpy.array(list(self.__all_heights))
1280 1280 H.sort()
1281 1281 for key in self.data:
1282 1282 shape = self.shape(key)[:-1] + H.shape
1283 1283 for tm, obj in list(self.data[key].items()):
1284 1284 h = self.__heights[self.__times.index(tm)]
1285 1285 if H.size == h.size:
1286 1286 continue
1287 1287 index = numpy.where(numpy.in1d(H, h))[0]
1288 1288 dummy = numpy.zeros(shape) + numpy.nan
1289 1289 if len(shape) == 2:
1290 1290 dummy[:, index] = obj
1291 1291 else:
1292 1292 dummy[index] = obj
1293 1293 self.data[key][tm] = dummy
1294 1294
1295 1295 self.__heights = [H for tm in self.__times]
1296 1296
1297 1297 def jsonify(self, plot_name, plot_type, decimate=False):
1298 1298 '''
1299 1299 Convert data to json
1300 1300 '''
1301 1301
1302 1302 tm = self.times[-1]
1303 1303 dy = int(self.heights.size/self.MAXNUMY) + 1
1304 1304 if self.key in ('spc', 'cspc') or not self.buffering:
1305 1305 dx = int(self.data[self.key].shape[1]/self.MAXNUMX) + 1
1306 1306 data = self.roundFloats(
1307 1307 self.data[self.key][::, ::dx, ::dy].tolist())
1308 1308 else:
1309 1309 data = self.roundFloats(self.data[self.key][tm].tolist())
1310 1310 if self.key is 'noise':
1311 1311 data = [[x] for x in data]
1312 1312
1313 1313 meta = {}
1314 1314 ret = {
1315 1315 'plot': plot_name,
1316 1316 'code': self.exp_code,
1317 1317 'time': float(tm),
1318 1318 'data': data,
1319 1319 }
1320 1320 meta['type'] = plot_type
1321 1321 meta['interval'] = float(self.interval)
1322 1322 meta['localtime'] = self.localtime
1323 1323 meta['yrange'] = self.roundFloats(self.heights[::dy].tolist())
1324 1324 if 'spc' in self.data or 'cspc' in self.data:
1325 1325 meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1326 1326 else:
1327 1327 meta['xrange'] = []
1328 1328
1329 1329 meta.update(self.meta)
1330 1330 ret['metadata'] = meta
1331 1331 return json.dumps(ret)
1332 1332
1333 1333 @property
1334 1334 def times(self):
1335 1335 '''
1336 1336 Return the list of times of the current data
1337 1337 '''
1338 1338
1339 1339 ret = numpy.array(self.__times)
1340 1340 ret.sort()
1341 1341 return ret
1342 1342
1343 1343 @property
1344 1344 def min_time(self):
1345 1345 '''
1346 1346 Return the minimun time value
1347 1347 '''
1348 1348
1349 1349 return self.times[0]
1350 1350
1351 1351 @property
1352 1352 def max_time(self):
1353 1353 '''
1354 1354 Return the maximun time value
1355 1355 '''
1356 1356
1357 1357 return self.times[-1]
1358 1358
1359 1359 @property
1360 1360 def heights(self):
1361 1361 '''
1362 1362 Return the list of heights of the current data
1363 1363 '''
1364 1364
1365 1365 return numpy.array(self.__heights[-1])
1366 1366
1367 1367 @staticmethod
1368 1368 def roundFloats(obj):
1369 1369 if isinstance(obj, list):
1370 1370 return list(map(PlotterData.roundFloats, obj))
1371 1371 elif isinstance(obj, float):
1372 1372 return round(obj, 2)
Show file before | Show file after | Hide comments
@@ -1,1833 +1,1833
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time
14 14 import datetime
15 15 import traceback
16 16 import zmq
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25 25 from schainpy.utils import log
26 26 import schainpy.admin
27 27
28 28 LOCALTIME = True
29 29
30 30
31 31 def isNumber(cad):
32 32 """
33 33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
34 34
35 35 Excepciones:
36 36 Si un determinado string no puede ser convertido a numero
37 37 Input:
38 38 str, string al cual se le analiza para determinar si convertible a un numero o no
39 39
40 40 Return:
41 41 True : si el string es uno numerico
42 42 False : no es un string numerico
43 43 """
44 44 try:
45 45 float(cad)
46 46 return True
47 47 except:
48 48 return False
49 49
50 50
51 51 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
52 52 """
53 53 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
54 54
55 55 Inputs:
56 56 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
57 57
58 58 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
59 59 segundos contados desde 01/01/1970.
60 60 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
61 61 segundos contados desde 01/01/1970.
62 62
63 63 Return:
64 64 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
65 65 fecha especificado, de lo contrario retorna False.
66 66
67 67 Excepciones:
68 68 Si el archivo no existe o no puede ser abierto
69 69 Si la cabecera no puede ser leida.
70 70
71 71 """
72 72 basicHeaderObj = BasicHeader(LOCALTIME)
73 73
74 74 try:
75 75 fp = open(filename, 'rb')
76 76 except IOError:
77 77 print("The file %s can't be opened" % (filename))
78 78 return 0
79 79
80 80 sts = basicHeaderObj.read(fp)
81 81 fp.close()
82 82
83 83 if not(sts):
84 84 print("Skipping the file %s because it has not a valid header" % (filename))
85 85 return 0
86 86
87 87 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
88 88 return 0
89 89
90 90 return 1
91 91
92 92
93 93 def isTimeInRange(thisTime, startTime, endTime):
94 94 if endTime >= startTime:
95 95 if (thisTime < startTime) or (thisTime > endTime):
96 96 return 0
97 97 return 1
98 98 else:
99 99 if (thisTime < startTime) and (thisTime > endTime):
100 100 return 0
101 101 return 1
102 102
103 103
104 104 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
105 105 """
106 106 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
107 107
108 108 Inputs:
109 109 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
110 110
111 111 startDate : fecha inicial del rango seleccionado en formato datetime.date
112 112
113 113 endDate : fecha final del rango seleccionado en formato datetime.date
114 114
115 115 startTime : tiempo inicial del rango seleccionado en formato datetime.time
116 116
117 117 endTime : tiempo final del rango seleccionado en formato datetime.time
118 118
119 119 Return:
120 120 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
121 121 fecha especificado, de lo contrario retorna False.
122 122
123 123 Excepciones:
124 124 Si el archivo no existe o no puede ser abierto
125 125 Si la cabecera no puede ser leida.
126 126
127 127 """
128 128
129 129 try:
130 130 fp = open(filename, 'rb')
131 131 except IOError:
132 132 print("The file %s can't be opened" % (filename))
133 133 return None
134 134
135 135 firstBasicHeaderObj = BasicHeader(LOCALTIME)
136 136 systemHeaderObj = SystemHeader()
137 137 radarControllerHeaderObj = RadarControllerHeader()
138 138 processingHeaderObj = ProcessingHeader()
139 139
140 140 lastBasicHeaderObj = BasicHeader(LOCALTIME)
141 141
142 142 sts = firstBasicHeaderObj.read(fp)
143 143
144 144 if not(sts):
145 145 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
146 146 return None
147 147
148 148 if not systemHeaderObj.read(fp):
149 149 return None
150 150
151 151 if not radarControllerHeaderObj.read(fp):
152 152 return None
153 153
154 154 if not processingHeaderObj.read(fp):
155 155 return None
156 156
157 157 filesize = os.path.getsize(filename)
158 158
159 159 offset = processingHeaderObj.blockSize + 24 # header size
160 160
161 161 if filesize <= offset:
162 162 print("[Reading] %s: This file has not enough data" % filename)
163 163 return None
164 164
165 165 fp.seek(-offset, 2)
166 166
167 167 sts = lastBasicHeaderObj.read(fp)
168 168
169 169 fp.close()
170 170
171 171 thisDatetime = lastBasicHeaderObj.datatime
172 172 thisTime_last_block = thisDatetime.time()
173 173
174 174 thisDatetime = firstBasicHeaderObj.datatime
175 175 thisDate = thisDatetime.date()
176 176 thisTime_first_block = thisDatetime.time()
177 177
178 178 # General case
179 179 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
180 180 #-----------o----------------------------o-----------
181 181 # startTime endTime
182 182
183 183 if endTime >= startTime:
184 184 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
185 185 return None
186 186
187 187 return thisDatetime
188 188
189 189 # If endTime < startTime then endTime belongs to the next day
190 190
191 191 #<<<<<<<<<<<o o>>>>>>>>>>>
192 192 #-----------o----------------------------o-----------
193 193 # endTime startTime
194 194
195 195 if (thisDate == startDate) and (thisTime_last_block < startTime):
196 196 return None
197 197
198 198 if (thisDate == endDate) and (thisTime_first_block > endTime):
199 199 return None
200 200
201 201 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
202 202 return None
203 203
204 204 return thisDatetime
205 205
206 206
207 207 def isFolderInDateRange(folder, startDate=None, endDate=None):
208 208 """
209 209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
210 210
211 211 Inputs:
212 212 folder : nombre completo del directorio.
213 213 Su formato deberia ser "/path_root/?YYYYDDD"
214 214
215 215 siendo:
216 216 YYYY : Anio (ejemplo 2015)
217 217 DDD : Dia del anio (ejemplo 305)
218 218
219 219 startDate : fecha inicial del rango seleccionado en formato datetime.date
220 220
221 221 endDate : fecha final del rango seleccionado en formato datetime.date
222 222
223 223 Return:
224 224 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
225 225 fecha especificado, de lo contrario retorna False.
226 226 Excepciones:
227 227 Si el directorio no tiene el formato adecuado
228 228 """
229 229
230 230 basename = os.path.basename(folder)
231 231
232 232 if not isRadarFolder(basename):
233 233 print("The folder %s has not the rigth format" % folder)
234 234 return 0
235 235
236 236 if startDate and endDate:
237 237 thisDate = getDateFromRadarFolder(basename)
238 238
239 239 if thisDate < startDate:
240 240 return 0
241 241
242 242 if thisDate > endDate:
243 243 return 0
244 244
245 245 return 1
246 246
247 247
248 248 def isFileInDateRange(filename, startDate=None, endDate=None):
249 249 """
250 250 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
251 251
252 252 Inputs:
253 253 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
254 254
255 255 Su formato deberia ser "?YYYYDDDsss"
256 256
257 257 siendo:
258 258 YYYY : Anio (ejemplo 2015)
259 259 DDD : Dia del anio (ejemplo 305)
260 260 sss : set
261 261
262 262 startDate : fecha inicial del rango seleccionado en formato datetime.date
263 263
264 264 endDate : fecha final del rango seleccionado en formato datetime.date
265 265
266 266 Return:
267 267 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
268 268 fecha especificado, de lo contrario retorna False.
269 269 Excepciones:
270 270 Si el archivo no tiene el formato adecuado
271 271 """
272 272
273 273 basename = os.path.basename(filename)
274 274
275 275 if not isRadarFile(basename):
276 276 print("The filename %s has not the rigth format" % filename)
277 277 return 0
278 278
279 279 if startDate and endDate:
280 280 thisDate = getDateFromRadarFile(basename)
281 281
282 282 if thisDate < startDate:
283 283 return 0
284 284
285 285 if thisDate > endDate:
286 286 return 0
287 287
288 288 return 1
289 289
290 290
291 291 def getFileFromSet(path, ext, set):
292 292 validFilelist = []
293 293 fileList = os.listdir(path)
294 294
295 295 # 0 1234 567 89A BCDE
296 296 # H YYYY DDD SSS .ext
297 297
298 298 for thisFile in fileList:
299 299 try:
300 300 year = int(thisFile[1:5])
301 301 doy = int(thisFile[5:8])
302 302 except:
303 303 continue
304 304
305 305 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
306 306 continue
307 307
308 308 validFilelist.append(thisFile)
309 309
310 310 myfile = fnmatch.filter(
311 311 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
312 312
313 313 if len(myfile) != 0:
314 314 return myfile[0]
315 315 else:
316 316 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
317 317 print('the filename %s does not exist' % filename)
318 318 print('...going to the last file: ')
319 319
320 320 if validFilelist:
321 321 validFilelist = sorted(validFilelist, key=str.lower)
322 322 return validFilelist[-1]
323 323
324 324 return None
325 325
326 326
327 327 def getlastFileFromPath(path, ext):
328 328 """
329 329 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
330 330 al final de la depuracion devuelve el ultimo file de la lista que quedo.
331 331
332 332 Input:
333 333 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
334 334 ext : extension de los files contenidos en una carpeta
335 335
336 336 Return:
337 337 El ultimo file de una determinada carpeta, no se considera el path.
338 338 """
339 339 validFilelist = []
340 340 fileList = os.listdir(path)
341 341
342 342 # 0 1234 567 89A BCDE
343 343 # H YYYY DDD SSS .ext
344 344
345 345 for thisFile in fileList:
346 346
347 347 year = thisFile[1:5]
348 348 if not isNumber(year):
349 349 continue
350 350
351 351 doy = thisFile[5:8]
352 352 if not isNumber(doy):
353 353 continue
354 354
355 355 year = int(year)
356 356 doy = int(doy)
357 357
358 358 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
359 359 continue
360 360
361 361 validFilelist.append(thisFile)
362 362
363 363 if validFilelist:
364 364 validFilelist = sorted(validFilelist, key=str.lower)
365 365 return validFilelist[-1]
366 366
367 367 return None
368 368
369 369
370 370 def checkForRealPath(path, foldercounter, year, doy, set, ext):
371 371 """
372 372 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
373 373 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
374 374 el path exacto de un determinado file.
375 375
376 376 Example :
377 377 nombre correcto del file es .../.../D2009307/P2009307367.ext
378 378
379 379 Entonces la funcion prueba con las siguientes combinaciones
380 380 .../.../y2009307367.ext
381 381 .../.../Y2009307367.ext
382 382 .../.../x2009307/y2009307367.ext
383 383 .../.../x2009307/Y2009307367.ext
384 384 .../.../X2009307/y2009307367.ext
385 385 .../.../X2009307/Y2009307367.ext
386 386 siendo para este caso, la ultima combinacion de letras, identica al file buscado
387 387
388 388 Return:
389 389 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
390 390 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
391 391 para el filename
392 392 """
393 393 fullfilename = None
394 394 find_flag = False
395 395 filename = None
396 396
397 397 prefixDirList = [None, 'd', 'D']
398 398 if ext.lower() == ".r": # voltage
399 399 prefixFileList = ['d', 'D']
400 400 elif ext.lower() == ".pdata": # spectra
401 401 prefixFileList = ['p', 'P']
402 402 else:
403 403 return None, filename
404 404
405 405 # barrido por las combinaciones posibles
406 406 for prefixDir in prefixDirList:
407 407 thispath = path
408 408 if prefixDir != None:
409 409 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
410 410 if foldercounter == 0:
411 411 thispath = os.path.join(path, "%s%04d%03d" %
412 412 (prefixDir, year, doy))
413 413 else:
414 414 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
415 415 prefixDir, year, doy, foldercounter))
416 416 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
417 417 # formo el nombre del file xYYYYDDDSSS.ext
418 418 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
419 419 fullfilename = os.path.join(
420 420 thispath, filename) # formo el path completo
421 421
422 422 if os.path.exists(fullfilename): # verifico que exista
423 423 find_flag = True
424 424 break
425 425 if find_flag:
426 426 break
427 427
428 428 if not(find_flag):
429 429 return None, filename
430 430
431 431 return fullfilename, filename
432 432
433 433
434 434 def isRadarFolder(folder):
435 435 try:
436 436 year = int(folder[1:5])
437 437 doy = int(folder[5:8])
438 438 except:
439 439 return 0
440 440
441 441 return 1
442 442
443 443
444 444 def isRadarFile(file):
445 445 try:
446 446 year = int(file[1:5])
447 447 doy = int(file[5:8])
448 448 set = int(file[8:11])
449 449 except:
450 450 return 0
451 451
452 452 return 1
453 453
454 454
455 455 def getDateFromRadarFile(file):
456 456 try:
457 457 year = int(file[1:5])
458 458 doy = int(file[5:8])
459 459 set = int(file[8:11])
460 460 except:
461 461 return None
462 462
463 463 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
464 464 return thisDate
465 465
466 466
467 467 def getDateFromRadarFolder(folder):
468 468 try:
469 469 year = int(folder[1:5])
470 470 doy = int(folder[5:8])
471 471 except:
472 472 return None
473 473
474 474 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
475 475 return thisDate
476 476
477 477
478 478 class JRODataIO:
479 479
480 480 c = 3E8
481 481
482 482 isConfig = False
483 483
484 484 basicHeaderObj = None
485 485
486 486 systemHeaderObj = None
487 487
488 488 radarControllerHeaderObj = None
489 489
490 490 processingHeaderObj = None
491 491
492 492 dtype = None
493 493
494 494 pathList = []
495 495
496 496 filenameList = []
497 497
498 498 filename = None
499 499
500 500 ext = None
501 501
502 502 flagIsNewFile = 1
503 503
504 504 flagDiscontinuousBlock = 0
505 505
506 506 flagIsNewBlock = 0
507 507
508 508 fp = None
509 509
510 510 firstHeaderSize = 0
511 511
512 512 basicHeaderSize = 24
513 513
514 514 versionFile = 1103
515 515
516 516 fileSize = None
517 517
518 518 # ippSeconds = None
519 519
520 520 fileSizeByHeader = None
521 521
522 522 fileIndex = None
523 523
524 524 profileIndex = None
525 525
526 526 blockIndex = None
527 527
528 528 nTotalBlocks = None
529 529
530 530 maxTimeStep = 30
531 531
532 532 lastUTTime = None
533 533
534 534 datablock = None
535 535
536 536 dataOut = None
537 537
538 538 blocksize = None
539 539
540 540 getByBlock = False
541 541
542 542 def __init__(self):
543 543
544 544 raise NotImplementedError
545 545
546 546 def run(self):
547 547
548 548 raise NotImplementedError
549 549
550 550 def getDtypeWidth(self):
551 551
552 552 dtype_index = get_dtype_index(self.dtype)
553 553 dtype_width = get_dtype_width(dtype_index)
554 554
555 555 return dtype_width
556 556
557 557 def getAllowedArgs(self):
558 558 if hasattr(self, '__attrs__'):
559 559 return self.__attrs__
560 560 else:
561 561 return inspect.getargspec(self.run).args
562 562
563 563
564 564 class JRODataReader(JRODataIO):
565 565
566 566 online = 0
567 567
568 568 realtime = 0
569 569
570 570 nReadBlocks = 0
571 571
572 572 delay = 10 # number of seconds waiting a new file
573 573
574 574 nTries = 3 # quantity tries
575 575
576 576 nFiles = 3 # number of files for searching
577 577
578 578 path = None
579 579
580 580 foldercounter = 0
581 581
582 582 flagNoMoreFiles = 0
583 583
584 584 datetimeList = []
585 585
586 586 __isFirstTimeOnline = 1
587 587
588 588 __printInfo = True
589 589
590 590 profileIndex = None
591 591
592 592 nTxs = 1
593 593
594 594 txIndex = None
595 595
596 596 # Added--------------------
597 597
598 598 selBlocksize = None
599 599
600 600 selBlocktime = None
601 601
602 602 def __init__(self):
603 603 """
604 604 This class is used to find data files
605 605
606 606 Example:
607 607 reader = JRODataReader()
608 608 fileList = reader.findDataFiles()
609 609
610 610 """
611 611 pass
612 612
613 613 def createObjByDefault(self):
614 614 """
615 615
616 616 """
617 617 raise NotImplementedError
618 618
619 619 def getBlockDimension(self):
620 620
621 621 raise NotImplementedError
622 622
623 623 def searchFilesOffLine(self,
624 624 path,
625 625 startDate=None,
626 626 endDate=None,
627 627 startTime=datetime.time(0, 0, 0),
628 628 endTime=datetime.time(23, 59, 59),
629 629 set=None,
630 630 expLabel='',
631 631 ext='.r',
632 632 cursor=None,
633 633 skip=None,
634 634 walk=True):
635 635
636 636 self.filenameList = []
637 637 self.datetimeList = []
638 638
639 639 pathList = []
640 640
641 641 dateList, pathList = self.findDatafiles(
642 642 path, startDate, endDate, expLabel, ext, walk, include_path=True)
643 643
644 644 if dateList == []:
645 645 return [], []
646 646
647 647 if len(dateList) > 1:
648 648 print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
649 649 else:
650 650 print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
651 651
652 652 filenameList = []
653 653 datetimeList = []
654 654
655 655 for thisPath in pathList:
656 656
657 657 fileList = glob.glob1(thisPath, "*%s" % ext)
658 658 fileList.sort()
659 659
660 660 for file in fileList:
661 661
662 662 filename = os.path.join(thisPath, file)
663 663
664 664 if not isFileInDateRange(filename, startDate, endDate):
665 665 continue
666 666
667 667 thisDatetime = isFileInTimeRange(
668 668 filename, startDate, endDate, startTime, endTime)
669 669
670 670 if not(thisDatetime):
671 671 continue
672 672
673 673 filenameList.append(filename)
674 674 datetimeList.append(thisDatetime)
675 675
676 676 if cursor is not None and skip is not None:
677 677 filenameList = filenameList[cursor * skip:cursor * skip + skip]
678 678 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
679 679
680 680 if not(filenameList):
681 681 print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
682 682 return [], []
683 683
684 684 print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
685 685
686 686 # for i in range(len(filenameList)):
687 687 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
688 688
689 689 self.filenameList = filenameList
690 690 self.datetimeList = datetimeList
691 691
692 692 return pathList, filenameList
693 693
694 694 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
695 695 """
696 696 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
697 697 devuelve el archivo encontrado ademas de otros datos.
698 698
699 699 Input:
700 700 path : carpeta donde estan contenidos los files que contiene data
701 701
702 702 expLabel : Nombre del subexperimento (subfolder)
703 703
704 704 ext : extension de los files
705 705
706 706 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
707 707
708 708 Return:
709 709 directory : eL directorio donde esta el file encontrado
710 710 filename : el ultimo file de una determinada carpeta
711 711 year : el anho
712 712 doy : el numero de dia del anho
713 713 set : el set del archivo
714 714
715 715
716 716 """
717 717 if not os.path.isdir(path):
718 718 return None, None, None, None, None, None
719 719
720 720 dirList = []
721 721
722 722 if not walk:
723 723 fullpath = path
724 724 foldercounter = 0
725 725 else:
726 726 # Filtra solo los directorios
727 727 for thisPath in os.listdir(path):
728 728 if not os.path.isdir(os.path.join(path, thisPath)):
729 729 continue
730 730 if not isRadarFolder(thisPath):
731 731 continue
732 732
733 733 dirList.append(thisPath)
734 734
735 735 if not(dirList):
736 736 return None, None, None, None, None, None
737 737
738 738 dirList = sorted(dirList, key=str.lower)
739 739
740 740 doypath = dirList[-1]
741 741 foldercounter = int(doypath.split('_')[1]) if len(
742 742 doypath.split('_')) > 1 else 0
743 743 fullpath = os.path.join(path, doypath, expLabel)
744 744
745 745 print("[Reading] %s folder was found: " % (fullpath))
746 746
747 747 if set == None:
748 748 filename = getlastFileFromPath(fullpath, ext)
749 749 else:
750 750 filename = getFileFromSet(fullpath, ext, set)
751 751
752 752 if not(filename):
753 753 return None, None, None, None, None, None
754 754
755 755 print("[Reading] %s file was found" % (filename))
756 756
757 757 if not(self.__verifyFile(os.path.join(fullpath, filename))):
758 758 return None, None, None, None, None, None
759 759
760 760 year = int(filename[1:5])
761 761 doy = int(filename[5:8])
762 762 set = int(filename[8:11])
763 763
764 764 return fullpath, foldercounter, filename, year, doy, set
765 765
766 766 def __setNextFileOffline(self):
767 767
768 768 idFile = self.fileIndex
769 769
770 770 while (True):
771 771 idFile += 1
772 772 if not(idFile < len(self.filenameList)):
773 773 self.flagNoMoreFiles = 1
774 774 # print "[Reading] No more Files"
775 775 return 0
776 776
777 777 filename = self.filenameList[idFile]
778 778
779 779 if not(self.__verifyFile(filename)):
780 780 continue
781 781
782 782 fileSize = os.path.getsize(filename)
783 783 fp = open(filename, 'rb')
784 784 break
785 785
786 786 self.flagIsNewFile = 1
787 787 self.fileIndex = idFile
788 788 self.filename = filename
789 789 self.fileSize = fileSize
790 790 self.fp = fp
791 791
792 792 # print "[Reading] Setting the file: %s"%self.filename
793 793
794 794 return 1
795 795
796 796 def __setNextFileOnline(self):
797 797 """
798 798 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
799 799 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
800 800 siguientes.
801 801
802 802 Affected:
803 803 self.flagIsNewFile
804 804 self.filename
805 805 self.fileSize
806 806 self.fp
807 807 self.set
808 808 self.flagNoMoreFiles
809 809
810 810 Return:
811 811 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
812 812 1 : si el file fue abierto con exito y esta listo a ser leido
813 813
814 814 Excepciones:
815 815 Si un determinado file no puede ser abierto
816 816 """
817 817 nFiles = 0
818 818 fileOk_flag = False
819 819 firstTime_flag = True
820 820
821 821 self.set += 1
822 822
823 823 if self.set > 999:
824 824 self.set = 0
825 825 self.foldercounter += 1
826 826
827 827 # busca el 1er file disponible
828 828 fullfilename, filename = checkForRealPath(
829 829 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
830 830 if fullfilename:
831 831 if self.__verifyFile(fullfilename, False):
832 832 fileOk_flag = True
833 833
834 834 # si no encuentra un file entonces espera y vuelve a buscar
835 835 if not(fileOk_flag):
836 836 # busco en los siguientes self.nFiles+1 files posibles
837 837 for nFiles in range(self.nFiles + 1):
838 838
839 839 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
840 840 tries = self.nTries
841 841 else:
842 842 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
843 843
844 844 for nTries in range(tries):
845 845 if firstTime_flag:
846 846 print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1))
847 847 sleep(self.delay)
848 848 else:
849 849 print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext))
850 850
851 851 fullfilename, filename = checkForRealPath(
852 852 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
853 853 if fullfilename:
854 854 if self.__verifyFile(fullfilename):
855 855 fileOk_flag = True
856 856 break
857 857
858 858 if fileOk_flag:
859 859 break
860 860
861 861 firstTime_flag = False
862 862
863 863 log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
864 864 self.set += 1
865 865
866 866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
867 867 if nFiles == (self.nFiles - 1):
868 868 self.set = 0
869 869 self.doy += 1
870 870 self.foldercounter = 0
871 871
872 872 if fileOk_flag:
873 873 self.fileSize = os.path.getsize(fullfilename)
874 874 self.filename = fullfilename
875 875 self.flagIsNewFile = 1
876 876 if self.fp != None:
877 877 self.fp.close()
878 878 self.fp = open(fullfilename, 'rb')
879 879 self.flagNoMoreFiles = 0
880 880 # print '[Reading] Setting the file: %s' % fullfilename
881 881 else:
882 882 self.fileSize = 0
883 883 self.filename = None
884 884 self.flagIsNewFile = 0
885 885 self.fp = None
886 886 self.flagNoMoreFiles = 1
887 887 # print '[Reading] No more files to read'
888 888
889 889 return fileOk_flag
890 890
891 891 def setNextFile(self):
892 892 if self.fp != None:
893 893 self.fp.close()
894 894
895 895 if self.online:
896 896 newFile = self.__setNextFileOnline()
897 897 else:
898 898 newFile = self.__setNextFileOffline()
899 899
900 900 if not(newFile):
901 901 self.dataOut.error = 'No more files to read'
902 902 return 0
903 903
904 904 if self.verbose:
905 905 print('[Reading] Setting the file: %s' % self.filename)
906 906
907 907 self.__readFirstHeader()
908 908 self.nReadBlocks = 0
909 909 return 1
910 910
911 911 def __waitNewBlock(self):
912 912 """
913 913 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
914 914
915 915 Si el modo de lectura es OffLine siempre retorn 0
916 916 """
917 917 if not self.online:
918 918 return 0
919 919
920 920 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
921 921 return 0
922 922
923 923 currentPointer = self.fp.tell()
924 924
925 925 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
926 926
927 927 for nTries in range(self.nTries):
928 928
929 929 self.fp.close()
930 930 self.fp = open(self.filename, 'rb')
931 931 self.fp.seek(currentPointer)
932 932
933 933 self.fileSize = os.path.getsize(self.filename)
934 934 currentSize = self.fileSize - currentPointer
935 935
936 936 if (currentSize >= neededSize):
937 937 self.basicHeaderObj.read(self.fp)
938 938 return 1
939 939
940 940 if self.fileSize == self.fileSizeByHeader:
941 941 # self.flagEoF = True
942 942 return 0
943 943
944 944 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
945 945 sleep(self.delay)
946 946
947 947 return 0
948 948
949 949 def waitDataBlock(self, pointer_location, blocksize=None):
950 950
951 951 currentPointer = pointer_location
952 952 if blocksize is None:
953 953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
954 954 else:
955 955 neededSize = blocksize
956 956
957 957 for nTries in range(self.nTries):
958 958 self.fp.close()
959 959 self.fp = open(self.filename, 'rb')
960 960 self.fp.seek(currentPointer)
961 961
962 962 self.fileSize = os.path.getsize(self.filename)
963 963 currentSize = self.fileSize - currentPointer
964 964
965 965 if (currentSize >= neededSize):
966 966 return 1
967 967
968 968 log.warning(
969 969 "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
970 970 self.name
971 971 )
972 972 sleep(self.delay)
973 973
974 974 return 0
975 975
976 976 def __jumpToLastBlock(self):
977 977
978 978 if not(self.__isFirstTimeOnline):
979 979 return
980 980
981 981 csize = self.fileSize - self.fp.tell()
982 982 blocksize = self.processingHeaderObj.blockSize
983 983
984 984 # salta el primer bloque de datos
985 985 if csize > self.processingHeaderObj.blockSize:
986 986 self.fp.seek(self.fp.tell() + blocksize)
987 987 else:
988 988 return
989 989
990 990 csize = self.fileSize - self.fp.tell()
991 991 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
992 992 while True:
993 993
994 994 if self.fp.tell() < self.fileSize:
995 995 self.fp.seek(self.fp.tell() + neededsize)
996 996 else:
997 997 self.fp.seek(self.fp.tell() - neededsize)
998 998 break
999 999
1000 1000 # csize = self.fileSize - self.fp.tell()
1001 1001 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1002 1002 # factor = int(csize/neededsize)
1003 1003 # if factor > 0:
1004 1004 # self.fp.seek(self.fp.tell() + factor*neededsize)
1005 1005
1006 1006 self.flagIsNewFile = 0
1007 1007 self.__isFirstTimeOnline = 0
1008 1008
1009 1009 def __setNewBlock(self):
1010 1010 # if self.server is None:
1011 1011 if self.fp == None:
1012 1012 return 0
1013 1013
1014 1014 # if self.online:
1015 1015 # self.__jumpToLastBlock()
1016 1016
1017 1017 if self.flagIsNewFile:
1018 1018 self.lastUTTime = self.basicHeaderObj.utc
1019 1019 return 1
1020 1020
1021 1021 if self.realtime:
1022 1022 self.flagDiscontinuousBlock = 1
1023 1023 if not(self.setNextFile()):
1024 1024 return 0
1025 1025 else:
1026 1026 return 1
1027 1027 # if self.server is None:
1028 1028 currentSize = self.fileSize - self.fp.tell()
1029 1029 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1030 1030 if (currentSize >= neededSize):
1031 1031 self.basicHeaderObj.read(self.fp)
1032 1032 self.lastUTTime = self.basicHeaderObj.utc
1033 1033 return 1
1034 1034 # else:
1035 1035 # self.basicHeaderObj.read(self.zHeader)
1036 1036 # self.lastUTTime = self.basicHeaderObj.utc
1037 1037 # return 1
1038 1038 if self.__waitNewBlock():
1039 1039 self.lastUTTime = self.basicHeaderObj.utc
1040 1040 return 1
1041 1041 # if self.server is None:
1042 1042 if not(self.setNextFile()):
1043 1043 return 0
1044 1044
1045 1045 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1046 1046 self.lastUTTime = self.basicHeaderObj.utc
1047 1047
1048 1048 self.flagDiscontinuousBlock = 0
1049 1049
1050 1050 if deltaTime > self.maxTimeStep:
1051 1051 self.flagDiscontinuousBlock = 1
1052 1052
1053 1053 return 1
1054 1054
1055 1055 def readNextBlock(self):
1056 1056
1057 1057 # Skip block out of startTime and endTime
1058 1058 while True:
1059 1059 if not(self.__setNewBlock()):
1060 1060 self.dataOut.error = 'No more files to read'
1061 1061 return 0
1062 1062
1063 1063 if not(self.readBlock()):
1064 1064 return 0
1065 1065
1066 1066 self.getBasicHeader()
1067 1067 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1068 1068 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1069 1069 self.processingHeaderObj.dataBlocksPerFile,
1070 1070 self.dataOut.datatime.ctime()))
1071 1071 continue
1072 1072
1073 1073 break
1074 1074
1075 1075 if self.verbose:
1076 1076 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1077 1077 self.processingHeaderObj.dataBlocksPerFile,
1078 1078 self.dataOut.datatime.ctime()))
1079 1079 return 1
1080 1080
1081 1081 def __readFirstHeader(self):
1082 1082
1083 1083 self.basicHeaderObj.read(self.fp)
1084 1084 self.systemHeaderObj.read(self.fp)
1085 1085 self.radarControllerHeaderObj.read(self.fp)
1086 1086 self.processingHeaderObj.read(self.fp)
1087 1087
1088 1088 self.firstHeaderSize = self.basicHeaderObj.size
1089 1089
1090 1090 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1091 1091 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1092 1092 if datatype == 0:
1093 1093 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1094 1094 elif datatype == 1:
1095 1095 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1096 1096 elif datatype == 2:
1097 1097 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1098 1098 elif datatype == 3:
1099 1099 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1100 1100 elif datatype == 4:
1101 1101 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1102 1102 elif datatype == 5:
1103 1103 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1104 1104 else:
1105 1105 raise ValueError('Data type was not defined')
1106 1106
1107 1107 self.dtype = datatype_str
1108 1108 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1109 1109 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1110 1110 self.firstHeaderSize + self.basicHeaderSize * \
1111 1111 (self.processingHeaderObj.dataBlocksPerFile - 1)
1112 1112 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1113 1113 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1114 1114 self.getBlockDimension()
1115 1115
1116 1116 def __verifyFile(self, filename, msgFlag=True):
1117 1117
1118 1118 msg = None
1119 1119
1120 1120 try:
1121 1121 fp = open(filename, 'rb')
1122 1122 except IOError:
1123 1123
1124 1124 if msgFlag:
1125 1125 print("[Reading] File %s can't be opened" % (filename))
1126 1126
1127 1127 return False
1128 1128
1129 1129 currentPosition = fp.tell()
1130 1130 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1131 1131
1132 1132 if neededSize == 0:
1133 1133 basicHeaderObj = BasicHeader(LOCALTIME)
1134 1134 systemHeaderObj = SystemHeader()
1135 1135 radarControllerHeaderObj = RadarControllerHeader()
1136 1136 processingHeaderObj = ProcessingHeader()
1137 1137
1138 1138 if not(basicHeaderObj.read(fp)):
1139 1139 fp.close()
1140 1140 return False
1141 1141
1142 1142 if not(systemHeaderObj.read(fp)):
1143 1143 fp.close()
1144 1144 return False
1145 1145
1146 1146 if not(radarControllerHeaderObj.read(fp)):
1147 1147 fp.close()
1148 1148 return False
1149 1149
1150 1150 if not(processingHeaderObj.read(fp)):
1151 1151 fp.close()
1152 1152 return False
1153 1153
1154 1154 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1155 1155 else:
1156 1156 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1157 1157
1158 1158 fp.close()
1159 1159
1160 1160 fileSize = os.path.getsize(filename)
1161 1161 currentSize = fileSize - currentPosition
1162 1162
1163 1163 if currentSize < neededSize:
1164 1164 if msgFlag and (msg != None):
1165 1165 print(msg)
1166 1166 return False
1167 1167
1168 1168 return True
1169 1169
1170 1170 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1171 1171
1172 1172 path_empty = True
1173 1173
1174 1174 dateList = []
1175 1175 pathList = []
1176 1176
1177 1177 multi_path = path.split(',')
1178 1178
1179 1179 if not walk:
1180 1180
1181 1181 for single_path in multi_path:
1182 1182
1183 1183 if not os.path.isdir(single_path):
1184 1184 continue
1185 1185
1186 1186 fileList = glob.glob1(single_path, "*" + ext)
1187 1187
1188 1188 if not fileList:
1189 1189 continue
1190 1190
1191 1191 path_empty = False
1192 1192
1193 1193 fileList.sort()
1194 1194
1195 1195 for thisFile in fileList:
1196 1196
1197 1197 if not os.path.isfile(os.path.join(single_path, thisFile)):
1198 1198 continue
1199 1199
1200 1200 if not isRadarFile(thisFile):
1201 1201 continue
1202 1202
1203 1203 if not isFileInDateRange(thisFile, startDate, endDate):
1204 1204 continue
1205 1205
1206 1206 thisDate = getDateFromRadarFile(thisFile)
1207 1207
1208 if thisDate in dateList:
1208 if thisDate in dateList or single_path in pathList:
1209 1209 continue
1210 1210
1211 1211 dateList.append(thisDate)
1212 1212 pathList.append(single_path)
1213 1213
1214 1214 else:
1215 1215 for single_path in multi_path:
1216 1216
1217 1217 if not os.path.isdir(single_path):
1218 1218 continue
1219 1219
1220 1220 dirList = []
1221 1221
1222 1222 for thisPath in os.listdir(single_path):
1223 1223
1224 1224 if not os.path.isdir(os.path.join(single_path, thisPath)):
1225 1225 continue
1226 1226
1227 1227 if not isRadarFolder(thisPath):
1228 1228 continue
1229 1229
1230 1230 if not isFolderInDateRange(thisPath, startDate, endDate):
1231 1231 continue
1232 1232
1233 1233 dirList.append(thisPath)
1234 1234
1235 1235 if not dirList:
1236 1236 continue
1237 1237
1238 1238 dirList.sort()
1239 1239
1240 1240 for thisDir in dirList:
1241 1241
1242 1242 datapath = os.path.join(single_path, thisDir, expLabel)
1243 1243 fileList = glob.glob1(datapath, "*" + ext)
1244 1244
1245 1245 if not fileList:
1246 1246 continue
1247 1247
1248 1248 path_empty = False
1249 1249
1250 1250 thisDate = getDateFromRadarFolder(thisDir)
1251 1251
1252 1252 pathList.append(datapath)
1253 1253 dateList.append(thisDate)
1254 1254
1255 1255 dateList.sort()
1256 1256
1257 1257 if walk:
1258 1258 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1259 1259 else:
1260 1260 pattern_path = multi_path[0]
1261 1261
1262 1262 if path_empty:
1263 1263 print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1264 1264 else:
1265 1265 if not dateList:
1266 1266 print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1267 1267
1268 1268 if include_path:
1269 1269 return dateList, pathList
1270 1270
1271 1271 return dateList
1272 1272
1273 1273 def setup(self,
1274 1274 path=None,
1275 1275 startDate=None,
1276 1276 endDate=None,
1277 1277 startTime=datetime.time(0, 0, 0),
1278 1278 endTime=datetime.time(23, 59, 59),
1279 1279 set=None,
1280 1280 expLabel="",
1281 1281 ext=None,
1282 1282 online=False,
1283 1283 delay=60,
1284 1284 walk=True,
1285 1285 getblock=False,
1286 1286 nTxs=1,
1287 1287 realtime=False,
1288 1288 blocksize=None,
1289 1289 blocktime=None,
1290 1290 skip=None,
1291 1291 cursor=None,
1292 1292 warnings=True,
1293 1293 verbose=True,
1294 1294 server=None,
1295 1295 format=None,
1296 1296 oneDDict=None,
1297 1297 twoDDict=None,
1298 1298 independentParam=None):
1299 1299 if server is not None:
1300 1300 if 'tcp://' in server:
1301 1301 address = server
1302 1302 else:
1303 1303 address = 'ipc:///tmp/%s' % server
1304 1304 self.server = address
1305 1305 self.context = zmq.Context()
1306 1306 self.receiver = self.context.socket(zmq.PULL)
1307 1307 self.receiver.connect(self.server)
1308 1308 time.sleep(0.5)
1309 1309 print('[Starting] ReceiverData from {}'.format(self.server))
1310 1310 else:
1311 1311 self.server = None
1312 1312 if path == None:
1313 1313 raise ValueError("[Reading] The path is not valid")
1314 1314
1315 1315 if ext == None:
1316 1316 ext = self.ext
1317 1317
1318 1318 if online:
1319 1319 print("[Reading] Searching files in online mode...")
1320 1320
1321 1321 for nTries in range(self.nTries):
1322 1322 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1323 1323 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1324 1324
1325 1325 if fullpath:
1326 1326 break
1327 1327
1328 1328 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
1329 1329 sleep(self.delay)
1330 1330
1331 1331 if not(fullpath):
1332 1332 self.dataOut.error = 'There isn\'t any valid file in {}'.format(path)
1333 1333 return
1334 1334
1335 1335 self.year = year
1336 1336 self.doy = doy
1337 1337 self.set = set - 1
1338 1338 self.path = path
1339 1339 self.foldercounter = foldercounter
1340 1340 last_set = None
1341 1341 else:
1342 1342 print("[Reading] Searching files in offline mode ...")
1343 1343 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1344 1344 startTime=startTime, endTime=endTime,
1345 1345 set=set, expLabel=expLabel, ext=ext,
1346 1346 walk=walk, cursor=cursor,
1347 1347 skip=skip)
1348 1348
1349 1349 if not(pathList):
1350 1350 self.fileIndex = -1
1351 1351 self.pathList = []
1352 1352 self.filenameList = []
1353 1353 return
1354 1354
1355 1355 self.fileIndex = -1
1356 1356 self.pathList = pathList
1357 1357 self.filenameList = filenameList
1358 1358 file_name = os.path.basename(filenameList[-1])
1359 1359 basename, ext = os.path.splitext(file_name)
1360 1360 last_set = int(basename[-3:])
1361 1361
1362 1362 self.online = online
1363 1363 self.realtime = realtime
1364 1364 self.delay = delay
1365 1365 ext = ext.lower()
1366 1366 self.ext = ext
1367 1367 self.getByBlock = getblock
1368 1368 self.nTxs = nTxs
1369 1369 self.startTime = startTime
1370 1370 self.endTime = endTime
1371 1371 self.endDate = endDate
1372 1372 self.startDate = startDate
1373 1373 # Added-----------------
1374 1374 self.selBlocksize = blocksize
1375 1375 self.selBlocktime = blocktime
1376 1376
1377 1377 # Verbose-----------
1378 1378 self.verbose = verbose
1379 1379 self.warnings = warnings
1380 1380
1381 1381 if not(self.setNextFile()):
1382 1382 if (startDate != None) and (endDate != None):
1383 1383 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
1384 1384 elif startDate != None:
1385 1385 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
1386 1386 else:
1387 1387 print("[Reading] No files")
1388 1388
1389 1389 self.fileIndex = -1
1390 1390 self.pathList = []
1391 1391 self.filenameList = []
1392 1392 return
1393 1393
1394 1394 # self.getBasicHeader()
1395 1395
1396 1396 if last_set != None:
1397 1397 self.dataOut.last_block = last_set * \
1398 1398 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1399 1399 return
1400 1400
1401 1401 def getBasicHeader(self):
1402 1402
1403 1403 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1404 1404 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1405 1405
1406 1406 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1407 1407
1408 1408 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1409 1409
1410 1410 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1411 1411
1412 1412 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1413 1413
1414 1414 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1415 1415
1416 1416 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1417 1417
1418 1418 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1419 1419
1420 1420 def getFirstHeader(self):
1421 1421
1422 1422 raise NotImplementedError
1423 1423
1424 1424 def getData(self):
1425 1425
1426 1426 raise NotImplementedError
1427 1427
1428 1428 def hasNotDataInBuffer(self):
1429 1429
1430 1430 raise NotImplementedError
1431 1431
1432 1432 def readBlock(self):
1433 1433
1434 1434 raise NotImplementedError
1435 1435
1436 1436 def isEndProcess(self):
1437 1437
1438 1438 return self.flagNoMoreFiles
1439 1439
1440 1440 def printReadBlocks(self):
1441 1441
1442 1442 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1443 1443
1444 1444 def printTotalBlocks(self):
1445 1445
1446 1446 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1447 1447
1448 1448 def printNumberOfBlock(self):
1449 1449 'SPAM!'
1450 1450
1451 1451 # if self.flagIsNewBlock:
1452 1452 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1453 1453 # self.processingHeaderObj.dataBlocksPerFile,
1454 1454 # self.dataOut.datatime.ctime())
1455 1455
1456 1456 def printInfo(self):
1457 1457
1458 1458 if self.__printInfo == False:
1459 1459 return
1460 1460
1461 1461 self.basicHeaderObj.printInfo()
1462 1462 self.systemHeaderObj.printInfo()
1463 1463 self.radarControllerHeaderObj.printInfo()
1464 1464 self.processingHeaderObj.printInfo()
1465 1465
1466 1466 self.__printInfo = False
1467 1467
1468 1468 def run(self,
1469 1469 path=None,
1470 1470 startDate=None,
1471 1471 endDate=None,
1472 1472 startTime=datetime.time(0, 0, 0),
1473 1473 endTime=datetime.time(23, 59, 59),
1474 1474 set=None,
1475 1475 expLabel="",
1476 1476 ext=None,
1477 1477 online=False,
1478 1478 delay=60,
1479 1479 walk=True,
1480 1480 getblock=False,
1481 1481 nTxs=1,
1482 1482 realtime=False,
1483 1483 blocksize=None,
1484 1484 blocktime=None,
1485 1485 skip=None,
1486 1486 cursor=None,
1487 1487 warnings=True,
1488 1488 server=None,
1489 1489 verbose=True,
1490 1490 format=None,
1491 1491 oneDDict=None,
1492 1492 twoDDict=None,
1493 1493 independentParam=None, **kwargs):
1494 1494
1495 1495 if not(self.isConfig):
1496 1496 self.setup(path=path,
1497 1497 startDate=startDate,
1498 1498 endDate=endDate,
1499 1499 startTime=startTime,
1500 1500 endTime=endTime,
1501 1501 set=set,
1502 1502 expLabel=expLabel,
1503 1503 ext=ext,
1504 1504 online=online,
1505 1505 delay=delay,
1506 1506 walk=walk,
1507 1507 getblock=getblock,
1508 1508 nTxs=nTxs,
1509 1509 realtime=realtime,
1510 1510 blocksize=blocksize,
1511 1511 blocktime=blocktime,
1512 1512 skip=skip,
1513 1513 cursor=cursor,
1514 1514 warnings=warnings,
1515 1515 server=server,
1516 1516 verbose=verbose,
1517 1517 format=format,
1518 1518 oneDDict=oneDDict,
1519 1519 twoDDict=twoDDict,
1520 1520 independentParam=independentParam)
1521 1521 self.isConfig = True
1522 1522 if server is None:
1523 1523 self.getData()
1524 1524 else:
1525 1525 self.getFromServer()
1526 1526
1527 1527
1528 1528 class JRODataWriter(JRODataIO):
1529 1529
1530 1530 """
1531 1531 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1532 1532 de los datos siempre se realiza por bloques.
1533 1533 """
1534 1534
1535 1535 blockIndex = 0
1536 1536
1537 1537 path = None
1538 1538
1539 1539 setFile = None
1540 1540
1541 1541 profilesPerBlock = None
1542 1542
1543 1543 blocksPerFile = None
1544 1544
1545 1545 nWriteBlocks = 0
1546 1546
1547 1547 fileDate = None
1548 1548
1549 1549 def __init__(self, dataOut=None):
1550 1550 raise NotImplementedError
1551 1551
1552 1552 def hasAllDataInBuffer(self):
1553 1553 raise NotImplementedError
1554 1554
1555 1555 def setBlockDimension(self):
1556 1556 raise NotImplementedError
1557 1557
1558 1558 def writeBlock(self):
1559 1559 raise NotImplementedError
1560 1560
1561 1561 def putData(self):
1562 1562 raise NotImplementedError
1563 1563
1564 1564 def getProcessFlags(self):
1565 1565
1566 1566 processFlags = 0
1567 1567
1568 1568 dtype_index = get_dtype_index(self.dtype)
1569 1569 procflag_dtype = get_procflag_dtype(dtype_index)
1570 1570
1571 1571 processFlags += procflag_dtype
1572 1572
1573 1573 if self.dataOut.flagDecodeData:
1574 1574 processFlags += PROCFLAG.DECODE_DATA
1575 1575
1576 1576 if self.dataOut.flagDeflipData:
1577 1577 processFlags += PROCFLAG.DEFLIP_DATA
1578 1578
1579 1579 if self.dataOut.code is not None:
1580 1580 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1581 1581
1582 1582 if self.dataOut.nCohInt > 1:
1583 1583 processFlags += PROCFLAG.COHERENT_INTEGRATION
1584 1584
1585 1585 if self.dataOut.type == "Spectra":
1586 1586 if self.dataOut.nIncohInt > 1:
1587 1587 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1588 1588
1589 1589 if self.dataOut.data_dc is not None:
1590 1590 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1591 1591
1592 1592 if self.dataOut.flagShiftFFT:
1593 1593 processFlags += PROCFLAG.SHIFT_FFT_DATA
1594 1594
1595 1595 return processFlags
1596 1596
1597 1597 def setBasicHeader(self):
1598 1598
1599 1599 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1600 1600 self.basicHeaderObj.version = self.versionFile
1601 1601 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1602
1602 log.warning(datetime.datetime.fromtimestamp(self.dataOut.utctime))
1603 1603 utc = numpy.floor(self.dataOut.utctime)
1604 1604 milisecond = (self.dataOut.utctime - utc) * 1000.0
1605
1605 log.warning(milisecond)
1606 1606 self.basicHeaderObj.utc = utc
1607 1607 self.basicHeaderObj.miliSecond = milisecond
1608 1608 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1609 1609 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1610 1610 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1611 1611
1612 1612 def setFirstHeader(self):
1613 1613 """
1614 1614 Obtiene una copia del First Header
1615 1615
1616 1616 Affected:
1617 1617
1618 1618 self.basicHeaderObj
1619 1619 self.systemHeaderObj
1620 1620 self.radarControllerHeaderObj
1621 1621 self.processingHeaderObj self.
1622 1622
1623 1623 Return:
1624 1624 None
1625 1625 """
1626 1626
1627 1627 raise NotImplementedError
1628 1628
1629 1629 def __writeFirstHeader(self):
1630 1630 """
1631 1631 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1632 1632
1633 1633 Affected:
1634 1634 __dataType
1635 1635
1636 1636 Return:
1637 1637 None
1638 1638 """
1639 1639
1640 1640 # CALCULAR PARAMETROS
1641 1641
1642 1642 sizeLongHeader = self.systemHeaderObj.size + \
1643 1643 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1644 1644 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1645 1645
1646 1646 self.basicHeaderObj.write(self.fp)
1647 1647 self.systemHeaderObj.write(self.fp)
1648 1648 self.radarControllerHeaderObj.write(self.fp)
1649 1649 self.processingHeaderObj.write(self.fp)
1650 1650
1651 1651 def __setNewBlock(self):
1652 1652 """
1653 1653 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1654 1654
1655 1655 Return:
1656 1656 0 : si no pudo escribir nada
1657 1657 1 : Si escribio el Basic el First Header
1658 1658 """
1659 1659 if self.fp == None:
1660 1660 self.setNextFile()
1661 1661
1662 1662 if self.flagIsNewFile:
1663 1663 return 1
1664 1664
1665 1665 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1666 1666 self.basicHeaderObj.write(self.fp)
1667 1667 return 1
1668 1668
1669 1669 if not(self.setNextFile()):
1670 1670 return 0
1671 1671
1672 1672 return 1
1673 1673
1674 1674 def writeNextBlock(self):
1675 1675 """
1676 1676 Selecciona el bloque siguiente de datos y los escribe en un file
1677 1677
1678 1678 Return:
1679 1679 0 : Si no hizo pudo escribir el bloque de datos
1680 1680 1 : Si no pudo escribir el bloque de datos
1681 1681 """
1682 1682 if not(self.__setNewBlock()):
1683 1683 return 0
1684 1684
1685 1685 self.writeBlock()
1686 1686
1687 1687 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1688 1688 self.processingHeaderObj.dataBlocksPerFile))
1689 1689
1690 1690 return 1
1691 1691
1692 1692 def setNextFile(self):
1693 1693 """
1694 1694 Determina el siguiente file que sera escrito
1695 1695
1696 1696 Affected:
1697 1697 self.filename
1698 1698 self.subfolder
1699 1699 self.fp
1700 1700 self.setFile
1701 1701 self.flagIsNewFile
1702 1702
1703 1703 Return:
1704 1704 0 : Si el archivo no puede ser escrito
1705 1705 1 : Si el archivo esta listo para ser escrito
1706 1706 """
1707 1707 ext = self.ext
1708 1708 path = self.path
1709 1709
1710 1710 if self.fp != None:
1711 1711 self.fp.close()
1712 1712
1713 1713 timeTuple = time.localtime(self.dataOut.utctime)
1714 1714 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1715 1715
1716 1716 fullpath = os.path.join(path, subfolder)
1717 1717 setFile = self.setFile
1718 1718
1719 1719 if not(os.path.exists(fullpath)):
1720 1720 os.mkdir(fullpath)
1721 1721 setFile = -1 # inicializo mi contador de seteo
1722 1722 else:
1723 1723 filesList = os.listdir(fullpath)
1724 1724 if len(filesList) > 0:
1725 1725 filesList = sorted(filesList, key=str.lower)
1726 1726 filen = filesList[-1]
1727 1727 # el filename debera tener el siguiente formato
1728 1728 # 0 1234 567 89A BCDE (hex)
1729 1729 # x YYYY DDD SSS .ext
1730 1730 if isNumber(filen[8:11]):
1731 1731 # inicializo mi contador de seteo al seteo del ultimo file
1732 1732 setFile = int(filen[8:11])
1733 1733 else:
1734 1734 setFile = -1
1735 1735 else:
1736 1736 setFile = -1 # inicializo mi contador de seteo
1737 1737
1738 1738 setFile += 1
1739 1739
1740 1740 # If this is a new day it resets some values
1741 1741 if self.dataOut.datatime.date() > self.fileDate:
1742 1742 setFile = 0
1743 1743 self.nTotalBlocks = 0
1744 1744
1745 1745 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1746 1746 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1747 1747
1748 1748 filename = os.path.join(path, subfolder, filen)
1749 1749
1750 1750 fp = open(filename, 'wb')
1751 1751
1752 1752 self.blockIndex = 0
1753 1753
1754 1754 # guardando atributos
1755 1755 self.filename = filename
1756 1756 self.subfolder = subfolder
1757 1757 self.fp = fp
1758 1758 self.setFile = setFile
1759 1759 self.flagIsNewFile = 1
1760 1760 self.fileDate = self.dataOut.datatime.date()
1761 1761
1762 1762 self.setFirstHeader()
1763 1763
1764 1764 print('[Writing] Opening file: %s' % self.filename)
1765 1765
1766 1766 self.__writeFirstHeader()
1767 1767
1768 1768 return 1
1769 1769
1770 1770 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1771 1771 """
1772 1772 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1773 1773
1774 1774 Inputs:
1775 1775 path : directory where data will be saved
1776 1776 profilesPerBlock : number of profiles per block
1777 1777 set : initial file set
1778 1778 datatype : An integer number that defines data type:
1779 1779 0 : int8 (1 byte)
1780 1780 1 : int16 (2 bytes)
1781 1781 2 : int32 (4 bytes)
1782 1782 3 : int64 (8 bytes)
1783 1783 4 : float32 (4 bytes)
1784 1784 5 : double64 (8 bytes)
1785 1785
1786 1786 Return:
1787 1787 0 : Si no realizo un buen seteo
1788 1788 1 : Si realizo un buen seteo
1789 1789 """
1790 1790
1791 1791 if ext == None:
1792 1792 ext = self.ext
1793 1793
1794 1794 self.ext = ext.lower()
1795 1795
1796 1796 self.path = path
1797 1797
1798 1798 if set is None:
1799 1799 self.setFile = -1
1800 1800 else:
1801 1801 self.setFile = set - 1
1802 1802
1803 1803 self.blocksPerFile = blocksPerFile
1804 1804
1805 1805 self.profilesPerBlock = profilesPerBlock
1806 1806
1807 1807 self.dataOut = dataOut
1808 1808 self.fileDate = self.dataOut.datatime.date()
1809 1809 # By default
1810 1810 self.dtype = self.dataOut.dtype
1811 1811
1812 1812 if datatype is not None:
1813 1813 self.dtype = get_numpy_dtype(datatype)
1814 1814
1815 1815 if not(self.setNextFile()):
1816 1816 print("[Writing] There isn't a next file")
1817 1817 return 0
1818 1818
1819 1819 self.setBlockDimension()
1820 1820
1821 1821 return 1
1822 1822
1823 1823 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1824 1824
1825 1825 if not(self.isConfig):
1826 1826
1827 1827 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1828 1828 set=set, ext=ext, datatype=datatype, **kwargs)
1829 1829 self.isConfig = True
1830 1830
1831 1831 self.dataOut = dataOut
1832 1832 self.putData()
1833 1833 return self.dataOut No newline at end of file
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (657 lines changed) Show them Hide them
@@ -1,1037 +1,1544
1 1 import numpy
2 2 import time
3 3 import os
4 4 import h5py
5 5 import re
6 6 import datetime
7 7
8 8 from schainpy.model.data.jrodata import *
9 9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 # from .jroIO_base import *
11 10 from schainpy.model.io.jroIO_base import *
12 import schainpy
13 11 from schainpy.utils import log
14 12
15 13 @MPDecorator
16 14 class ParamReader(JRODataReader,ProcessingUnit):
17 15 '''
18 16 Reads HDF5 format files
19
20 17 path
21
22 18 startDate
23
24 19 endDate
25
26 20 startTime
27
28 21 endTime
29 22 '''
30 23
31 24 ext = ".hdf5"
32
33 25 optchar = "D"
34
35 26 timezone = None
36
37 27 startTime = None
38
39 28 endTime = None
40
41 29 fileIndex = None
42
43 30 utcList = None #To select data in the utctime list
44
45 31 blockList = None #List to blocks to be read from the file
46
47 32 blocksPerFile = None #Number of blocks to be read
48
49 33 blockIndex = None
50
51 34 path = None
52
53 35 #List of Files
54
55 36 filenameList = None
56
57 37 datetimeList = None
58
59 38 #Hdf5 File
60
61 39 listMetaname = None
62
63 40 listMeta = None
64
65 41 listDataname = None
66
67 42 listData = None
68
69 43 listShapes = None
70
71 44 fp = None
72
73 45 #dataOut reconstruction
74
75 46 dataOut = None
76 47
77
78 48 def __init__(self):#, **kwargs):
79 49 ProcessingUnit.__init__(self) #, **kwargs)
80 50 self.dataOut = Parameters()
81 51 return
82 52
83 53 def setup(self, **kwargs):
84 54
85 55 path = kwargs['path']
86 56 startDate = kwargs['startDate']
87 57 endDate = kwargs['endDate']
88 58 startTime = kwargs['startTime']
89 59 endTime = kwargs['endTime']
90 60 walk = kwargs['walk']
91 61 if 'ext' in kwargs:
92 62 ext = kwargs['ext']
93 63 else:
94 64 ext = '.hdf5'
95 65 if 'timezone' in kwargs:
96 66 self.timezone = kwargs['timezone']
97 67 else:
98 68 self.timezone = 'lt'
99 69
100 70 print("[Reading] Searching files in offline mode ...")
101 71 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
102 72 startTime=startTime, endTime=endTime,
103 73 ext=ext, walk=walk)
104 74
105 75 if not(filenameList):
106 76 print("There is no files into the folder: %s"%(path))
107 77 sys.exit(-1)
108 78
109 79 self.fileIndex = -1
110 80 self.startTime = startTime
111 81 self.endTime = endTime
112 82
113 83 self.__readMetadata()
114 84
115 85 self.__setNextFileOffline()
116 86
117 87 return
118 88
119 89 def searchFilesOffLine(self,
120 90 path,
121 91 startDate=None,
122 92 endDate=None,
123 93 startTime=datetime.time(0,0,0),
124 94 endTime=datetime.time(23,59,59),
125 95 ext='.hdf5',
126 96 walk=True):
127 97
128 98 expLabel = ''
129 99 self.filenameList = []
130 100 self.datetimeList = []
131 101
132 102 pathList = []
133 103
134 104 JRODataObj = JRODataReader()
135 105 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
136 106
137 107 if dateList == []:
138 108 print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
139 109 datetime.datetime.combine(startDate,startTime).ctime(),
140 110 datetime.datetime.combine(endDate,endTime).ctime()))
141 111
142 112 return None, None
143 113
144 114 if len(dateList) > 1:
145 115 print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
146 116 else:
147 117 print("[Reading] data was found for the date %s" %(dateList[0]))
148 118
149 119 filenameList = []
150 120 datetimeList = []
151 121
152 122 #----------------------------------------------------------------------------------
153 123
154 124 for thisPath in pathList:
155 # thisPath = pathList[pathDict[file]]
156 125
157 126 fileList = glob.glob1(thisPath, "*%s" %ext)
158 127 fileList.sort()
159 128
160 129 for file in fileList:
161 130
162 131 filename = os.path.join(thisPath,file)
163 132
164 133 if not isFileInDateRange(filename, startDate, endDate):
165 134 continue
166 135
167 136 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
168 137
169 138 if not(thisDatetime):
170 139 continue
171 140
172 141 filenameList.append(filename)
173 142 datetimeList.append(thisDatetime)
174 143
175 144 if not(filenameList):
176 145 print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
177 146 return None, None
178 147
179 148 print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
180 149 print()
181 150
182 151 self.filenameList = filenameList
183 152 self.datetimeList = datetimeList
184 153
185 154 return pathList, filenameList
186 155
187 156 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
188 157
189 158 """
190 159 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
191 160
192 161 Inputs:
193 162 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
194
195 163 startDate : fecha inicial del rango seleccionado en formato datetime.date
196
197 164 endDate : fecha final del rango seleccionado en formato datetime.date
198
199 165 startTime : tiempo inicial del rango seleccionado en formato datetime.time
200
201 166 endTime : tiempo final del rango seleccionado en formato datetime.time
202 167
203 168 Return:
204 169 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
205 170 fecha especificado, de lo contrario retorna False.
206 171
207 172 Excepciones:
208 173 Si el archivo no existe o no puede ser abierto
209 174 Si la cabecera no puede ser leida.
210 175
211 176 """
212 177
213 178 try:
214 179 fp = h5py.File(filename,'r')
215 180 grp1 = fp['Data']
216 181
217 182 except IOError:
218 183 traceback.print_exc()
219 184 raise IOError("The file %s can't be opened" %(filename))
220 #chino rata
185
221 186 #In case has utctime attribute
222 187 grp2 = grp1['utctime']
223 188 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
224 189 thisUtcTime = grp2.value[0]
225 190
226 191 fp.close()
227 192
228 193 if self.timezone == 'lt':
229 194 thisUtcTime -= 5*3600
230 195
231 196 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
232 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
233 197 thisDate = thisDatetime.date()
234 198 thisTime = thisDatetime.time()
235 199
236 200 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
237 201 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
238 202
239 203 #General case
240 204 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
241 205 #-----------o----------------------------o-----------
242 206 # startTime endTime
243 207
244 208 if endTime >= startTime:
245 209 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
246 210 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
247 211 return thisDatetime
248 212 return None
249 213
250 214 #If endTime < startTime then endTime belongs to the next day
251 215 #<<<<<<<<<<<o o>>>>>>>>>>>
252 216 #-----------o----------------------------o-----------
253 217 # endTime startTime
254 218
255 219 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
256 220 return None
257 221
258 222 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
259 223 return None
260 224
261 225 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
262 226 return None
263 227
264 228 return thisDatetime
265 229
266 230 def __setNextFileOffline(self):
267 231
268 232 self.fileIndex += 1
269 233 idFile = self.fileIndex
270 234
271 235 if not(idFile < len(self.filenameList)):
272 print("No more Files")
236 self.dataOut.error = "No more Files"
273 237 return 0
274 238
275 239 filename = self.filenameList[idFile]
276
277 240 filePointer = h5py.File(filename,'r')
278
279 241 self.filename = filename
280
281 242 self.fp = filePointer
282 243
283 244 print("Setting the file: %s"%self.filename)
284 245
285 # self.__readMetadata()
286 246 self.__setBlockList()
287 247 self.__readData()
288 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
289 # self.nRecords = self.fp['Data'].attrs['nRecords']
290 248 self.blockIndex = 0
291 249 return 1
292 250
293 251 def __setBlockList(self):
294 252 '''
295 253 Selects the data within the times defined
296 254
297 255 self.fp
298 256 self.startTime
299 257 self.endTime
300 258
301 259 self.blockList
302 260 self.blocksPerFile
303 261
304 262 '''
305 263 fp = self.fp
306 264 startTime = self.startTime
307 265 endTime = self.endTime
308 266
309 267 grp = fp['Data']
310 268 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
311 269
312 270 #ERROOOOR
313 271 if self.timezone == 'lt':
314 272 thisUtcTime -= 5*3600
315 273
316 274 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
317 275
318 276 thisDate = thisDatetime.date()
319 277 thisTime = thisDatetime.time()
320 278
321 279 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
322 280 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
323 281
324 282 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
325 283
326 284 self.blockList = ind
327 285 self.blocksPerFile = len(ind)
328 286
329 287 return
330 288
331 289 def __readMetadata(self):
332 290 '''
333 291 Reads Metadata
334 292
335 293 self.pathMeta
336
337 294 self.listShapes
338 295 self.listMetaname
339 296 self.listMeta
340 297
341 298 '''
342 299
343 # grp = self.fp['Data']
344 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
345 #
346 # if pathMeta == self.pathMeta:
347 # return
348 # else:
349 # self.pathMeta = pathMeta
350 #
351 # filePointer = h5py.File(self.pathMeta,'r')
352 # groupPointer = filePointer['Metadata']
353
354 300 filename = self.filenameList[0]
355
356 301 fp = h5py.File(filename,'r')
357
358 302 gp = fp['Metadata']
359 303
360 304 listMetaname = []
361 305 listMetadata = []
362 306 for item in list(gp.items()):
363 307 name = item[0]
364 308
365 309 if name=='array dimensions':
366 310 table = gp[name][:]
367 311 listShapes = {}
368 312 for shapes in table:
369 313 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
370 314 else:
371 315 data = gp[name].value
372 316 listMetaname.append(name)
373 317 listMetadata.append(data)
374 318
375 # if name=='type':
376 # self.__initDataOut(data)
377
378 319 self.listShapes = listShapes
379 320 self.listMetaname = listMetaname
380 321 self.listMeta = listMetadata
381 322
382 323 fp.close()
383 324 return
384 325
385 326 def __readData(self):
386 327 grp = self.fp['Data']
387 328 listdataname = []
388 329 listdata = []
389 330
390 331 for item in list(grp.items()):
391 332 name = item[0]
392 333 listdataname.append(name)
393 334
394 335 array = self.__setDataArray(grp[name],self.listShapes[name])
395 336 listdata.append(array)
396 337
397 338 self.listDataname = listdataname
398 339 self.listData = listdata
399 340 return
400 341
401 342 def __setDataArray(self, dataset, shapes):
402 343
403 344 nDims = shapes[0]
404
405 345 nDim2 = shapes[1] #Dimension 0
406
407 346 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
408
409 347 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
410
411 348 mode = shapes[4] #Mode of storing
412
413 349 blockList = self.blockList
414
415 350 blocksPerFile = self.blocksPerFile
416 351
417 352 #Depending on what mode the data was stored
418 353 if mode == 0: #Divided in channels
419 354 arrayData = dataset.value.astype(numpy.float)[0][blockList]
420 355 if mode == 1: #Divided in parameter
421 356 strds = 'table'
422 357 nDatas = nDim1
423 358 newShapes = (blocksPerFile,nDim2,nDim0)
424 359 elif mode==2: #Concatenated in a table
425 360 strds = 'table0'
426 361 arrayData = dataset[strds].value
427 362 #Selecting part of the dataset
428 363 utctime = arrayData[:,0]
429 364 u, indices = numpy.unique(utctime, return_index=True)
430 365
431 366 if blockList.size != indices.size:
432 367 indMin = indices[blockList[0]]
433 368 if blockList[1] + 1 >= indices.size:
434 369 arrayData = arrayData[indMin:,:]
435 370 else:
436 371 indMax = indices[blockList[1] + 1]
437 372 arrayData = arrayData[indMin:indMax,:]
438 373 return arrayData
439 374
440 375 # One dimension
441 376 if nDims == 0:
442 377 arrayData = dataset.value.astype(numpy.float)[0][blockList]
443 378
444 379 # Two dimensions
445 380 elif nDims == 2:
446 381 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
447 382 newShapes = (blocksPerFile,nDim0)
448 383 nDatas = nDim1
449 384
450 385 for i in range(nDatas):
451 386 data = dataset[strds + str(i)].value
452 387 arrayData[:,i,:] = data[blockList,:]
453 388
454 389 # Three dimensions
455 390 else:
456 391 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
457 392 for i in range(nDatas):
458 393
459 394 data = dataset[strds + str(i)].value
460 395
461 396 for b in range(blockList.size):
462 397 arrayData[b,:,i,:] = data[:,:,blockList[b]]
463 398
464 399 return arrayData
465 400
466 401 def __setDataOut(self):
467 402 listMeta = self.listMeta
468 403 listMetaname = self.listMetaname
469 404 listDataname = self.listDataname
470 405 listData = self.listData
471 406 listShapes = self.listShapes
472 407
473 408 blockIndex = self.blockIndex
474 409 # blockList = self.blockList
475 410
476 411 for i in range(len(listMeta)):
477 412 setattr(self.dataOut,listMetaname[i],listMeta[i])
478 413
479 414 for j in range(len(listData)):
480 415 nShapes = listShapes[listDataname[j]][0]
481 416 mode = listShapes[listDataname[j]][4]
482 417 if nShapes == 1:
483 418 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
484 419 elif nShapes > 1:
485 420 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
486 421 elif mode==0:
487 422 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
488 423 #Mode Meteors
489 424 elif mode ==2:
490 425 selectedData = self.__selectDataMode2(listData[j], blockIndex)
491 426 setattr(self.dataOut, listDataname[j], selectedData)
492 427 return
493 428
494 429 def __selectDataMode2(self, data, blockIndex):
495 430 utctime = data[:,0]
496 431 aux, indices = numpy.unique(utctime, return_inverse=True)
497 432 selInd = numpy.where(indices == blockIndex)[0]
498 433 selData = data[selInd,:]
499 434
500 435 return selData
501 436
502 437 def getData(self):
503 438
504 439 if self.blockIndex==self.blocksPerFile:
505 440 if not( self.__setNextFileOffline() ):
506 441 self.dataOut.flagNoData = True
507 442 return 0
508 443
509 444 self.__setDataOut()
510 445 self.dataOut.flagNoData = False
511 446
512 447 self.blockIndex += 1
513 448
514 449 return
515 450
516 451 def run(self, **kwargs):
517 452
518 453 if not(self.isConfig):
519 454 self.setup(**kwargs)
520 # self.setObjProperties()
521 455 self.isConfig = True
522 456
523 457 self.getData()
524 458
525 459 return
526 460
527 461 @MPDecorator
528 462 class ParamWriter(Operation):
529 463 '''
530 464 HDF5 Writer, stores parameters data in HDF5 format files
531 465
532 466 path: path where the files will be stored
533
534 467 blocksPerFile: number of blocks that will be saved in per HDF5 format file
535
536 468 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
537
538 469 metadataList: list of attributes that will be stored as metadata
539
540 470 dataList: list of attributes that will be stores as data
541
542 471 '''
543 472
544
545 473 ext = ".hdf5"
546 474 optchar = "D"
547 475 metaoptchar = "M"
548 476 metaFile = None
549 477 filename = None
550 478 path = None
551 479 setFile = None
552 480 fp = None
553 481 grp = None
554 482 ds = None
555 483 firsttime = True
556 484 #Configurations
557 485 blocksPerFile = None
558 486 blockIndex = None
559 487 dataOut = None
560 488 #Data Arrays
561 489 dataList = None
562 490 metadataList = None
563 491 dsList = None #List of dictionaries with dataset properties
564 492 tableDim = None
565 493 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
566 494 currentDay = None
567 495 lastTime = None
568 496 setType = None
569 497
570 498 def __init__(self):
571 499
572 500 Operation.__init__(self)
573 501 return
574 502
575 503 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
576 504 self.path = path
577 505 self.blocksPerFile = blocksPerFile
578 506 self.metadataList = metadataList
579 507 self.dataList = dataList
580 508 self.dataOut = dataOut
581 509 self.mode = mode
582 510 if self.mode is not None:
583 511 self.mode = numpy.zeros(len(self.dataList)) + mode
584 512 else:
585 513 self.mode = numpy.ones(len(self.dataList))
586 514
587 515 self.setType = setType
588 516
589 517 arrayDim = numpy.zeros((len(self.dataList),5))
590 518
591 519 #Table dimensions
592 520 dtype0 = self.dtype
593 521 tableList = []
594 522
595 523 #Dictionary and list of tables
596 524 dsList = []
597 525
598 526 for i in range(len(self.dataList)):
599 527 dsDict = {}
600 528 dataAux = getattr(self.dataOut, self.dataList[i])
601 529 dsDict['variable'] = self.dataList[i]
602 530 #--------------------- Conditionals ------------------------
603 531 #There is no data
604 532
605 533 if dataAux is None:
606 534
607 535 return 0
608 536
609 537 if isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
610 538 dsDict['mode'] = 0
611 539 dsDict['nDim'] = 0
612 540 arrayDim[i,0] = 0
613 541 dsList.append(dsDict)
614 542
615 543 #Mode 2: meteors
616 544 elif self.mode[i] == 2:
617 545 dsDict['dsName'] = 'table0'
618 546 dsDict['mode'] = 2 # Mode meteors
619 547 dsDict['shape'] = dataAux.shape[-1]
620 548 dsDict['nDim'] = 0
621 549 dsDict['dsNumber'] = 1
622 550 arrayDim[i,3] = dataAux.shape[-1]
623 551 arrayDim[i,4] = self.mode[i] #Mode the data was stored
624 552 dsList.append(dsDict)
625 553
626 554 #Mode 1
627 555 else:
628 556 arrayDim0 = dataAux.shape #Data dimensions
629 557 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
630 558 arrayDim[i,4] = self.mode[i] #Mode the data was stored
631 559 strtable = 'table'
632 560 dsDict['mode'] = 1 # Mode parameters
633 561
634 562 # Three-dimension arrays
635 563 if len(arrayDim0) == 3:
636 564 arrayDim[i,1:-1] = numpy.array(arrayDim0)
637 565 nTables = int(arrayDim[i,2])
638 566 dsDict['dsNumber'] = nTables
639 567 dsDict['shape'] = arrayDim[i,2:4]
640 568 dsDict['nDim'] = 3
641 569
642 570 for j in range(nTables):
643 571 dsDict = dsDict.copy()
644 572 dsDict['dsName'] = strtable + str(j)
645 573 dsList.append(dsDict)
646 574
647 575 # Two-dimension arrays
648 576 elif len(arrayDim0) == 2:
649 577 arrayDim[i,2:-1] = numpy.array(arrayDim0)
650 578 nTables = int(arrayDim[i,2])
651 579 dsDict['dsNumber'] = nTables
652 580 dsDict['shape'] = arrayDim[i,3]
653 581 dsDict['nDim'] = 2
654 582
655 583 for j in range(nTables):
656 584 dsDict = dsDict.copy()
657 585 dsDict['dsName'] = strtable + str(j)
658 586 dsList.append(dsDict)
659 587
660 588 # One-dimension arrays
661 589 elif len(arrayDim0) == 1:
662 590 arrayDim[i,3] = arrayDim0[0]
663 591 dsDict['shape'] = arrayDim0[0]
664 592 dsDict['dsNumber'] = 1
665 593 dsDict['dsName'] = strtable + str(0)
666 594 dsDict['nDim'] = 1
667 595 dsList.append(dsDict)
668 596
669 597 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
670 598 tableList.append(table)
671 599
672 600 self.dsList = dsList
673 601 self.tableDim = numpy.array(tableList, dtype = dtype0)
674 602 self.blockIndex = 0
675 603 timeTuple = time.localtime(dataOut.utctime)
676 604 self.currentDay = timeTuple.tm_yday
677 605
678 606 def putMetadata(self):
679 607
680 608 fp = self.createMetadataFile()
681 609 self.writeMetadata(fp)
682 610 fp.close()
683 611 return
684 612
685 613 def createMetadataFile(self):
686 614 ext = self.ext
687 615 path = self.path
688 616 setFile = self.setFile
689 617
690 618 timeTuple = time.localtime(self.dataOut.utctime)
691 619
692 620 subfolder = ''
693 621 fullpath = os.path.join( path, subfolder )
694 622
695 623 if not( os.path.exists(fullpath) ):
696 624 os.mkdir(fullpath)
697 625 setFile = -1 #inicializo mi contador de seteo
698 626
699 627 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
700 628 fullpath = os.path.join( path, subfolder )
701 629
702 630 if not( os.path.exists(fullpath) ):
703 631 os.mkdir(fullpath)
704 632 setFile = -1 #inicializo mi contador de seteo
705 633
706 634 else:
707 635 filesList = os.listdir( fullpath )
708 636 filesList = sorted( filesList, key=str.lower )
709 637 if len( filesList ) > 0:
710 638 filesList = [k for k in filesList if k.startswith(self.metaoptchar)]
711 639 filen = filesList[-1]
712 640 # el filename debera tener el siguiente formato
713 641 # 0 1234 567 89A BCDE (hex)
714 642 # x YYYY DDD SSS .ext
715 643 if isNumber( filen[8:11] ):
716 644 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
717 645 else:
718 646 setFile = -1
719 647 else:
720 648 setFile = -1 #inicializo mi contador de seteo
721 649
722 650 if self.setType is None:
723 651 setFile += 1
724 652 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
725 653 timeTuple.tm_year,
726 654 timeTuple.tm_yday,
727 655 setFile,
728 656 ext )
729 657 else:
730 658 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
731 659 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
732 660 timeTuple.tm_year,
733 661 timeTuple.tm_yday,
734 662 setFile,
735 663 ext )
736 664
737 665 filename = os.path.join( path, subfolder, file )
738 666 self.metaFile = file
739 667 #Setting HDF5 File
740 668 fp = h5py.File(filename,'w')
741 669
742 670 return fp
743 671
744 672 def writeMetadata(self, fp):
745 673
746 674 grp = fp.create_group("Metadata")
747 675 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
748 676
749 677 for i in range(len(self.metadataList)):
750 678 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
751 679 return
752 680
753 681 def timeFlag(self):
754 682 currentTime = self.dataOut.utctime
755 683
756 684 if self.lastTime is None:
757 685 self.lastTime = currentTime
758 686
759 687 #Day
760 688 timeTuple = time.localtime(currentTime)
761 689 dataDay = timeTuple.tm_yday
762 690
763 691 #Time
764 692 timeDiff = currentTime - self.lastTime
765 693
766 694 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
767 695 if dataDay != self.currentDay:
768 696 self.currentDay = dataDay
769 697 return True
770 698 elif timeDiff > 3*60*60:
771 699 self.lastTime = currentTime
772 700 return True
773 701 else:
774 702 self.lastTime = currentTime
775 703 return False
776 704
777 705 def setNextFile(self):
778 706
779 707 ext = self.ext
780 708 path = self.path
781 709 setFile = self.setFile
782 710 mode = self.mode
783 711
784 712 timeTuple = time.localtime(self.dataOut.utctime)
785 713 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
786 714
787 715 fullpath = os.path.join( path, subfolder )
788 716
789 717 if os.path.exists(fullpath):
790 718 filesList = os.listdir( fullpath )
791 719 filesList = [k for k in filesList if 'M' in k]
792 720 if len( filesList ) > 0:
793 721 filesList = sorted( filesList, key=str.lower )
794 722 filen = filesList[-1]
795 723 # el filename debera tener el siguiente formato
796 724 # 0 1234 567 89A BCDE (hex)
797 725 # x YYYY DDD SSS .ext
798 726 if isNumber( filen[8:11] ):
799 727 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
800 728 else:
801 729 setFile = -1
802 730 else:
803 731 setFile = -1 #inicializo mi contador de seteo
804 732 else:
805 733 os.makedirs(fullpath)
806 734 setFile = -1 #inicializo mi contador de seteo
807 735
808 736 if self.setType is None:
809 737 setFile += 1
810 738 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
811 739 timeTuple.tm_year,
812 740 timeTuple.tm_yday,
813 741 setFile,
814 742 ext )
815 743 else:
816 744 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
817 745 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
818 746 timeTuple.tm_year,
819 747 timeTuple.tm_yday,
820 748 setFile,
821 749 ext )
822 750
823 751 filename = os.path.join( path, subfolder, file )
824 752
825 753 #Setting HDF5 File
826 754 fp = h5py.File(filename,'w')
827 755 #write metadata
828 756 self.writeMetadata(fp)
829 757 #Write data
830 758 grp = fp.create_group("Data")
831 759 ds = []
832 760 data = []
833 761 dsList = self.dsList
834 762 i = 0
835 763 while i < len(dsList):
836 764 dsInfo = dsList[i]
837 765 #One-dimension data
838 766 if dsInfo['mode'] == 0:
839 767 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
840 768 ds.append(ds0)
841 769 data.append([])
842 770 i += 1
843 771 continue
844 772
845 773 elif dsInfo['mode'] == 2:
846 774 grp0 = grp.create_group(dsInfo['variable'])
847 775 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
848 776 ds.append(ds0)
849 777 data.append([])
850 778 i += 1
851 779 continue
852 780
853 781 elif dsInfo['mode'] == 1:
854 782 grp0 = grp.create_group(dsInfo['variable'])
855 783
856 784 for j in range(dsInfo['dsNumber']):
857 785 dsInfo = dsList[i]
858 786 tableName = dsInfo['dsName']
859 787
860 788
861 789 if dsInfo['nDim'] == 3:
862 790 shape = dsInfo['shape'].astype(int)
863 791 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
864 792 else:
865 793 shape = int(dsInfo['shape'])
866 794 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
867 795
868 796 ds.append(ds0)
869 797 data.append([])
870 798 i += 1
871 799
872 800 fp.flush()
873 801 fp.close()
874 802
875 803 log.log('creating file: {}'.format(filename), 'Writing')
876 804 self.filename = filename
877 805 self.ds = ds
878 806 self.data = data
879 807 self.firsttime = True
880 808 self.blockIndex = 0
881 809 return
882 810
883 811 def putData(self):
884 812
885 813 if self.blockIndex == self.blocksPerFile or self.timeFlag():
886 814 self.setNextFile()
887 815
888 816 self.readBlock()
889 817 self.setBlock() #Prepare data to be written
890 818 self.writeBlock() #Write data
891 819
892 820 return
893 821
894 822 def readBlock(self):
895 823
896 824 '''
897 825 data Array configured
898 826
899 827
900 828 self.data
901 829 '''
902 830 dsList = self.dsList
903 831 ds = self.ds
904 832 #Setting HDF5 File
905 833 fp = h5py.File(self.filename,'r+')
906 834 grp = fp["Data"]
907 835 ind = 0
908 836
909 837 while ind < len(dsList):
910 838 dsInfo = dsList[ind]
911 839
912 840 if dsInfo['mode'] == 0:
913 841 ds0 = grp[dsInfo['variable']]
914 842 ds[ind] = ds0
915 843 ind += 1
916 844 else:
917 845
918 846 grp0 = grp[dsInfo['variable']]
919 847
920 848 for j in range(dsInfo['dsNumber']):
921 849 dsInfo = dsList[ind]
922 850 ds0 = grp0[dsInfo['dsName']]
923 851 ds[ind] = ds0
924 852 ind += 1
925 853
926 854 self.fp = fp
927 855 self.grp = grp
928 856 self.ds = ds
929 857
930 858 return
931 859
932 860 def setBlock(self):
933 861 '''
934 862 data Array configured
935 863
936 864
937 865 self.data
938 866 '''
939 867 #Creating Arrays
940 868 dsList = self.dsList
941 869 data = self.data
942 870 ind = 0
943 871
944 872 while ind < len(dsList):
945 873 dsInfo = dsList[ind]
946 874 dataAux = getattr(self.dataOut, dsInfo['variable'])
947 875
948 876 mode = dsInfo['mode']
949 877 nDim = dsInfo['nDim']
950 878
951 879 if mode == 0 or mode == 2 or nDim == 1:
952 880 data[ind] = dataAux
953 881 ind += 1
954 882 # elif nDim == 1:
955 883 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
956 884 # ind += 1
957 885 elif nDim == 2:
958 886 for j in range(dsInfo['dsNumber']):
959 887 data[ind] = dataAux[j,:]
960 888 ind += 1
961 889 elif nDim == 3:
962 890 for j in range(dsInfo['dsNumber']):
963 891 data[ind] = dataAux[:,j,:]
964 892 ind += 1
965 893
966 894 self.data = data
967 895 return
968 896
969 897 def writeBlock(self):
970 898 '''
971 899 Saves the block in the HDF5 file
972 900 '''
973 901 dsList = self.dsList
974 902
975 903 for i in range(len(self.ds)):
976 904 dsInfo = dsList[i]
977 905 nDim = dsInfo['nDim']
978 906 mode = dsInfo['mode']
979 907
980 908 # First time
981 909 if self.firsttime:
982 910 if type(self.data[i]) == numpy.ndarray:
983 911
984 912 if nDim == 3:
985 913 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
986 914 self.ds[i].resize(self.data[i].shape)
987 915 if mode == 2:
988 916 self.ds[i].resize(self.data[i].shape)
989 917 self.ds[i][:] = self.data[i]
990 918 else:
991 919
992 920 # From second time
993 921 # Meteors!
994 922 if mode == 2:
995 923 dataShape = self.data[i].shape
996 924 dsShape = self.ds[i].shape
997 925 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
998 926 self.ds[i][dsShape[0]:,:] = self.data[i]
999 927 # No dimension
1000 928 elif mode == 0:
1001 929 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1002 930 self.ds[i][0,-1] = self.data[i]
1003 931 # One dimension
1004 932 elif nDim == 1:
1005 933 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1006 934 self.ds[i][-1,:] = self.data[i]
1007 935 # Two dimension
1008 936 elif nDim == 2:
1009 937 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1010 938 self.ds[i][self.blockIndex,:] = self.data[i]
1011 939 # Three dimensions
1012 940 elif nDim == 3:
1013 941 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1014 942 self.ds[i][:,:,-1] = self.data[i]
1015 943
1016 944 self.firsttime = False
1017 945 self.blockIndex += 1
1018 946
1019 947 #Close to save changes
1020 948 self.fp.flush()
1021 949 self.fp.close()
1022 950 return
1023 951
1024 952 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
1025 953
1026 954 self.dataOut = dataOut
1027 955 if not(self.isConfig):
1028 956 self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1029 957 metadataList=metadataList, dataList=dataList, mode=mode,
1030 958 setType=setType)
1031 959
1032 960 self.isConfig = True
1033 961 self.setNextFile()
1034 962
1035 963 self.putData()
1036 964 return
1037 No newline at end of file
965
966
967 @MPDecorator
968 class ParameterReader(JRODataReader,ProcessingUnit):
969 '''
970 Reads HDF5 format files
971 '''
972
973 ext = ".hdf5"
974 optchar = "D"
975 timezone = None
976 startTime = None
977 endTime = None
978 fileIndex = None
979 blockList = None #List to blocks to be read from the file
980 blocksPerFile = None #Number of blocks to be read
981 blockIndex = None
982 path = None
983 #List of Files
984 filenameList = None
985 datetimeList = None
986 #Hdf5 File
987 listMetaname = None
988 listMeta = None
989 listDataname = None
990 listData = None
991 listShapes = None
992 fp = None
993 #dataOut reconstruction
994 dataOut = None
995
996 def __init__(self):
997 ProcessingUnit.__init__(self)
998 self.dataOut = Parameters()
999 return
1000
1001 def setup(self, **kwargs):
1002
1003 path = kwargs['path']
1004 startDate = kwargs['startDate']
1005 endDate = kwargs['endDate']
1006 startTime = kwargs['startTime']
1007 endTime = kwargs['endTime']
1008 walk = kwargs['walk']
1009 if 'ext' in kwargs:
1010 ext = kwargs['ext']
1011 else:
1012 ext = '.hdf5'
1013 if 'timezone' in kwargs:
1014 self.timezone = kwargs['timezone']
1015 else:
1016 self.timezone = 'lt'
1017
1018 print("[Reading] Searching files in offline mode ...")
1019 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1020 startTime=startTime, endTime=endTime,
1021 ext=ext, walk=walk)
1022
1023 if not(filenameList):
1024 print("There is no files into the folder: %s"%(path))
1025 sys.exit(-1)
1026
1027 self.fileIndex = -1
1028 self.startTime = startTime
1029 self.endTime = endTime
1030 self.__readMetadata()
1031 self.__setNextFileOffline()
1032
1033 return
1034
1035 def searchFilesOffLine(self, path, startDate=None, endDate=None, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), ext='.hdf5', walk=True):
1036
1037 expLabel = ''
1038 self.filenameList = []
1039 self.datetimeList = []
1040 pathList = []
1041 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
1042
1043 if dateList == []:
1044 print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
1045 datetime.datetime.combine(startDate,startTime).ctime(),
1046 datetime.datetime.combine(endDate,endTime).ctime()))
1047
1048 return None, None
1049
1050 if len(dateList) > 1:
1051 print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
1052 else:
1053 print("[Reading] data was found for the date %s" %(dateList[0]))
1054
1055 filenameList = []
1056 datetimeList = []
1057
1058 for thisPath in pathList:
1059
1060 fileList = glob.glob1(thisPath, "*%s" %ext)
1061 fileList.sort()
1062
1063 for file in fileList:
1064
1065 filename = os.path.join(thisPath,file)
1066
1067 if not isFileInDateRange(filename, startDate, endDate):
1068 continue
1069
1070 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
1071
1072 if not(thisDatetime):
1073 continue
1074
1075 filenameList.append(filename)
1076 datetimeList.append(thisDatetime)
1077
1078 if not(filenameList):
1079 print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
1080 return None, None
1081
1082 print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
1083 print()
1084
1085 self.filenameList = filenameList
1086 self.datetimeList = datetimeList
1087
1088 return pathList, filenameList
1089
1090 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
1091
1092 """
1093 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
1094
1095 Inputs:
1096 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
1097 startDate : fecha inicial del rango seleccionado en formato datetime.date
1098 endDate : fecha final del rango seleccionado en formato datetime.date
1099 startTime : tiempo inicial del rango seleccionado en formato datetime.time
1100 endTime : tiempo final del rango seleccionado en formato datetime.time
1101
1102 Return:
1103 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
1104 fecha especificado, de lo contrario retorna False.
1105
1106 Excepciones:
1107 Si el archivo no existe o no puede ser abierto
1108 Si la cabecera no puede ser leida.
1109
1110 """
1111
1112 try:
1113 fp = h5py.File(filename, 'r')
1114 grp1 = fp['Data']
1115
1116 except IOError:
1117 traceback.print_exc()
1118 raise IOError("The file %s can't be opened" %(filename))
1119 #In case has utctime attribute
1120 grp2 = grp1['utctime']
1121 thisUtcTime = grp2.value[0]
1122
1123 fp.close()
1124
1125 if self.timezone == 'lt':
1126 thisUtcTime -= 5*3600
1127
1128 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime + 5*3600)
1129 thisDate = thisDatetime.date()
1130 thisTime = thisDatetime.time()
1131
1132 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
1133 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
1134
1135 #General case
1136 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
1137 #-----------o----------------------------o-----------
1138 # startTime endTime
1139
1140 if endTime >= startTime:
1141 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
1142 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
1143 return thisDatetime
1144 return None
1145
1146 #If endTime < startTime then endTime belongs to the next day
1147 #<<<<<<<<<<<o o>>>>>>>>>>>
1148 #-----------o----------------------------o-----------
1149 # endTime startTime
1150
1151 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
1152 return None
1153
1154 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
1155 return None
1156
1157 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
1158 return None
1159
1160 return thisDatetime
1161
1162 def __setNextFileOffline(self):
1163
1164 self.fileIndex += 1
1165 idFile = self.fileIndex
1166
1167 if not(idFile < len(self.filenameList)):
1168 self.dataOut.error = 'No more files'
1169 return 0
1170
1171 filename = self.filenameList[idFile]
1172 self.fp = h5py.File(filename, 'r')
1173 self.filename = filename
1174
1175 print("Setting the file: %s"%self.filename)
1176
1177 self.__setBlockList()
1178 self.__readData()
1179 self.blockIndex = 0
1180 return 1
1181
1182 def __setBlockList(self):
1183 '''
1184 Selects the data within the times defined
1185
1186 self.fp
1187 self.startTime
1188 self.endTime
1189 self.blockList
1190 self.blocksPerFile
1191
1192 '''
1193 fp = self.fp
1194 startTime = self.startTime
1195 endTime = self.endTime
1196
1197 grp = fp['Data']
1198 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
1199
1200 if self.timezone == 'lt':
1201 thisUtcTime -= 5*3600
1202
1203 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime + 5*3600)
1204
1205 thisDate = thisDatetime.date()
1206 thisTime = thisDatetime.time()
1207
1208 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
1209 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
1210
1211 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
1212
1213 self.blockList = ind
1214 self.blocksPerFile = len(ind)
1215
1216 return
1217
1218 def __readMetadata(self):
1219 '''
1220 Reads Metadata
1221 '''
1222
1223 filename = self.filenameList[0]
1224 fp = h5py.File(filename, 'r')
1225 gp = fp['Metadata']
1226 listMetaname = []
1227 listMetadata = []
1228
1229 for item in list(gp.items()):
1230 name = item[0]
1231
1232 if name=='variables':
1233 table = gp[name][:]
1234 listShapes = {}
1235 for shapes in table:
1236 listShapes[shapes[0].decode()] = numpy.array([shapes[1]])
1237 else:
1238 data = gp[name].value
1239 listMetaname.append(name)
1240 listMetadata.append(data)
1241
1242 self.listShapes = listShapes
1243 self.listMetaname = listMetaname
1244 self.listMeta = listMetadata
1245
1246 fp.close()
1247 return
1248
1249 def __readData(self):
1250
1251 grp = self.fp['Data']
1252 listdataname = []
1253 listdata = []
1254
1255 for item in list(grp.items()):
1256 name = item[0]
1257 listdataname.append(name)
1258 dim = self.listShapes[name][0]
1259 if dim == 0:
1260 array = grp[name].value
1261 else:
1262 array = []
1263 for i in range(dim):
1264 array.append(grp[name]['table{:02d}'.format(i)].value)
1265 array = numpy.array(array)
1266
1267 listdata.append(array)
1268
1269 self.listDataname = listdataname
1270 self.listData = listdata
1271 return
1272
1273 def getData(self):
1274
1275 for i in range(len(self.listMeta)):
1276 setattr(self.dataOut, self.listMetaname[i], self.listMeta[i])
1277
1278 for j in range(len(self.listData)):
1279 dim = self.listShapes[self.listDataname[j]][0]
1280 if dim == 0:
1281 setattr(self.dataOut, self.listDataname[j], self.listData[j][self.blockIndex])
1282 else:
1283 setattr(self.dataOut, self.listDataname[j], self.listData[j][:,self.blockIndex])
1284
1285 self.dataOut.flagNoData = False
1286 self.blockIndex += 1
1287
1288 return
1289
1290 def run(self, **kwargs):
1291
1292 if not(self.isConfig):
1293 self.setup(**kwargs)
1294 self.isConfig = True
1295
1296 if self.blockIndex == self.blocksPerFile:
1297 if not(self.__setNextFileOffline()):
1298 self.dataOut.flagNoData = True
1299 return 0
1300
1301 self.getData()
1302
1303 return
1304
1305 @MPDecorator
1306 class ParameterWriter(Operation):
1307 '''
1308 HDF5 Writer, stores parameters data in HDF5 format files
1309
1310 path: path where the files will be stored
1311 blocksPerFile: number of blocks that will be saved in per HDF5 format file
1312 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
1313 metadataList: list of attributes that will be stored as metadata
1314 dataList: list of attributes that will be stores as data
1315 '''
1316
1317
1318 ext = ".hdf5"
1319 optchar = "D"
1320 metaoptchar = "M"
1321 metaFile = None
1322 filename = None
1323 path = None
1324 setFile = None
1325 fp = None
1326 grp = None
1327 ds = None
1328 firsttime = True
1329 #Configurations
1330 blocksPerFile = None
1331 blockIndex = None
1332 dataOut = None
1333 #Data Arrays
1334 dataList = None
1335 metadataList = None
1336 dsList = None #List of dictionaries with dataset properties
1337 tableDim = None
1338 dtype = [('name', 'S20'),('nDim', 'i')]
1339 currentDay = None
1340 lastTime = None
1341
1342 def __init__(self):
1343
1344 Operation.__init__(self)
1345 return
1346
1347 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None):
1348 self.path = path
1349 self.blocksPerFile = blocksPerFile
1350 self.metadataList = metadataList
1351 self.dataList = dataList
1352 self.setType = setType
1353
1354 tableList = []
1355 dsList = []
1356
1357 for i in range(len(self.dataList)):
1358 dsDict = {}
1359 dataAux = getattr(self.dataOut, self.dataList[i])
1360 dsDict['variable'] = self.dataList[i]
1361
1362 if dataAux is None:
1363 continue
1364 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
1365 dsDict['nDim'] = 0
1366 else:
1367 dsDict['nDim'] = len(dataAux.shape)
1368 dsDict['shape'] = dataAux.shape
1369 dsDict['dsNumber'] = dataAux.shape[0]
1370
1371 dsList.append(dsDict)
1372 tableList.append((self.dataList[i], dsDict['nDim']))
1373
1374 self.dsList = dsList
1375 self.tableDim = numpy.array(tableList, dtype=self.dtype)
1376 self.currentDay = self.dataOut.datatime.date()
1377
1378 def timeFlag(self):
1379 currentTime = self.dataOut.utctime
1380 timeTuple = time.localtime(currentTime)
1381 dataDay = timeTuple.tm_yday
1382
1383 if self.lastTime is None:
1384 self.lastTime = currentTime
1385 self.currentDay = dataDay
1386 return False
1387
1388 timeDiff = currentTime - self.lastTime
1389
1390 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
1391 if dataDay != self.currentDay:
1392 self.currentDay = dataDay
1393 return True
1394 elif timeDiff > 3*60*60:
1395 self.lastTime = currentTime
1396 return True
1397 else:
1398 self.lastTime = currentTime
1399 return False
1400
1401 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, setType=None):
1402
1403 self.dataOut = dataOut
1404 if not(self.isConfig):
1405 self.setup(path=path, blocksPerFile=blocksPerFile,
1406 metadataList=metadataList, dataList=dataList,
1407 setType=setType)
1408
1409 self.isConfig = True
1410 self.setNextFile()
1411
1412 self.putData()
1413 return
1414
1415 def setNextFile(self):
1416
1417 ext = self.ext
1418 path = self.path
1419 setFile = self.setFile
1420
1421 timeTuple = time.localtime(self.dataOut.utctime)
1422 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1423 fullpath = os.path.join(path, subfolder)
1424
1425 if os.path.exists(fullpath):
1426 filesList = os.listdir(fullpath)
1427 filesList = [k for k in filesList if k.startswith(self.optchar)]
1428 if len( filesList ) > 0:
1429 filesList = sorted(filesList, key=str.lower)
1430 filen = filesList[-1]
1431 # el filename debera tener el siguiente formato
1432 # 0 1234 567 89A BCDE (hex)
1433 # x YYYY DDD SSS .ext
1434 if isNumber(filen[8:11]):
1435 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
1436 else:
1437 setFile = -1
1438 else:
1439 setFile = -1 #inicializo mi contador de seteo
1440 else:
1441 os.makedirs(fullpath)
1442 setFile = -1 #inicializo mi contador de seteo
1443
1444 if self.setType is None:
1445 setFile += 1
1446 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
1447 timeTuple.tm_year,
1448 timeTuple.tm_yday,
1449 setFile,
1450 ext )
1451 else:
1452 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
1453 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
1454 timeTuple.tm_year,
1455 timeTuple.tm_yday,
1456 setFile,
1457 ext )
1458
1459 self.filename = os.path.join( path, subfolder, file )
1460
1461 #Setting HDF5 File
1462 self.fp = h5py.File(self.filename, 'w')
1463 #write metadata
1464 self.writeMetadata(self.fp)
1465 #Write data
1466 self.writeData(self.fp)
1467
1468 def writeMetadata(self, fp):
1469
1470 grp = fp.create_group("Metadata")
1471 grp.create_dataset('variables', data=self.tableDim, dtype=self.dtype)
1472
1473 for i in range(len(self.metadataList)):
1474 if not hasattr(self.dataOut, self.metadataList[i]):
1475 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
1476 continue
1477 value = getattr(self.dataOut, self.metadataList[i])
1478 grp.create_dataset(self.metadataList[i], data=value)
1479 return
1480
1481 def writeData(self, fp):
1482
1483 grp = fp.create_group("Data")
1484 dtsets = []
1485 data = []
1486
1487 for dsInfo in self.dsList:
1488 if dsInfo['nDim'] == 0:
1489 ds = grp.create_dataset(
1490 dsInfo['variable'],
1491 (self.blocksPerFile, ),
1492 chunks=True,
1493 dtype=numpy.float64)
1494 dtsets.append(ds)
1495 data.append((dsInfo['variable'], -1))
1496 else:
1497 sgrp = grp.create_group(dsInfo['variable'])
1498 for i in range(dsInfo['dsNumber']):
1499 ds = sgrp.create_dataset(
1500 'table{:02d}'.format(i),
1501 (self.blocksPerFile, ) + dsInfo['shape'][1:],
1502 chunks=True)
1503 dtsets.append(ds)
1504 data.append((dsInfo['variable'], i))
1505 fp.flush()
1506
1507 log.log('creating file: {}'.format(fp.filename), 'Writing')
1508
1509 self.ds = dtsets
1510 self.data = data
1511 self.firsttime = True
1512 self.blockIndex = 0
1513 return
1514
1515 def putData(self):
1516
1517 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
1518 self.closeFile()
1519 self.setNextFile()
1520
1521 for i, ds in enumerate(self.ds):
1522 attr, ch = self.data[i]
1523 if ch == -1:
1524 ds[self.blockIndex] = getattr(self.dataOut, attr)
1525 else:
1526 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
1527
1528 self.fp.flush()
1529 self.blockIndex += 1
1530
1531 return
1532
1533 def closeFile(self):
1534
1535 if self.blockIndex != self.blocksPerFile:
1536 for ds in self.ds:
1537 ds.resize(self.blockIndex, axis=0)
1538
1539 self.fp.flush()
1540 self.fp.close()
1541
1542 def close(self):
1543
1544 self.closeFile()
General Comments 0
You need to be logged in to leave comments. Login now