| @@ -20,7 +20,7 from schainpy.model.data.jrodata import Parameters | |||
|
|
20 | 20 | from schainpy.utils import log |
|
|
21 | 21 | |
|
|
22 | 22 | try: |
|
|
23 |
import madrigal.cedar |
|
|
|
23 | import madrigal.cedar | |
|
|
24 | 24 | except: |
|
|
25 | 25 | log.warning( |
|
|
26 | 26 | 'You should install "madrigal library" module if you want to read/write Madrigal data' |
| @@ -306,6 +306,8 class MADReader(JRODataReader, ProcessingUnit): | |||
|
|
306 | 306 | dum = [] |
|
|
307 | 307 | if self.ext == '.txt': |
|
|
308 | 308 | dt = self.data[self.counter_records][:6].astype(int) |
|
|
309 | if datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]).date() > self.datatime.date(): | |
|
|
310 | self.flagDiscontinuousBlock = 1 | |
|
|
309 | 311 | self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]) |
|
|
310 | 312 | while True: |
|
|
311 | 313 | dt = self.data[self.counter_records][:6].astype(int) |
| @@ -317,9 +319,7 class MADReader(JRODataReader, ProcessingUnit): | |||
|
|
317 | 319 | self.flagIsNewFile = True |
|
|
318 | 320 | break |
|
|
319 | 321 | continue |
|
|
320 | self.intervals.add((datatime-self.datatime).seconds) | |
|
|
321 | if datatime.date() > self.datatime.date(): | |
|
|
322 | self.flagDiscontinuousBlock = 1 | |
|
|
322 | self.intervals.add((datatime-self.datatime).seconds) | |
|
|
323 | 323 | break |
|
|
324 | 324 | elif self.ext == '.hdf5': |
|
|
325 | 325 | datatime = datetime.datetime.utcfromtimestamp( |
| @@ -506,15 +506,15 class MADWriter(Operation): | |||
|
|
506 | 506 | log.success( |
|
|
507 | 507 | 'Creating file: {}'.format(self.fullname), |
|
|
508 | 508 | 'MADWriter') |
|
|
509 |
self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True) |
|
|
|
509 | self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True) | |
|
|
510 | 510 | except ValueError, e: |
|
|
511 | 511 | log.error( |
|
|
512 | 512 | 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"', |
|
|
513 | 513 | 'MADWriter') |
|
|
514 | 514 | return |
|
|
515 | ||
|
|
516 |
return 1 |
|
|
|
517 | ||
|
|
515 | ||
|
|
516 | return 1 | |
|
|
517 | ||
|
|
518 | 518 | def writeBlock(self): |
|
|
519 | 519 | ''' |
|
|
520 | 520 | Add data records to cedar file taking data from oneDDict and twoDDict |
| @@ -525,30 +525,39 class MADWriter(Operation): | |||
|
|
525 | 525 | startTime = datetime.datetime.fromtimestamp(self.dataOut.utctime) |
|
|
526 | 526 | endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval) |
|
|
527 | 527 | heights = self.dataOut.heightList |
|
|
528 | ||
|
|
528 | ||
|
|
529 | 529 | if self.ext == '.dat': |
|
|
530 | invalid = numpy.isnan(self.dataOut.data_output) | |
|
|
531 | self.dataOut.data_output[invalid] = self.missing | |
|
|
532 | out = {} | |
|
|
530 | for key, value in self.twoDDict.items(): | |
|
|
531 | if isinstance(value, str): | |
|
|
532 | data = getattr(self.dataOut, value) | |
|
|
533 | invalid = numpy.isnan(data) | |
|
|
534 | data[invalid] = self.missing | |
|
|
535 | elif isinstance(value, (tuple, list)): | |
|
|
536 | attr, key = value | |
|
|
537 | data = getattr(self.dataOut, attr) | |
|
|
538 | invalid = numpy.isnan(data) | |
|
|
539 | data[invalid] = self.missing | |
|
|
540 | ||
|
|
541 | out = {} | |
|
|
533 | 542 | for key, value in self.twoDDict.items(): |
|
|
534 | 543 | key = key.lower() |
|
|
535 |
if isinstance(value, str): |
|
|
|
544 | if isinstance(value, str): | |
|
|
536 | 545 | if 'db' in value.lower(): |
|
|
537 | 546 | tmp = getattr(self.dataOut, value.replace('_db', '')) |
|
|
538 |
SNRavg = numpy.average(tmp, axis=0) |
|
|
|
547 | SNRavg = numpy.average(tmp, axis=0) | |
|
|
539 | 548 | tmp = 10*numpy.log10(SNRavg) |
|
|
540 | 549 | else: |
|
|
541 |
tmp = getattr(self.dataOut, value) |
|
|
|
550 | tmp = getattr(self.dataOut, value) | |
|
|
542 | 551 | out[key] = tmp.flatten() |
|
|
543 | 552 | elif isinstance(value, (tuple, list)): |
|
|
544 |
attr, x = value |
|
|
|
553 | attr, x = value | |
|
|
545 | 554 | data = getattr(self.dataOut, attr) |
|
|
546 | 555 | out[key] = data[int(x)] |
|
|
547 | 556 | |
|
|
548 | 557 | a = numpy.array([out[k] for k in self.keys]) |
|
|
549 |
nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))]) |
|
|
|
558 | nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))]) | |
|
|
550 | 559 | index = numpy.where(nrows == False)[0] |
|
|
551 | ||
|
|
560 | ||
|
|
552 | 561 | rec = madrigal.cedar.MadrigalDataRecord( |
|
|
553 | 562 | self.kinst, |
|
|
554 | 563 | self.kindat, |
| @@ -586,7 +595,7 class MADWriter(Operation): | |||
|
|
586 | 595 | self.fp.append(rec) |
|
|
587 | 596 | if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0: |
|
|
588 | 597 | self.fp.dump() |
|
|
589 | if self.counter % 10 == 0 and self.counter > 0: | |
|
|
598 | if self.counter % 100 == 0 and self.counter > 0: | |
|
|
590 | 599 | log.log( |
|
|
591 | 600 | 'Writing {} records'.format( |
|
|
592 | 601 | self.counter), |
| @@ -616,12 +625,12 class MADWriter(Operation): | |||
|
|
616 | 625 | return 0 |
|
|
617 | 626 | |
|
|
618 | 627 | if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks: |
|
|
619 |
if self.counter > 0: |
|
|
|
628 | if self.counter > 0: | |
|
|
620 | 629 | self.setHeader() |
|
|
621 | 630 | self.counter = 0 |
|
|
622 | 631 | |
|
|
623 | 632 | if self.counter == 0: |
|
|
624 |
self.setFile() |
|
|
|
633 | self.setFile() | |
|
|
625 | 634 | |
|
|
626 | 635 | self.writeBlock() |
|
|
627 | 636 | self.counter += 1 |
General Comments 0
You need to be logged in to leave comments.
Login now