schain-jc Commit - r1073:9fc44081e2dc

20

from schainpy.utils import log

20

from schainpy.utils import log

21

22

try:

22

try:

23

import madrigal.cedar

23

import madrigal.cedar

24

except:

24

except:

25

log.warning(

25

log.warning(

26

'You should install "madrigal library" module if you want to read/write Madrigal data'

26

'You should install "madrigal library" module if you want to read/write Madrigal data'

306

dum = []

306

dum = []

307

if self.ext == '.txt':

307

if self.ext == '.txt':

308

dt = self.data[self.counter_records][:6].astype(int)

308

dt = self.data[self.counter_records][:6].astype(int)

309

if datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]).date() > self.datatime.date():

310

self.flagDiscontinuousBlock = 1

309

self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])

311

self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])

310

while True:

312

while True:

311

dt = self.data[self.counter_records][:6].astype(int)

313

dt = self.data[self.counter_records][:6].astype(int)

317

self.flagIsNewFile = True

319

self.flagIsNewFile = True

318

break

320

break

319

continue

321

continue

320

self.intervals.add((datatime-self.datatime).seconds)

322

self.intervals.add((datatime-self.datatime).seconds)

321

if datatime.date() > self.datatime.date():

322

self.flagDiscontinuousBlock = 1

323

break

323

break

324

elif self.ext == '.hdf5':

324

elif self.ext == '.hdf5':

325

datatime = datetime.datetime.utcfromtimestamp(

325

datatime = datetime.datetime.utcfromtimestamp(

506

log.success(

506

log.success(

507

'Creating file: {}'.format(self.fullname),

507

'Creating file: {}'.format(self.fullname),

508

'MADWriter')

508

'MADWriter')

509

self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)

509

self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)

510

except ValueError, e:

510

except ValueError, e:

511

log.error(

511

log.error(

512

'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',

512

'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',

513

'MADWriter')

513

'MADWriter')

514

return

514

return

515

516

return 1

516

return 1

517

518

def writeBlock(self):

518

def writeBlock(self):

519

'''

519

'''

520

Add data records to cedar file taking data from oneDDict and twoDDict

520

Add data records to cedar file taking data from oneDDict and twoDDict

525

startTime = datetime.datetime.fromtimestamp(self.dataOut.utctime)

525

startTime = datetime.datetime.fromtimestamp(self.dataOut.utctime)

526

endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)

526

endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)

527

heights = self.dataOut.heightList

527

heights = self.dataOut.heightList

528

529

if self.ext == '.dat':

529

if self.ext == '.dat':

530

invalid = numpy.isnan(self.dataOut.data_output)

530

for key, value in self.twoDDict.items():

531

self.dataOut.data_output[invalid] = self.missing

531

if isinstance(value, str):

532

out = {}

532

data = getattr(self.dataOut, value)

533

invalid = numpy.isnan(data)

534

data[invalid] = self.missing

535

elif isinstance(value, (tuple, list)):

536

attr, key = value

537

data = getattr(self.dataOut, attr)

538

invalid = numpy.isnan(data)

539

data[invalid] = self.missing

540

541

out = {}

533

for key, value in self.twoDDict.items():

542

for key, value in self.twoDDict.items():

534

key = key.lower()

543

key = key.lower()

535

if isinstance(value, str):

544

if isinstance(value, str):

536

if 'db' in value.lower():

545

if 'db' in value.lower():

537

tmp = getattr(self.dataOut, value.replace('_db', ''))

546

tmp = getattr(self.dataOut, value.replace('_db', ''))

538

SNRavg = numpy.average(tmp, axis=0)

547

SNRavg = numpy.average(tmp, axis=0)

539

tmp = 10*numpy.log10(SNRavg)

548

tmp = 10*numpy.log10(SNRavg)

540

else:

549

else:

541

tmp = getattr(self.dataOut, value)

550

tmp = getattr(self.dataOut, value)

542

out[key] = tmp.flatten()

551

out[key] = tmp.flatten()

543

elif isinstance(value, (tuple, list)):

552

elif isinstance(value, (tuple, list)):

544

attr, x = value

553

attr, x = value

545

data = getattr(self.dataOut, attr)

554

data = getattr(self.dataOut, attr)

546

out[key] = data[int(x)]

555

out[key] = data[int(x)]

547

556

548

a = numpy.array([out[k] for k in self.keys])

557

a = numpy.array([out[k] for k in self.keys])

549

nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])

558

nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])

550

index = numpy.where(nrows == False)[0]

559

index = numpy.where(nrows == False)[0]

551

560

552

rec = madrigal.cedar.MadrigalDataRecord(

561

rec = madrigal.cedar.MadrigalDataRecord(

553

self.kinst,

562

self.kinst,

554

self.kindat,

563

self.kindat,

586

self.fp.append(rec)

595

self.fp.append(rec)

587

if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:

596

if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:

588

self.fp.dump()

597

self.fp.dump()

589

if self.counter % 10 == 0 and self.counter > 0:

598

if self.counter % 100 == 0 and self.counter > 0:

590

log.log(

599

log.log(

591

'Writing {} records'.format(

600

'Writing {} records'.format(

592

self.counter),

601

self.counter),

616

return 0

625

return 0

617

626

618

if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:

627

if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:

619

if self.counter > 0:

628

if self.counter > 0:

620

self.setHeader()

629

self.setHeader()

621

self.counter = 0

630

self.counter = 0

622

631

623

if self.counter == 0:

632

if self.counter == 0:

624

self.setFile()

633

self.setFile()

625

634

626

self.writeBlock()

635

self.writeBlock()

627

self.counter += 1

636

self.counter += 1

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages

             from schainpy.utils import log
             try:
                 import madrigal.cedar
             except:
                 log.warning(
                     'You should install "madrigal library" module if you want to read/write Madrigal data'
                     dum = []
                     if self.ext == '.txt':
                         dt = self.data[self.counter_records][:6].astype(int)
+                        if datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]).date() > self.datatime.date():
+                            self.flagDiscontinuousBlock = 1
                         self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
                         while True:
                             dt = self.data[self.counter_records][:6].astype(int)
                                     self.flagIsNewFile = True
                                     break
                                 continue
                             self.intervals.add((datatime-self.datatime).seconds)
-                            if datatime.date() > self.datatime.date():
-                                self.flagDiscontinuousBlock = 1
                             break
                     elif self.ext == '.hdf5':
                         datatime = datetime.datetime.utcfromtimestamp(
                         log.success(
                             'Creating file: {}'.format(self.fullname),
                             'MADWriter')
                         self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
                     except ValueError, e:
                         log.error(
                             'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
                             'MADWriter')
                         return
                     return 1
                 def writeBlock(self):
                     '''
                     Add data records to cedar file taking data from oneDDict and twoDDict
                     startTime = datetime.datetime.fromtimestamp(self.dataOut.utctime)
                     endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
                     heights = self.dataOut.heightList
                     if self.ext == '.dat':
-                        invalid = numpy.isnan(self.dataOut.data_output)
+                        for key, value in self.twoDDict.items():
-                        self.dataOut.data_output[invalid] = self.missing
+                            if isinstance(value, str):
-                    out = {}
+                                data = getattr(self.dataOut, value)
+                                invalid = numpy.isnan(data)
+                                data[invalid] = self.missing
+                            elif isinstance(value, (tuple, list)):
+                                attr, key = value
+                                data = getattr(self.dataOut, attr)
+                                invalid = numpy.isnan(data)
+                                data[invalid] = self.missing
+                    out = {}
                     for key, value in self.twoDDict.items():
                         key = key.lower()
                         if isinstance(value, str):
                             if 'db' in value.lower():
                                 tmp = getattr(self.dataOut, value.replace('_db', ''))
                                 SNRavg = numpy.average(tmp, axis=0)
                                 tmp = 10*numpy.log10(SNRavg)
                             else:
                                 tmp = getattr(self.dataOut, value)
                             out[key] = tmp.flatten()
                         elif isinstance(value, (tuple, list)):
                             attr, x = value
                             data = getattr(self.dataOut, attr)
                             out[key] = data[int(x)]
                     a = numpy.array([out[k] for k in self.keys])
                     nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
                     index = numpy.where(nrows == False)[0]
                     rec = madrigal.cedar.MadrigalDataRecord(
                         self.kinst,
                         self.kindat,
                     self.fp.append(rec)
                     if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
                         self.fp.dump()
-                    if self.counter % 10 == 0 and self.counter > 0:
+                    if self.counter % 100 == 0 and self.counter > 0:
                         log.log(
                             'Writing {} records'.format(
                                 self.counter),
                         return 0
                     if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
                         if self.counter > 0:
                             self.setHeader()
                         self.counter = 0
                     if self.counter == 0:
                         self.setFile()
                     self.writeBlock()
                     self.counter += 1