##// END OF EJS Templates
schain-jc
RSS
Madrigal 3 reading/writing task #1154
jespinoza -
r1070:d6c1e5e8fd34
parent child
Show More
Side by Side Unified
Context file:
r1070:d6c1e5e8fd34 Loading diff...:
Show file before | Show file after | Hide comments
@@ -14,22 +14,21 import datetime
14 14 import numpy
15 15 import h5py
16 16
17 try:
18 import madrigal
19 import madrigal.cedar
20 except:
21 print 'You should install "madrigal library" module if you want to read/write Madrigal data'
22
23 from schainpy.model.io.jroIO_base import JRODataReader
17 from schainpy.model.io.jroIO_base import JRODataReader
24 18 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 19 from schainpy.model.data.jrodata import Parameters
26 20 from schainpy.utils import log
27 21
22 try:
23 import madrigal.cedar
24 except:
25 log.warning(
26 'You should install "madrigal library" module if you want to read/write Madrigal data'
27 )
28 28
29 29 DEF_CATALOG = {
30 30 'principleInvestigator': 'Marco Milla',
31 31 'expPurpose': None,
32 'expMode': None,
33 32 'cycleTime': None,
34 33 'correlativeExp': None,
35 34 'sciRemarks': None,
@@ -60,6 +59,8 def load_json(obj):
60 59
61 60 if isinstance(obj, str):
62 61 iterable = json.loads(obj)
62 else:
63 iterable = obj
63 64
64 65 if isinstance(iterable, dict):
65 66 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
@@ -92,8 +93,7 class MADReader(JRODataReader, ProcessingUnit):
92 93 startTime=datetime.time(0, 0, 0),
93 94 endTime=datetime.time(23, 59, 59),
94 95 **kwargs):
95
96 self.started = True
96
97 97 self.path = path
98 98 self.startDate = startDate
99 99 self.endDate = endDate
@@ -135,7 +135,7 class MADReader(JRODataReader, ProcessingUnit):
135 135 path - Path to find files
136 136 '''
137 137
138 print 'Searching files {} in {} '.format(self.ext, path)
138 log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
139 139 foldercounter = 0
140 140 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
141 141 fileList0.sort()
@@ -200,16 +200,18 class MADReader(JRODataReader, ProcessingUnit):
200 200
201 201 for param in self.oneDDict.keys():
202 202 if param.lower() not in self.parameters:
203 print('\x1b[33m[Warning]\x1b[0m Parameter \x1b[1;32m{}\x1b[0m not found will be ignored'.format(
204 param
205 ))
203 log.warning(
204 'Parameter {} not found will be ignored'.format(
205 param),
206 'MADReader')
206 207 self.oneDDict.pop(param, None)
207 208
208 209 for param, value in self.twoDDict.items():
209 210 if param.lower() not in self.parameters:
210 print('\x1b[33m[Warning]\x1b[0m Parameter \x1b[1;32m{}\x1b[0m not found will be ignored'.format(
211 param
212 ))
211 log.warning(
212 'Parameter {} not found, it will be ignored'.format(
213 param),
214 'MADReader')
213 215 self.twoDDict.pop(param, None)
214 216 continue
215 217 if isinstance(value, list):
@@ -237,14 +239,15 class MADReader(JRODataReader, ProcessingUnit):
237 239 file_id = self.fileId
238 240
239 241 if file_id == len(self.fileList):
240 print '\nNo more files in the folder'
241 print 'Total number of file(s) read : {}'.format(self.fileId)
242 log.success('No more files', 'MADReader')
242 243 self.flagNoMoreFiles = 1
243 244 return 0
244 245
245 print('\x1b[32m[Info]\x1b[0m Opening: {}'.format(
246 self.fileList[file_id]
247 ))
246 log.success(
247 'Opening: {}'.format(self.fileList[file_id]),
248 'MADReader'
249 )
250
248 251 filename = os.path.join(self.path, self.fileList[file_id])
249 252
250 253 if self.filename is not None:
@@ -270,7 +273,7 class MADReader(JRODataReader, ProcessingUnit):
270 273 def readNextBlock(self):
271 274
272 275 while True:
273
276 self.flagDiscontinuousBlock = 0
274 277 if self.flagIsNewFile:
275 278 if not self.setNextFile():
276 279 return 0
@@ -279,17 +282,21 class MADReader(JRODataReader, ProcessingUnit):
279 282
280 283 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
281 284 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
282 print "\x1b[32m[Reading]\x1b[0m Record No. %d/%d -> %s \x1b[33m[Skipping]\x1b[0m" %(
283 self.counter_records,
284 self.nrecords,
285 self.datatime.ctime())
285 log.warning(
286 'Reading Record No. {}/{} -> {} [Skipping]'.format(
287 self.counter_records,
288 self.nrecords,
289 self.datatime.ctime()),
290 'MADReader')
286 291 continue
287 292 break
288 293
289 print "\x1b[32m[Reading]\x1b[0m Record No. %d/%d -> %s" %(
290 self.counter_records,
291 self.nrecords,
292 self.datatime.ctime())
294 log.log(
295 'Reading Record No. {}/{} -> {}'.format(
296 self.counter_records,
297 self.nrecords,
298 self.datatime.ctime()),
299 'MADReader')
293 300
294 301 return 1
295 302
@@ -311,6 +318,8 class MADReader(JRODataReader, ProcessingUnit):
311 318 break
312 319 continue
313 320 self.intervals.add((datatime-self.datatime).seconds)
321 if datatime.date() > self.datatime.date():
322 self.flagDiscontinuousBlock = 1
314 323 break
315 324 elif self.ext == '.hdf5':
316 325 datatime = datetime.datetime.utcfromtimestamp(
@@ -326,12 +335,14 class MADReader(JRODataReader, ProcessingUnit):
326 335 tmp = self.data['2D Parameters'][param].value.T
327 336 dum.append(tmp[self.counter_records])
328 337 self.intervals.add((datatime-self.datatime).seconds)
338 if datatime.date()>self.datatime.date():
339 self.flagDiscontinuousBlock = 1
329 340 self.datatime = datatime
330 341 self.counter_records += 1
331 342 if self.counter_records == self.nrecords:
332 343 self.flagIsNewFile = True
333 344
334 self.buffer = numpy.array(dum)
345 self.buffer = numpy.array(dum)
335 346 return
336 347
337 348 def set_output(self):
@@ -344,7 +355,7 class MADReader(JRODataReader, ProcessingUnit):
344 355 for param, attr in self.oneDDict.items():
345 356 x = self.parameters.index(param.lower())
346 357 setattr(self.dataOut, attr, self.buffer[0][x])
347
358
348 359 for param, value in self.twoDDict.items():
349 360 x = self.parameters.index(param.lower())
350 361 if self.ext == '.txt':
@@ -355,10 +366,9 class MADReader(JRODataReader, ProcessingUnit):
355 366 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
356 367 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
357 368 dummy[index] = self.buffer[:,x]
358 else:
359
360 dummy = self.buffer[x]
361
369 else:
370 dummy = self.buffer[x]
371
362 372 if isinstance(value, str):
363 373 if value not in self.ind2DList:
364 374 setattr(self.dataOut, value, dummy.reshape(1,-1))
@@ -375,8 +385,9 class MADReader(JRODataReader, ProcessingUnit):
375 385 self.dataOut.utctimeInit = self.dataOut.utctime
376 386 self.dataOut.paramInterval = min(self.intervals)
377 387 self.dataOut.useLocalTime = False
378 self.dataOut.flagNoData = False
379 self.dataOut.started = self.started
388 self.dataOut.flagNoData = False
389 self.dataOut.nrecords = self.nrecords
390 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
380 391
381 392 def getData(self):
382 393 '''
@@ -384,7 +395,7 class MADReader(JRODataReader, ProcessingUnit):
384 395 '''
385 396 if self.flagNoMoreFiles:
386 397 self.dataOut.flagNoData = True
387 print 'No file left to process'
398 log.error('No file left to process', 'MADReader')
388 399 return 0
389 400
390 401 if not self.readNextBlock():
@@ -396,19 +407,19 class MADReader(JRODataReader, ProcessingUnit):
396 407 return 1
397 408
398 409
399 class MAD2Writer(Operation):
410 class MADWriter(Operation):
400 411
401 missing = -32767
402 ext = '.dat'
412 missing = -32767
403 413
404 414 def __init__(self, **kwargs):
405
415
406 416 Operation.__init__(self, **kwargs)
407 417 self.dataOut = Parameters()
408 418 self.path = None
409 self.dataOut = None
410
411 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}', metadata='{}', **kwargs):
419 self.fp = None
420
421 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
422 metadata='{}', format='cedar', **kwargs):
412 423 '''
413 424 Inputs:
414 425 path - path where files will be created
@@ -434,22 +445,21 class MAD2Writer(Operation):
434 445 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
435 446 '''
436 447 if not self.isConfig:
437 self.setup(dataOut, path, oneDDict, ind2DList, twoDDict, metadata, **kwargs)
448 self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
438 449 self.isConfig = True
439
450
451 self.dataOut = dataOut
440 452 self.putData()
441 453 return
442 454
443 def setup(self, dataOut, path, oneDDict, ind2DList, twoDDict, metadata, **kwargs):
455 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
444 456 '''
445 457 Configure Operation
446 458 '''
447
448 self.dataOut = dataOut
449 self.nmodes = self.dataOut.nmodes
459
450 460 self.path = path
451 461 self.blocks = kwargs.get('blocks', None)
452 self.counter = 0
462 self.counter = 0
453 463 self.oneDDict = load_json(oneDDict)
454 464 self.twoDDict = load_json(twoDDict)
455 465 self.ind2DList = load_json(ind2DList)
@@ -458,8 +468,18 class MAD2Writer(Operation):
458 468 self.kindat = meta.get('kindat')
459 469 self.catalog = meta.get('catalog', DEF_CATALOG)
460 470 self.header = meta.get('header', DEF_HEADER)
461
462 return
471 if format == 'cedar':
472 self.ext = '.dat'
473 self.extra_args = {}
474 elif format == 'hdf5':
475 self.ext = '.hdf5'
476 self.extra_args = {'ind2DList': self.ind2DList}
477
478 self.keys = [k.lower() for k in self.twoDDict]
479 if 'range' in self.keys:
480 self.keys.remove('range')
481 if 'gdalt' in self.keys:
482 self.keys.remove('gdalt')
463 483
464 484 def setFile(self):
465 485 '''
@@ -467,24 +487,30 class MAD2Writer(Operation):
467 487 '''
468 488
469 489 self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
470 date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
490 date = datetime.datetime.fromtimestamp(self.dataOut.utctime)
471 491
472 filename = '%s%s_%s%s' % (self.mnemonic,
473 date.strftime('%Y%m%d_%H%M%S'),
474 self.dataOut.mode,
475 self.ext)
492 filename = '{}{}{}'.format(self.mnemonic,
493 date.strftime('%Y%m%d_%H%M%S'),
494 self.ext)
476 495
477 496 self.fullname = os.path.join(self.path, filename)
478 497
479 498 if os.path.isfile(self.fullname) :
480 print "Destination path '%s' already exists. Previous file deleted. " %self.fullname
499 log.warning(
500 'Destination path {} already exists. Previous file deleted.'.format(
501 self.fullname),
502 'MADWriter')
481 503 os.remove(self.fullname)
482 504
483 505 try:
484 print '[Writing] creating file : %s' % (self.fullname)
485 self.cedarObj = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
506 log.success(
507 'Creating file: {}'.format(self.fullname),
508 'MADWriter')
509 self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
486 510 except ValueError, e:
487 print '[Error]: Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile" '
511 log.error(
512 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
513 'MADWriter')
488 514 return
489 515
490 516 return 1
@@ -496,10 +522,33 class MAD2Writer(Operation):
496 522 Allowed parameters in: parcodes.tab
497 523 '''
498 524
499 startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
525 startTime = datetime.datetime.fromtimestamp(self.dataOut.utctime)
500 526 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
501 nrows = len(getattr(self.dataOut, self.ind2DList))
502
527 heights = self.dataOut.heightList
528
529 if self.ext == '.dat':
530 invalid = numpy.isnan(self.dataOut.data_output)
531 self.dataOut.data_output[invalid] = self.missing
532 out = {}
533 for key, value in self.twoDDict.items():
534 key = key.lower()
535 if isinstance(value, str):
536 if 'db' in value.lower():
537 tmp = getattr(self.dataOut, value.replace('_db', ''))
538 SNRavg = numpy.average(tmp, axis=0)
539 tmp = 10*numpy.log10(SNRavg)
540 else:
541 tmp = getattr(self.dataOut, value)
542 out[key] = tmp.flatten()
543 elif isinstance(value, (tuple, list)):
544 attr, x = value
545 data = getattr(self.dataOut, attr)
546 out[key] = data[int(x)]
547
548 a = numpy.array([out[k] for k in self.keys])
549 nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
550 index = numpy.where(nrows == False)[0]
551
503 552 rec = madrigal.cedar.MadrigalDataRecord(
504 553 self.kinst,
505 554 self.kindat,
@@ -519,45 +568,43 class MAD2Writer(Operation):
519 568 endTime.microsecond/10000,
520 569 self.oneDDict.keys(),
521 570 self.twoDDict.keys(),
522 nrows
523 )
524
571 len(index),
572 **self.extra_args
573 )
574
525 575 # Setting 1d values
526 576 for key in self.oneDDict:
527 577 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
528 578
529 579 # Setting 2d values
530 invalid = numpy.isnan(self.dataOut.data_output)
531 self.dataOut.data_output[invalid] = self.missing
532 out = {}
533 for key, value in self.twoDDict.items():
534 if isinstance(value, str):
535 out[key] = getattr(self.dataOut, value)
536 elif isinstance(value, tuple):
537 attr, x = value
538 if isinstance(x, (int, float)):
539 out[key] = getattr(self.dataOut, attr)[int(x)]
540 elif x.lower()=='db':
541 tmp = getattr(self.dataOut, attr)
542 SNRavg = numpy.average(tmp, axis=0)
543 out[key] = 10*numpy.log10(SNRavg)
544
545 for n in range(nrows):
580 nrec = 0
581 for n in index:
546 582 for key in out:
547 rec.set2D(key, n, out[key][n])
548
549 self.cedarObj.append(rec)
550 self.cedarObj.dump()
551 print '[Writing] Record No. {} (mode {}).'.format(
552 self.counter,
553 self.dataOut.mode
554 )
583 rec.set2D(key, nrec, out[key][n])
584 nrec += 1
585
586 self.fp.append(rec)
587 if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
588 self.fp.dump()
589 if self.counter % 10 == 0 and self.counter > 0:
590 log.log(
591 'Writing {} records'.format(
592 self.counter),
593 'MADWriter')
555 594
556 595 def setHeader(self):
557 596 '''
558 597 Create an add catalog and header to cedar file
559 598 '''
560 599
600 log.success('Closing file {}'.format(self.fullname), 'MADWriter')
601
602 if self.ext == '.dat':
603 self.fp.write()
604 else:
605 self.fp.dump()
606 self.fp.close()
607
561 608 header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
562 609 header.createCatalog(**self.catalog)
563 610 header.createHeader(**self.header)
@@ -566,15 +613,20 class MAD2Writer(Operation):
566 613 def putData(self):
567 614
568 615 if self.dataOut.flagNoData:
569 return 0
616 return 0
570 617
618 if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
619 if self.counter > 0:
620 self.setHeader()
621 self.counter = 0
622
571 623 if self.counter == 0:
572 624 self.setFile()
573 625
574 if self.counter <= self.dataOut.nrecords:
575 self.writeBlock()
576 self.counter += 1
626 self.writeBlock()
627 self.counter += 1
628
629 def close(self):
577 630
578 if self.counter == self.dataOut.nrecords or self.counter == self.blocks:
631 if self.counter > 0:
579 632 self.setHeader()
580 self.counter = 0
General Comments 0
You need to be logged in to leave comments. Login now