##// END OF EJS Templates
schain-jc
RSS
Madrigal 3 reading/writing task #1154
jespinoza -
r1070:d6c1e5e8fd34
parent child
Show More
Side by Side Unified
Context file:
r1070:d6c1e5e8fd34 Loading diff...:
Show file before | Show file after | Hide comments
@@ -14,22 +14,21 import datetime
14 import numpy
14 import numpy
15 import h5py
15 import h5py
16
16
17 try:
17 from schainpy.model.io.jroIO_base import JRODataReader
18 import madrigal
19 import madrigal.cedar
20 except:
21 print 'You should install "madrigal library" module if you want to read/write Madrigal data'
22
23 from schainpy.model.io.jroIO_base import JRODataReader
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
18 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 from schainpy.model.data.jrodata import Parameters
19 from schainpy.model.data.jrodata import Parameters
26 from schainpy.utils import log
20 from schainpy.utils import log
27
21
22 try:
23 import madrigal.cedar
24 except:
25 log.warning(
26 'You should install "madrigal library" module if you want to read/write Madrigal data'
27 )
28
28
29 DEF_CATALOG = {
29 DEF_CATALOG = {
30 'principleInvestigator': 'Marco Milla',
30 'principleInvestigator': 'Marco Milla',
31 'expPurpose': None,
31 'expPurpose': None,
32 'expMode': None,
33 'cycleTime': None,
32 'cycleTime': None,
34 'correlativeExp': None,
33 'correlativeExp': None,
35 'sciRemarks': None,
34 'sciRemarks': None,
@@ -60,6 +59,8 def load_json(obj):
60
59
61 if isinstance(obj, str):
60 if isinstance(obj, str):
62 iterable = json.loads(obj)
61 iterable = json.loads(obj)
62 else:
63 iterable = obj
63
64
64 if isinstance(iterable, dict):
65 if isinstance(iterable, dict):
65 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
66 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
@@ -92,8 +93,7 class MADReader(JRODataReader, ProcessingUnit):
92 startTime=datetime.time(0, 0, 0),
93 startTime=datetime.time(0, 0, 0),
93 endTime=datetime.time(23, 59, 59),
94 endTime=datetime.time(23, 59, 59),
94 **kwargs):
95 **kwargs):
95
96
96 self.started = True
97 self.path = path
97 self.path = path
98 self.startDate = startDate
98 self.startDate = startDate
99 self.endDate = endDate
99 self.endDate = endDate
@@ -135,7 +135,7 class MADReader(JRODataReader, ProcessingUnit):
135 path - Path to find files
135 path - Path to find files
136 '''
136 '''
137
137
138 print 'Searching files {} in {} '.format(self.ext, path)
138 log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
139 foldercounter = 0
139 foldercounter = 0
140 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
140 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
141 fileList0.sort()
141 fileList0.sort()
@@ -200,16 +200,18 class MADReader(JRODataReader, ProcessingUnit):
200
200
201 for param in self.oneDDict.keys():
201 for param in self.oneDDict.keys():
202 if param.lower() not in self.parameters:
202 if param.lower() not in self.parameters:
203 print('\x1b[33m[Warning]\x1b[0m Parameter \x1b[1;32m{}\x1b[0m not found will be ignored'.format(
203 log.warning(
204 param
204 'Parameter {} not found will be ignored'.format(
205 ))
205 param),
206 'MADReader')
206 self.oneDDict.pop(param, None)
207 self.oneDDict.pop(param, None)
207
208
208 for param, value in self.twoDDict.items():
209 for param, value in self.twoDDict.items():
209 if param.lower() not in self.parameters:
210 if param.lower() not in self.parameters:
210 print('\x1b[33m[Warning]\x1b[0m Parameter \x1b[1;32m{}\x1b[0m not found will be ignored'.format(
211 log.warning(
211 param
212 'Parameter {} not found, it will be ignored'.format(
212 ))
213 param),
214 'MADReader')
213 self.twoDDict.pop(param, None)
215 self.twoDDict.pop(param, None)
214 continue
216 continue
215 if isinstance(value, list):
217 if isinstance(value, list):
@@ -237,14 +239,15 class MADReader(JRODataReader, ProcessingUnit):
237 file_id = self.fileId
239 file_id = self.fileId
238
240
239 if file_id == len(self.fileList):
241 if file_id == len(self.fileList):
240 print '\nNo more files in the folder'
242 log.success('No more files', 'MADReader')
241 print 'Total number of file(s) read : {}'.format(self.fileId)
242 self.flagNoMoreFiles = 1
243 self.flagNoMoreFiles = 1
243 return 0
244 return 0
244
245
245 print('\x1b[32m[Info]\x1b[0m Opening: {}'.format(
246 log.success(
246 self.fileList[file_id]
247 'Opening: {}'.format(self.fileList[file_id]),
247 ))
248 'MADReader'
249 )
250
248 filename = os.path.join(self.path, self.fileList[file_id])
251 filename = os.path.join(self.path, self.fileList[file_id])
249
252
250 if self.filename is not None:
253 if self.filename is not None:
@@ -270,7 +273,7 class MADReader(JRODataReader, ProcessingUnit):
270 def readNextBlock(self):
273 def readNextBlock(self):
271
274
272 while True:
275 while True:
273
276 self.flagDiscontinuousBlock = 0
274 if self.flagIsNewFile:
277 if self.flagIsNewFile:
275 if not self.setNextFile():
278 if not self.setNextFile():
276 return 0
279 return 0
@@ -279,17 +282,21 class MADReader(JRODataReader, ProcessingUnit):
279
282
280 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
283 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
281 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
284 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
282 print "\x1b[32m[Reading]\x1b[0m Record No. %d/%d -> %s \x1b[33m[Skipping]\x1b[0m" %(
285 log.warning(
283 self.counter_records,
286 'Reading Record No. {}/{} -> {} [Skipping]'.format(
284 self.nrecords,
287 self.counter_records,
285 self.datatime.ctime())
288 self.nrecords,
289 self.datatime.ctime()),
290 'MADReader')
286 continue
291 continue
287 break
292 break
288
293
289 print "\x1b[32m[Reading]\x1b[0m Record No. %d/%d -> %s" %(
294 log.log(
290 self.counter_records,
295 'Reading Record No. {}/{} -> {}'.format(
291 self.nrecords,
296 self.counter_records,
292 self.datatime.ctime())
297 self.nrecords,
298 self.datatime.ctime()),
299 'MADReader')
293
300
294 return 1
301 return 1
295
302
@@ -311,6 +318,8 class MADReader(JRODataReader, ProcessingUnit):
311 break
318 break
312 continue
319 continue
313 self.intervals.add((datatime-self.datatime).seconds)
320 self.intervals.add((datatime-self.datatime).seconds)
321 if datatime.date() > self.datatime.date():
322 self.flagDiscontinuousBlock = 1
314 break
323 break
315 elif self.ext == '.hdf5':
324 elif self.ext == '.hdf5':
316 datatime = datetime.datetime.utcfromtimestamp(
325 datatime = datetime.datetime.utcfromtimestamp(
@@ -326,12 +335,14 class MADReader(JRODataReader, ProcessingUnit):
326 tmp = self.data['2D Parameters'][param].value.T
335 tmp = self.data['2D Parameters'][param].value.T
327 dum.append(tmp[self.counter_records])
336 dum.append(tmp[self.counter_records])
328 self.intervals.add((datatime-self.datatime).seconds)
337 self.intervals.add((datatime-self.datatime).seconds)
338 if datatime.date()>self.datatime.date():
339 self.flagDiscontinuousBlock = 1
329 self.datatime = datatime
340 self.datatime = datatime
330 self.counter_records += 1
341 self.counter_records += 1
331 if self.counter_records == self.nrecords:
342 if self.counter_records == self.nrecords:
332 self.flagIsNewFile = True
343 self.flagIsNewFile = True
333
344
334 self.buffer = numpy.array(dum)
345 self.buffer = numpy.array(dum)
335 return
346 return
336
347
337 def set_output(self):
348 def set_output(self):
@@ -344,7 +355,7 class MADReader(JRODataReader, ProcessingUnit):
344 for param, attr in self.oneDDict.items():
355 for param, attr in self.oneDDict.items():
345 x = self.parameters.index(param.lower())
356 x = self.parameters.index(param.lower())
346 setattr(self.dataOut, attr, self.buffer[0][x])
357 setattr(self.dataOut, attr, self.buffer[0][x])
347
358
348 for param, value in self.twoDDict.items():
359 for param, value in self.twoDDict.items():
349 x = self.parameters.index(param.lower())
360 x = self.parameters.index(param.lower())
350 if self.ext == '.txt':
361 if self.ext == '.txt':
@@ -355,10 +366,9 class MADReader(JRODataReader, ProcessingUnit):
355 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
366 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
356 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
367 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
357 dummy[index] = self.buffer[:,x]
368 dummy[index] = self.buffer[:,x]
358 else:
369 else:
359
370 dummy = self.buffer[x]
360 dummy = self.buffer[x]
371
361
362 if isinstance(value, str):
372 if isinstance(value, str):
363 if value not in self.ind2DList:
373 if value not in self.ind2DList:
364 setattr(self.dataOut, value, dummy.reshape(1,-1))
374 setattr(self.dataOut, value, dummy.reshape(1,-1))
@@ -375,8 +385,9 class MADReader(JRODataReader, ProcessingUnit):
375 self.dataOut.utctimeInit = self.dataOut.utctime
385 self.dataOut.utctimeInit = self.dataOut.utctime
376 self.dataOut.paramInterval = min(self.intervals)
386 self.dataOut.paramInterval = min(self.intervals)
377 self.dataOut.useLocalTime = False
387 self.dataOut.useLocalTime = False
378 self.dataOut.flagNoData = False
388 self.dataOut.flagNoData = False
379 self.dataOut.started = self.started
389 self.dataOut.nrecords = self.nrecords
390 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
380
391
381 def getData(self):
392 def getData(self):
382 '''
393 '''
@@ -384,7 +395,7 class MADReader(JRODataReader, ProcessingUnit):
384 '''
395 '''
385 if self.flagNoMoreFiles:
396 if self.flagNoMoreFiles:
386 self.dataOut.flagNoData = True
397 self.dataOut.flagNoData = True
387 print 'No file left to process'
398 log.error('No file left to process', 'MADReader')
388 return 0
399 return 0
389
400
390 if not self.readNextBlock():
401 if not self.readNextBlock():
@@ -396,19 +407,19 class MADReader(JRODataReader, ProcessingUnit):
396 return 1
407 return 1
397
408
398
409
399 class MAD2Writer(Operation):
410 class MADWriter(Operation):
400
411
401 missing = -32767
412 missing = -32767
402 ext = '.dat'
403
413
404 def __init__(self, **kwargs):
414 def __init__(self, **kwargs):
405
415
406 Operation.__init__(self, **kwargs)
416 Operation.__init__(self, **kwargs)
407 self.dataOut = Parameters()
417 self.dataOut = Parameters()
408 self.path = None
418 self.path = None
409 self.dataOut = None
419 self.fp = None
410
420
411 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}', metadata='{}', **kwargs):
421 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
422 metadata='{}', format='cedar', **kwargs):
412 '''
423 '''
413 Inputs:
424 Inputs:
414 path - path where files will be created
425 path - path where files will be created
@@ -434,22 +445,21 class MAD2Writer(Operation):
434 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
445 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
435 '''
446 '''
436 if not self.isConfig:
447 if not self.isConfig:
437 self.setup(dataOut, path, oneDDict, ind2DList, twoDDict, metadata, **kwargs)
448 self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
438 self.isConfig = True
449 self.isConfig = True
439
450
451 self.dataOut = dataOut
440 self.putData()
452 self.putData()
441 return
453 return
442
454
443 def setup(self, dataOut, path, oneDDict, ind2DList, twoDDict, metadata, **kwargs):
455 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
444 '''
456 '''
445 Configure Operation
457 Configure Operation
446 '''
458 '''
447
459
448 self.dataOut = dataOut
449 self.nmodes = self.dataOut.nmodes
450 self.path = path
460 self.path = path
451 self.blocks = kwargs.get('blocks', None)
461 self.blocks = kwargs.get('blocks', None)
452 self.counter = 0
462 self.counter = 0
453 self.oneDDict = load_json(oneDDict)
463 self.oneDDict = load_json(oneDDict)
454 self.twoDDict = load_json(twoDDict)
464 self.twoDDict = load_json(twoDDict)
455 self.ind2DList = load_json(ind2DList)
465 self.ind2DList = load_json(ind2DList)
@@ -458,8 +468,18 class MAD2Writer(Operation):
458 self.kindat = meta.get('kindat')
468 self.kindat = meta.get('kindat')
459 self.catalog = meta.get('catalog', DEF_CATALOG)
469 self.catalog = meta.get('catalog', DEF_CATALOG)
460 self.header = meta.get('header', DEF_HEADER)
470 self.header = meta.get('header', DEF_HEADER)
461
471 if format == 'cedar':
462 return
472 self.ext = '.dat'
473 self.extra_args = {}
474 elif format == 'hdf5':
475 self.ext = '.hdf5'
476 self.extra_args = {'ind2DList': self.ind2DList}
477
478 self.keys = [k.lower() for k in self.twoDDict]
479 if 'range' in self.keys:
480 self.keys.remove('range')
481 if 'gdalt' in self.keys:
482 self.keys.remove('gdalt')
463
483
464 def setFile(self):
484 def setFile(self):
465 '''
485 '''
@@ -467,24 +487,30 class MAD2Writer(Operation):
467 '''
487 '''
468
488
469 self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
489 self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
470 date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
490 date = datetime.datetime.fromtimestamp(self.dataOut.utctime)
471
491
472 filename = '%s%s_%s%s' % (self.mnemonic,
492 filename = '{}{}{}'.format(self.mnemonic,
473 date.strftime('%Y%m%d_%H%M%S'),
493 date.strftime('%Y%m%d_%H%M%S'),
474 self.dataOut.mode,
494 self.ext)
475 self.ext)
476
495
477 self.fullname = os.path.join(self.path, filename)
496 self.fullname = os.path.join(self.path, filename)
478
497
479 if os.path.isfile(self.fullname) :
498 if os.path.isfile(self.fullname) :
480 print "Destination path '%s' already exists. Previous file deleted. " %self.fullname
499 log.warning(
500 'Destination path {} already exists. Previous file deleted.'.format(
501 self.fullname),
502 'MADWriter')
481 os.remove(self.fullname)
503 os.remove(self.fullname)
482
504
483 try:
505 try:
484 print '[Writing] creating file : %s' % (self.fullname)
506 log.success(
485 self.cedarObj = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
507 'Creating file: {}'.format(self.fullname),
508 'MADWriter')
509 self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
486 except ValueError, e:
510 except ValueError, e:
487 print '[Error]: Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile" '
511 log.error(
512 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
513 'MADWriter')
488 return
514 return
489
515
490 return 1
516 return 1
@@ -496,10 +522,33 class MAD2Writer(Operation):
496 Allowed parameters in: parcodes.tab
522 Allowed parameters in: parcodes.tab
497 '''
523 '''
498
524
499 startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
525 startTime = datetime.datetime.fromtimestamp(self.dataOut.utctime)
500 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
526 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
501 nrows = len(getattr(self.dataOut, self.ind2DList))
527 heights = self.dataOut.heightList
502
528
529 if self.ext == '.dat':
530 invalid = numpy.isnan(self.dataOut.data_output)
531 self.dataOut.data_output[invalid] = self.missing
532 out = {}
533 for key, value in self.twoDDict.items():
534 key = key.lower()
535 if isinstance(value, str):
536 if 'db' in value.lower():
537 tmp = getattr(self.dataOut, value.replace('_db', ''))
538 SNRavg = numpy.average(tmp, axis=0)
539 tmp = 10*numpy.log10(SNRavg)
540 else:
541 tmp = getattr(self.dataOut, value)
542 out[key] = tmp.flatten()
543 elif isinstance(value, (tuple, list)):
544 attr, x = value
545 data = getattr(self.dataOut, attr)
546 out[key] = data[int(x)]
547
548 a = numpy.array([out[k] for k in self.keys])
549 nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
550 index = numpy.where(nrows == False)[0]
551
503 rec = madrigal.cedar.MadrigalDataRecord(
552 rec = madrigal.cedar.MadrigalDataRecord(
504 self.kinst,
553 self.kinst,
505 self.kindat,
554 self.kindat,
@@ -519,45 +568,43 class MAD2Writer(Operation):
519 endTime.microsecond/10000,
568 endTime.microsecond/10000,
520 self.oneDDict.keys(),
569 self.oneDDict.keys(),
521 self.twoDDict.keys(),
570 self.twoDDict.keys(),
522 nrows
571 len(index),
523 )
572 **self.extra_args
524
573 )
574
525 # Setting 1d values
575 # Setting 1d values
526 for key in self.oneDDict:
576 for key in self.oneDDict:
527 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
577 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
528
578
529 # Setting 2d values
579 # Setting 2d values
530 invalid = numpy.isnan(self.dataOut.data_output)
580 nrec = 0
531 self.dataOut.data_output[invalid] = self.missing
581 for n in index:
532 out = {}
533 for key, value in self.twoDDict.items():
534 if isinstance(value, str):
535 out[key] = getattr(self.dataOut, value)
536 elif isinstance(value, tuple):
537 attr, x = value
538 if isinstance(x, (int, float)):
539 out[key] = getattr(self.dataOut, attr)[int(x)]
540 elif x.lower()=='db':
541 tmp = getattr(self.dataOut, attr)
542 SNRavg = numpy.average(tmp, axis=0)
543 out[key] = 10*numpy.log10(SNRavg)
544
545 for n in range(nrows):
546 for key in out:
582 for key in out:
547 rec.set2D(key, n, out[key][n])
583 rec.set2D(key, nrec, out[key][n])
548
584 nrec += 1
549 self.cedarObj.append(rec)
585
550 self.cedarObj.dump()
586 self.fp.append(rec)
551 print '[Writing] Record No. {} (mode {}).'.format(
587 if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
552 self.counter,
588 self.fp.dump()
553 self.dataOut.mode
589 if self.counter % 10 == 0 and self.counter > 0:
554 )
590 log.log(
591 'Writing {} records'.format(
592 self.counter),
593 'MADWriter')
555
594
556 def setHeader(self):
595 def setHeader(self):
557 '''
596 '''
558 Create an add catalog and header to cedar file
597 Create an add catalog and header to cedar file
559 '''
598 '''
560
599
600 log.success('Closing file {}'.format(self.fullname), 'MADWriter')
601
602 if self.ext == '.dat':
603 self.fp.write()
604 else:
605 self.fp.dump()
606 self.fp.close()
607
561 header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
608 header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
562 header.createCatalog(**self.catalog)
609 header.createCatalog(**self.catalog)
563 header.createHeader(**self.header)
610 header.createHeader(**self.header)
@@ -566,15 +613,20 class MAD2Writer(Operation):
566 def putData(self):
613 def putData(self):
567
614
568 if self.dataOut.flagNoData:
615 if self.dataOut.flagNoData:
569 return 0
616 return 0
570
617
618 if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
619 if self.counter > 0:
620 self.setHeader()
621 self.counter = 0
622
571 if self.counter == 0:
623 if self.counter == 0:
572 self.setFile()
624 self.setFile()
573
625
574 if self.counter <= self.dataOut.nrecords:
626 self.writeBlock()
575 self.writeBlock()
627 self.counter += 1
576 self.counter += 1
628
629 def close(self):
577
630
578 if self.counter == self.dataOut.nrecords or self.counter == self.blocks:
631 if self.counter > 0:
579 self.setHeader()
632 self.setHeader()
580 self.counter = 0
General Comments 0
You need to be logged in to leave comments. Login now