jroIO_madrigal.py
243 lines
| 7.4 KiB
| text/x-python
|
PythonLexer
Juan C. Espinoza
|
r1010 | ''' | ||
| Created on Aug 1, 2017 | ||||
| @author: Juan C. Espinoza | ||||
| ''' | ||||
| import os | ||||
| import sys | ||||
| import time | ||||
|
Juan C. Espinoza
|
r1021 | import json | ||
|
Juan C. Espinoza
|
r1010 | import datetime | ||
| import numpy | ||||
| try: | ||||
| import madrigal | ||||
| import madrigal.cedar | ||||
| except: | ||||
| print 'You should install "madrigal library" module if you want to read/write Madrigal data' | ||||
|
Juan C. Espinoza
|
r1021 | from schainpy.model.proc.jroproc_base import Operation | ||
| from schainpy.model.data.jrodata import Parameters | ||||
| MISSING = -32767 | ||||
| DEF_CATALOG = { | ||||
| 'principleInvestigator': 'Marco Milla', | ||||
| 'expPurpose': None, | ||||
| 'expMode': None, | ||||
| 'cycleTime': None, | ||||
| 'correlativeExp': None, | ||||
| 'sciRemarks': None, | ||||
| 'instRemarks': None | ||||
| } | ||||
| DEF_HEADER = { | ||||
| 'kindatDesc': None, | ||||
| 'analyst': 'Jicamarca User', | ||||
| 'comments': None, | ||||
| 'history': None | ||||
| } | ||||
| MNEMONICS = { | ||||
| 10: 'jro', | ||||
| 11: 'jbr', | ||||
| 840: 'jul', | ||||
| 13: 'jas', | ||||
| 1000: 'pbr', | ||||
| 1001: 'hbr', | ||||
| 1002: 'obr', | ||||
| } | ||||
| def load_json(obj): | ||||
| ''' | ||||
| Parse json as string instead of unicode | ||||
| ''' | ||||
| if isinstance(obj, str): | ||||
| obj = json.loads(obj) | ||||
| return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v | ||||
| for k, v in obj.items()} | ||||
|
Juan C. Espinoza
|
r1010 | |||
|
Juan C. Espinoza
|
r1021 | class MAD2Writer(Operation): | ||
|
Juan C. Espinoza
|
r1010 | |||
|
Juan C. Espinoza
|
r1021 | def __init__(self, **kwargs): | ||
|
Juan C. Espinoza
|
r1010 | |||
|
Juan C. Espinoza
|
r1021 | Operation.__init__(self, **kwargs) | ||
|
Juan C. Espinoza
|
r1010 | self.dataOut = Parameters() | ||
| self.path = None | ||||
| self.dataOut = None | ||||
|
Juan C. Espinoza
|
r1021 | self.ext = '.dat' | ||
|
Juan C. Espinoza
|
r1010 | |||
| return | ||||
|
Juan C. Espinoza
|
r1021 | def run(self, dataOut, path, oneDList, twoDParam='', twoDList='{}', metadata='{}', **kwargs): | ||
| ''' | ||||
| Inputs: | ||||
| path - path where files will be created | ||||
| oneDList - json of one-dimensional parameters in record where keys | ||||
| are Madrigal codes (integers or mnemonics) and values the corresponding | ||||
| dataOut attribute e.g: { | ||||
| 'gdlatr': 'lat', | ||||
| 'gdlonr': 'lon', | ||||
| 'gdlat2':'lat', | ||||
| 'glon2':'lon'} | ||||
| twoDParam - independent parameter to get the number of rows e.g: | ||||
| heighList | ||||
| twoDList - json of two-dimensional parameters in record where keys | ||||
| are Madrigal codes (integers or mnemonics) and values the corresponding | ||||
| dataOut attribute if multidimensional array specify as tupple | ||||
| ('attr', pos) e.g: { | ||||
| 'gdalt': 'heightList', | ||||
| 'vn1p2': ('data_output', 0), | ||||
| 'vn2p2': ('data_output', 1), | ||||
| 'vn3': ('data_output', 2), | ||||
| 'snl': ('data_SNR', 'db') | ||||
| } | ||||
| metadata - json of madrigal metadata (kinst, kindat, catalog and header) | ||||
| ''' | ||||
| if not self.isConfig: | ||||
| self.setup(dataOut, path, oneDList, twoDParam, twoDList, metadata, **kwargs) | ||||
| self.isConfig = True | ||||
|
Juan C. Espinoza
|
r1010 | |||
| self.putData() | ||||
| return | ||||
|
Juan C. Espinoza
|
r1021 | def setup(self, dataOut, path, oneDList, twoDParam, twoDList, metadata, **kwargs): | ||
|
Juan C. Espinoza
|
r1010 | ''' | ||
|
Juan C. Espinoza
|
r1021 | Configure Operation | ||
|
Juan C. Espinoza
|
r1010 | ''' | ||
| self.dataOut = dataOut | ||||
|
Juan C. Espinoza
|
r1021 | self.nmodes = self.dataOut.nmodes | ||
|
Juan C. Espinoza
|
r1010 | self.path = path | ||
|
Juan C. Espinoza
|
r1021 | self.blocks = kwargs.get('blocks', None) | ||
| self.counter = 0 | ||||
| self.oneDList = load_json(oneDList) | ||||
| self.twoDList = load_json(twoDList) | ||||
| self.twoDParam = twoDParam | ||||
| meta = load_json(metadata) | ||||
| self.kinst = meta.get('kinst') | ||||
| self.kindat = meta.get('kindat') | ||||
| self.catalog = meta.get('catalog', DEF_CATALOG) | ||||
| self.header = meta.get('header', DEF_HEADER) | ||||
| return | ||||
|
Juan C. Espinoza
|
r1010 | def setFile(self): | ||
| ''' | ||||
|
Juan C. Espinoza
|
r1021 | Create new cedar file object | ||
| ''' | ||||
| self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal | ||||
| date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime) | ||||
| filename = '%s%s_%s%s' % (self.mnemonic, | ||||
| date.strftime('%Y%m%d_%H%M%S'), | ||||
| self.dataOut.mode, | ||||
| self.ext) | ||||
|
Juan C. Espinoza
|
r1010 | |||
|
Juan C. Espinoza
|
r1021 | self.fullname = os.path.join(self.path, filename) | ||
|
Juan C. Espinoza
|
r1010 | |||
| if os.path.isfile(self.fullname) : | ||||
| print "Destination path '%s' already exists. Previous file deleted. " %self.fullname | ||||
| os.remove(self.fullname) | ||||
| try: | ||||
|
Juan C. Espinoza
|
r1021 | print '[Writing] creating file : %s' % (self.fullname) | ||
|
Juan C. Espinoza
|
r1010 | self.cedarObj = madrigal.cedar.MadrigalCedarFile(self.fullname, True) | ||
|
Juan C. Espinoza
|
r1021 | except ValueError, e: | ||
|
Juan C. Espinoza
|
r1010 | print '[Error]: Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile" ' | ||
| return | ||||
| return 1 | ||||
| def writeBlock(self): | ||||
| ''' | ||||
|
Juan C. Espinoza
|
r1021 | Add data records to cedar file taking data from oneDList and twoDList | ||
| attributes. | ||||
| Allowed parameters in: parcodes.tab | ||||
|
Juan C. Espinoza
|
r1010 | ''' | ||
|
Juan C. Espinoza
|
r1021 | startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime) | ||
| endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval) | ||||
| nrows = len(getattr(self.dataOut, self.twoDParam)) | ||||
| rec = madrigal.cedar.MadrigalDataRecord( | ||||
| self.kinst, | ||||
| self.kindat, | ||||
| startTime.year, | ||||
| startTime.month, | ||||
| startTime.day, | ||||
| startTime.hour, | ||||
| startTime.minute, | ||||
| startTime.second, | ||||
| startTime.microsecond/10000, | ||||
| endTime.year, | ||||
| endTime.month, | ||||
| endTime.day, | ||||
| endTime.hour, | ||||
| endTime.minute, | ||||
| endTime.second, | ||||
| endTime.microsecond/10000, | ||||
| self.oneDList.keys(), | ||||
| self.twoDList.keys(), | ||||
| nrows | ||||
| ) | ||||
|
Juan C. Espinoza
|
r1010 | |||
|
Juan C. Espinoza
|
r1021 | # Setting 1d values | ||
| for key in self.oneDList: | ||||
| rec.set1D(key, getattr(self.dataOut, self.oneDList[key])) | ||||
|
Juan C. Espinoza
|
r1010 | |||
| # Setting 2d values | ||||
|
Juan C. Espinoza
|
r1021 | invalid = numpy.isnan(self.dataOut.data_output) | ||
| self.dataOut.data_output[invalid] = MISSING | ||||
| out = {} | ||||
| for key, value in self.twoDList.items(): | ||||
| if isinstance(value, str): | ||||
| out[key] = getattr(self.dataOut, value) | ||||
| elif isinstance(value, tuple): | ||||
| attr, x = value | ||||
| if isinstance(x, (int, float)): | ||||
| out[key] = getattr(self.dataOut, attr)[int(x)] | ||||
| elif x.lower()=='db': | ||||
| tmp = getattr(self.dataOut, attr) | ||||
| SNRavg = numpy.average(tmp, axis=0) | ||||
| out[key] = 10*numpy.log10(SNRavg) | ||||
| for n in range(nrows): | ||||
| for key in out: | ||||
| rec.set2D(key, n, out[key][n]) | ||||
| self.cedarObj.append(rec) | ||||
|
Juan C. Espinoza
|
r1010 | self.cedarObj.dump() | ||
|
Juan C. Espinoza
|
r1021 | print '[Writing] Record No. {} (mode {}).'.format( | ||
| self.counter, | ||||
| self.dataOut.mode | ||||
| ) | ||||
|
Juan C. Espinoza
|
r1010 | |||
| def setHeader(self): | ||||
| ''' | ||||
|
Juan C. Espinoza
|
r1021 | Create an add catalog and header to cedar file | ||
|
Juan C. Espinoza
|
r1010 | ''' | ||
|
Juan C. Espinoza
|
r1021 | header = madrigal.cedar.CatalogHeaderCreator(self.fullname) | ||
| header.createCatalog(**self.catalog) | ||||
| header.createHeader(**self.header) | ||||
| header.write() | ||||
|
Juan C. Espinoza
|
r1010 | |||
| def putData(self): | ||||
| if self.dataOut.flagNoData: | ||||
| return 0 | ||||
|
Juan C. Espinoza
|
r1021 | if self.counter == 0: | ||
| self.setFile() | ||||
|
Juan C. Espinoza
|
r1010 | |||
|
Juan C. Espinoza
|
r1021 | if self.counter <= self.dataOut.nrecords: | ||
|
Juan C. Espinoza
|
r1010 | self.writeBlock() | ||
|
Juan C. Espinoza
|
r1021 | self.counter += 1 | ||
|
Juan C. Espinoza
|
r1010 | |||
|
Juan C. Espinoza
|
r1021 | if self.counter == self.dataOut.nrecords or self.counter == self.blocks: | ||
|
Juan C. Espinoza
|
r1010 | self.setHeader() | ||
|
Juan C. Espinoza
|
r1021 | self.counter = 0 | ||