schain-jc Commit - r1208:05917071ce48

Fix Madrigal HDF5 reader (v2 & v3)

Juan C. Espinoza -

r1208:05917071ce48

parent child

Context file:

r1208:05917071ce48

Collapse all files

schainpy/model/io/jroIO_base.py +3 -3

                           format=None,
                           oneDDict=None,
                           twoDDict=None,
-                          ind2DList=None):
+                          independentParam=None):
                     if server is not None:
                         if 'tcp://' in server:
                             address = server
                         format=None,
                         oneDDict=None,
                         twoDDict=None,
-                        ind2DList=None, **kwargs):
+                        independentParam=None, **kwargs):
                     if not(self.isConfig):
                         self.setup(path=path,
                                    format=format,
                                    oneDDict=oneDDict,
                                    twoDDict=twoDDict,
-                                   ind2DList=ind2DList)
+                                   independentParam=independentParam)
                         self.isConfig = True
                     if server is None:
                         self.getData()

schainpy/model/io/jroIO_madrigal.py +43 -51

                     'You should install "madrigal library" module if you want to read/write Madrigal data'
                     )
+            try:
+                basestring
+            except:
+                basestring = str
             DEF_CATALOG = {
                 'principleInvestigator': 'Marco Milla',
                 'expPurpose': '',
                     iterable = obj
                 if isinstance(iterable, dict):
-                    return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, (str,unicode)) else v
+                    return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, basestring) else v
                         for k, v in list(iterable.items())}
                 elif isinstance(iterable, (list, tuple)):
-                    return [str(v) if isinstance(v, str) else v for v in iterable]
+                    return [str(v) if isinstance(v, basestring) else v for v in iterable]
                 return iterable
                                                          "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
                     self.twoDDict = load_json(kwargs.get('twoDDict',
                                                          "{\"GDALT\": \"heightList\"}"))
-                    self.ind2DList = load_json(kwargs.get('ind2DList',
+                    self.independentParam = 'GDALT'
-                                                          "[\"GDALT\"]"))
                     if self.path is None:
                         raise ValueError('The path is not valid')
                     '''
                     log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
-                    foldercounter = 0
                     fileList0 = glob.glob1(path, '*{}'.format(self.ext))
                     fileList0.sort()
                     self.version = '2'
                     s_parameters = None
                     if self.ext == '.txt':
-                        self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
+                        self.parameters = [s.strip().lower() for s in self.fp.readline().decode().strip().split(' ') if s]
                     elif self.ext == '.hdf5':
-                        metadata = self.fp['Metadata']
+                        self.metadata = self.fp['Metadata']
-                        data = self.fp['Data']['Array Layout']
+                        if '_record_layout' in self.metadata:
-                        if 'Independent Spatial Parameters' in metadata:
+                            s_parameters = [s[0].lower().decode() for s in self.metadata['Independent Spatial Parameters']]
-                            s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
                             self.version = '3'
-                        one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
+                        self.parameters = [s[0].lower().decode() for s in self.metadata['Data Parameters']]
-                        one_d = [1 for s in one]
-                        two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
-                        two_d = [2 for s in two]
-                        self.parameters = one + two
-                        self.parameters_d = one_d + two_d
                     log.success('Parameters found: {}'.format(self.parameters),
                                 'MADReader')
                     if s_parameters:
-                        log.success('Spatial parameters: {}'.format(','.join(str(s_parameters))),
+                        log.success('Spatial parameters found: {}'.format(s_parameters),
                                     'MADReader')
                     for param in list(self.oneDDict.keys()):
                         if isinstance(value, list):
                             if value[0] not in self.output:
                                 self.output[value[0]] = []
-                            self.output[value[0]].append(None)
+                            self.output[value[0]].append([])
                 def parseData(self):
                     '''
                     if self.ext == '.txt':
                         self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
                         self.nrecords = self.data.shape[0]
-                        self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
+                        self.ranges = numpy.unique(self.data[:,self.parameters.index(self.independentParam.lower())])
+                        self.counter_records = 0
                     elif self.ext == '.hdf5':
-                        self.data = self.fp['Data']['Array Layout']
+                        self.data = self.fp['Data']
-                        self.nrecords = len(self.data['timestamps'].value)
+                        self.ranges = numpy.unique(self.data['Table Layout'][self.independentParam.lower()])
-                        self.ranges = self.data['range'].value
+                        self.times = numpy.unique(self.data['Table Layout']['ut1_unix'])
+                        self.counter_records = int(self.data['Table Layout']['recno'][0])
+                        self.nrecords = int(self.data['Table Layout']['recno'][-1])
                 def setNextFile(self):
                     '''
                     '''
                     self.parseHeader()
                     self.parseData()
                     self.sizeOfFile = os.path.getsize(self.filename)
-                    self.counter_records = 0
                     self.flagIsNewFile = 0
                     self.fileId += 1
                             break
                     elif self.ext == '.hdf5':
                         datatime = datetime.datetime.utcfromtimestamp(
-                            self.data['timestamps'][self.counter_records])
+                            self.times[self.counter_records])
-                        nHeights = len(self.ranges)
+                        dum = self.data['Table Layout'][self.data['Table Layout']['recno']==self.counter_records]
-                        for n, param in enumerate(self.parameters):
-                            if self.parameters_d[n] == 1:
-                                dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
-                            else:
-                                if self.version == '2':
-                                    dum.append(self.data['2D Parameters'][param][self.counter_records])
-                                else:
-                                    tmp = self.data['2D Parameters'][param].value.T
-                                    dum.append(tmp[self.counter_records])
                         self.intervals.add((datatime-self.datatime).seconds)
                         if datatime.date()>self.datatime.date():
                             self.flagDiscontinuousBlock = 1
                         x = self.parameters.index(param.lower())
                         setattr(self.dataOut, attr, self.buffer[0][x])
                     for param, value in list(self.twoDDict.items()):
-                        x = self.parameters.index(param.lower())
+                        dummy = numpy.zeros(self.ranges.shape) + numpy.nan
                         if self.ext == '.txt':
-                            y = self.parameters.index(self.ind2DList[0].lower())
+                            x = self.parameters.index(param.lower())
+                            y = self.parameters.index(self.independentParam.lower())
                             ranges = self.buffer[:,y]
                             #if self.ranges.size == ranges.size:
                             #    continue
                             index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
-                            dummy = numpy.zeros(self.ranges.shape) + numpy.nan
                             dummy[index] = self.buffer[:,x]
                         else:
-                            dummy = self.buffer[x]
+                            ranges = self.buffer[self.independentParam.lower()]
+                            index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
+                            dummy[index] = self.buffer[param.lower()]
                         if isinstance(value, str):
-                            if value not in self.ind2DList:
+                            if value not in self.independentParam:
                                 setattr(self.dataOut, value, dummy.reshape(1,-1))
                         elif isinstance(value, list):
                             self.output[value[0]][value[1]] = dummy
                             parameters[value[1]] = param
                     for key, value in list(self.output.items()):
                         setattr(self.dataOut, key, numpy.array(value))
                     self.dataOut.parameters = [s for s in parameters if s]
                     self.dataOut.heightList = self.ranges
                     self.dataOut.utctime = (self.datatime - datetime.datetime(1970, 1, 1)).total_seconds()
                     self.path = None
                     self.fp = None
-                def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
+                def run(self, dataOut, path, oneDDict, independentParam='[]', twoDDict='{}',
                         metadata='{}', format='cedar', **kwargs):
                     '''
                     Inputs:
                             'gdlonr': 'lon',
                             'gdlat2':'lat',
                             'glon2':'lon'}
-                        ind2DList - list of independent spatial two-dimensional parameters e.g:
+                        independentParam - list of independent spatial two-dimensional parameters e.g:
-                            ['heighList']
+                            ['heigthList']
                         twoDDict - json of two-dimensional parameters in record where keys
                         are Madrigal codes (integers or mnemonics) and values the corresponding
                         dataOut attribute if multidimensional array specify as tupple
                         metadata - json of madrigal metadata (kinst, kindat, catalog and header)
                     '''
                     if not self.isConfig:
-                        self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
+                        self.setup(path, oneDDict, independentParam, twoDDict, metadata, format, **kwargs)
                         self.isConfig = True
                     self.dataOut = dataOut
                     self.putData()
                     return 1
-                def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
+                def setup(self, path, oneDDict, independentParam, twoDDict, metadata, format, **kwargs):
                     '''
                     Configure Operation
                     '''
                     self.counter = 0
                     self.oneDDict = load_json(oneDDict)
                     self.twoDDict = load_json(twoDDict)
-                    self.ind2DList = load_json(ind2DList)
+                    self.independentParam = load_json(independentParam)
                     meta = load_json(metadata)
                     self.kinst = meta.get('kinst')
                     self.kindat = meta.get('kindat')
                         self.extra_args = {}
                     elif format == 'hdf5':
                         self.ext = '.hdf5'
-                        self.extra_args = {'ind2DList': self.ind2DList}
+                        self.extra_args = {'independentParam': self.independentParam}
                     self.keys = [k.lower() for k in self.twoDDict]
                     if 'range' in self.keys:

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages