##// END OF EJS Templates
schain-jc
RSS
Fix Madrigal HDF5 reader (v2 & v3)
Juan C. Espinoza -
r1208:05917071ce48
parent child
Show More
Side by Side Unified
Context file:
r1208:05917071ce48 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1290,7 +1290,7 class JRODataReader(JRODataIO):
1290 format=None,
1290 format=None,
1291 oneDDict=None,
1291 oneDDict=None,
1292 twoDDict=None,
1292 twoDDict=None,
1293 ind2DList=None):
1293 independentParam=None):
1294 if server is not None:
1294 if server is not None:
1295 if 'tcp://' in server:
1295 if 'tcp://' in server:
1296 address = server
1296 address = server
@@ -1485,7 +1485,7 class JRODataReader(JRODataIO):
1485 format=None,
1485 format=None,
1486 oneDDict=None,
1486 oneDDict=None,
1487 twoDDict=None,
1487 twoDDict=None,
1488 ind2DList=None, **kwargs):
1488 independentParam=None, **kwargs):
1489
1489
1490 if not(self.isConfig):
1490 if not(self.isConfig):
1491 self.setup(path=path,
1491 self.setup(path=path,
@@ -1512,7 +1512,7 class JRODataReader(JRODataIO):
1512 format=format,
1512 format=format,
1513 oneDDict=oneDDict,
1513 oneDDict=oneDDict,
1514 twoDDict=twoDDict,
1514 twoDDict=twoDDict,
1515 ind2DList=ind2DList)
1515 independentParam=independentParam)
1516 self.isConfig = True
1516 self.isConfig = True
1517 if server is None:
1517 if server is None:
1518 self.getData()
1518 self.getData()
Show file before | Show file after | Hide comments
@@ -25,6 +25,11 except:
25 'You should install "madrigal library" module if you want to read/write Madrigal data'
25 'You should install "madrigal library" module if you want to read/write Madrigal data'
26 )
26 )
27
27
28 try:
29 basestring
30 except:
31 basestring = str
32
28 DEF_CATALOG = {
33 DEF_CATALOG = {
29 'principleInvestigator': 'Marco Milla',
34 'principleInvestigator': 'Marco Milla',
30 'expPurpose': '',
35 'expPurpose': '',
@@ -66,10 +71,10 def load_json(obj):
66 iterable = obj
71 iterable = obj
67
72
68 if isinstance(iterable, dict):
73 if isinstance(iterable, dict):
69 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, (str,unicode)) else v
74 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, basestring) else v
70 for k, v in list(iterable.items())}
75 for k, v in list(iterable.items())}
71 elif isinstance(iterable, (list, tuple)):
76 elif isinstance(iterable, (list, tuple)):
72 return [str(v) if isinstance(v, str) else v for v in iterable]
77 return [str(v) if isinstance(v, basestring) else v for v in iterable]
73
78
74 return iterable
79 return iterable
75
80
@@ -107,8 +112,8 class MADReader(JRODataReader, ProcessingUnit):
107 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
112 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
108 self.twoDDict = load_json(kwargs.get('twoDDict',
113 self.twoDDict = load_json(kwargs.get('twoDDict',
109 "{\"GDALT\": \"heightList\"}"))
114 "{\"GDALT\": \"heightList\"}"))
110 self.ind2DList = load_json(kwargs.get('ind2DList',
115 self.independentParam = 'GDALT'
111 "[\"GDALT\"]"))
116
112 if self.path is None:
117 if self.path is None:
113 raise ValueError('The path is not valid')
118 raise ValueError('The path is not valid')
114
119
@@ -139,7 +144,6 class MADReader(JRODataReader, ProcessingUnit):
139 '''
144 '''
140
145
141 log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
146 log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
142 foldercounter = 0
143 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
147 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
144 fileList0.sort()
148 fileList0.sort()
145
149
@@ -181,24 +185,18 class MADReader(JRODataReader, ProcessingUnit):
181 self.version = '2'
185 self.version = '2'
182 s_parameters = None
186 s_parameters = None
183 if self.ext == '.txt':
187 if self.ext == '.txt':
184 self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
188 self.parameters = [s.strip().lower() for s in self.fp.readline().decode().strip().split(' ') if s]
185 elif self.ext == '.hdf5':
189 elif self.ext == '.hdf5':
186 metadata = self.fp['Metadata']
190 self.metadata = self.fp['Metadata']
187 data = self.fp['Data']['Array Layout']
191 if '_record_layout' in self.metadata:
188 if 'Independent Spatial Parameters' in metadata:
192 s_parameters = [s[0].lower().decode() for s in self.metadata['Independent Spatial Parameters']]
189 s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
190 self.version = '3'
193 self.version = '3'
191 one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
194 self.parameters = [s[0].lower().decode() for s in self.metadata['Data Parameters']]
192 one_d = [1 for s in one]
193 two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
194 two_d = [2 for s in two]
195 self.parameters = one + two
196 self.parameters_d = one_d + two_d
197
195
198 log.success('Parameters found: {}'.format(self.parameters),
196 log.success('Parameters found: {}'.format(self.parameters),
199 'MADReader')
197 'MADReader')
200 if s_parameters:
198 if s_parameters:
201 log.success('Spatial parameters: {}'.format(','.join(str(s_parameters))),
199 log.success('Spatial parameters found: {}'.format(s_parameters),
202 'MADReader')
200 'MADReader')
203
201
204 for param in list(self.oneDDict.keys()):
202 for param in list(self.oneDDict.keys()):
@@ -220,7 +218,7 class MADReader(JRODataReader, ProcessingUnit):
220 if isinstance(value, list):
218 if isinstance(value, list):
221 if value[0] not in self.output:
219 if value[0] not in self.output:
222 self.output[value[0]] = []
220 self.output[value[0]] = []
223 self.output[value[0]].append(None)
221 self.output[value[0]].append([])
224
222
225 def parseData(self):
223 def parseData(self):
226 '''
224 '''
@@ -229,11 +227,14 class MADReader(JRODataReader, ProcessingUnit):
229 if self.ext == '.txt':
227 if self.ext == '.txt':
230 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
228 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
231 self.nrecords = self.data.shape[0]
229 self.nrecords = self.data.shape[0]
232 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
230 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.independentParam.lower())])
231 self.counter_records = 0
233 elif self.ext == '.hdf5':
232 elif self.ext == '.hdf5':
234 self.data = self.fp['Data']['Array Layout']
233 self.data = self.fp['Data']
235 self.nrecords = len(self.data['timestamps'].value)
234 self.ranges = numpy.unique(self.data['Table Layout'][self.independentParam.lower()])
236 self.ranges = self.data['range'].value
235 self.times = numpy.unique(self.data['Table Layout']['ut1_unix'])
236 self.counter_records = int(self.data['Table Layout']['recno'][0])
237 self.nrecords = int(self.data['Table Layout']['recno'][-1])
237
238
238 def setNextFile(self):
239 def setNextFile(self):
239 '''
240 '''
@@ -267,7 +268,6 class MADReader(JRODataReader, ProcessingUnit):
267 self.parseHeader()
268 self.parseHeader()
268 self.parseData()
269 self.parseData()
269 self.sizeOfFile = os.path.getsize(self.filename)
270 self.sizeOfFile = os.path.getsize(self.filename)
270 self.counter_records = 0
271 self.flagIsNewFile = 0
271 self.flagIsNewFile = 0
272 self.fileId += 1
272 self.fileId += 1
273
273
@@ -326,17 +326,8 class MADReader(JRODataReader, ProcessingUnit):
326 break
326 break
327 elif self.ext == '.hdf5':
327 elif self.ext == '.hdf5':
328 datatime = datetime.datetime.utcfromtimestamp(
328 datatime = datetime.datetime.utcfromtimestamp(
329 self.data['timestamps'][self.counter_records])
329 self.times[self.counter_records])
330 nHeights = len(self.ranges)
330 dum = self.data['Table Layout'][self.data['Table Layout']['recno']==self.counter_records]
331 for n, param in enumerate(self.parameters):
332 if self.parameters_d[n] == 1:
333 dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
334 else:
335 if self.version == '2':
336 dum.append(self.data['2D Parameters'][param][self.counter_records])
337 else:
338 tmp = self.data['2D Parameters'][param].value.T
339 dum.append(tmp[self.counter_records])
340 self.intervals.add((datatime-self.datatime).seconds)
331 self.intervals.add((datatime-self.datatime).seconds)
341 if datatime.date()>self.datatime.date():
332 if datatime.date()>self.datatime.date():
342 self.flagDiscontinuousBlock = 1
333 self.flagDiscontinuousBlock = 1
@@ -360,25 +351,26 class MADReader(JRODataReader, ProcessingUnit):
360 setattr(self.dataOut, attr, self.buffer[0][x])
351 setattr(self.dataOut, attr, self.buffer[0][x])
361
352
362 for param, value in list(self.twoDDict.items()):
353 for param, value in list(self.twoDDict.items()):
363 x = self.parameters.index(param.lower())
354 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
364 if self.ext == '.txt':
355 if self.ext == '.txt':
365 y = self.parameters.index(self.ind2DList[0].lower())
356 x = self.parameters.index(param.lower())
357 y = self.parameters.index(self.independentParam.lower())
366 ranges = self.buffer[:,y]
358 ranges = self.buffer[:,y]
367 #if self.ranges.size == ranges.size:
359 #if self.ranges.size == ranges.size:
368 # continue
360 # continue
369 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
361 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
370 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
371 dummy[index] = self.buffer[:,x]
362 dummy[index] = self.buffer[:,x]
372 else:
363 else:
373 dummy = self.buffer[x]
364 ranges = self.buffer[self.independentParam.lower()]
365 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
366 dummy[index] = self.buffer[param.lower()]
374
367
375 if isinstance(value, str):
368 if isinstance(value, str):
376 if value not in self.ind2DList:
369 if value not in self.independentParam:
377 setattr(self.dataOut, value, dummy.reshape(1,-1))
370 setattr(self.dataOut, value, dummy.reshape(1,-1))
378 elif isinstance(value, list):
371 elif isinstance(value, list):
379 self.output[value[0]][value[1]] = dummy
372 self.output[value[0]][value[1]] = dummy
380 parameters[value[1]] = param
373 parameters[value[1]] = param
381
382 for key, value in list(self.output.items()):
374 for key, value in list(self.output.items()):
383 setattr(self.dataOut, key, numpy.array(value))
375 setattr(self.dataOut, key, numpy.array(value))
384
376
@@ -422,7 +414,7 class MADWriter(Operation):
422 self.path = None
414 self.path = None
423 self.fp = None
415 self.fp = None
424
416
425 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
417 def run(self, dataOut, path, oneDDict, independentParam='[]', twoDDict='{}',
426 metadata='{}', format='cedar', **kwargs):
418 metadata='{}', format='cedar', **kwargs):
427 '''
419 '''
428 Inputs:
420 Inputs:
@@ -434,8 +426,8 class MADWriter(Operation):
434 'gdlonr': 'lon',
426 'gdlonr': 'lon',
435 'gdlat2':'lat',
427 'gdlat2':'lat',
436 'glon2':'lon'}
428 'glon2':'lon'}
437 ind2DList - list of independent spatial two-dimensional parameters e.g:
429 independentParam - list of independent spatial two-dimensional parameters e.g:
438 ['heighList']
430 ['heigthList']
439 twoDDict - json of two-dimensional parameters in record where keys
431 twoDDict - json of two-dimensional parameters in record where keys
440 are Madrigal codes (integers or mnemonics) and values the corresponding
432 are Madrigal codes (integers or mnemonics) and values the corresponding
441 dataOut attribute if multidimensional array specify as tupple
433 dataOut attribute if multidimensional array specify as tupple
@@ -449,14 +441,14 class MADWriter(Operation):
449 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
441 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
450 '''
442 '''
451 if not self.isConfig:
443 if not self.isConfig:
452 self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
444 self.setup(path, oneDDict, independentParam, twoDDict, metadata, format, **kwargs)
453 self.isConfig = True
445 self.isConfig = True
454
446
455 self.dataOut = dataOut
447 self.dataOut = dataOut
456 self.putData()
448 self.putData()
457 return 1
449 return 1
458
450
459 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
451 def setup(self, path, oneDDict, independentParam, twoDDict, metadata, format, **kwargs):
460 '''
452 '''
461 Configure Operation
453 Configure Operation
462 '''
454 '''
@@ -466,7 +458,7 class MADWriter(Operation):
466 self.counter = 0
458 self.counter = 0
467 self.oneDDict = load_json(oneDDict)
459 self.oneDDict = load_json(oneDDict)
468 self.twoDDict = load_json(twoDDict)
460 self.twoDDict = load_json(twoDDict)
469 self.ind2DList = load_json(ind2DList)
461 self.independentParam = load_json(independentParam)
470 meta = load_json(metadata)
462 meta = load_json(metadata)
471 self.kinst = meta.get('kinst')
463 self.kinst = meta.get('kinst')
472 self.kindat = meta.get('kindat')
464 self.kindat = meta.get('kindat')
@@ -477,7 +469,7 class MADWriter(Operation):
477 self.extra_args = {}
469 self.extra_args = {}
478 elif format == 'hdf5':
470 elif format == 'hdf5':
479 self.ext = '.hdf5'
471 self.ext = '.hdf5'
480 self.extra_args = {'ind2DList': self.ind2DList}
472 self.extra_args = {'independentParam': self.independentParam}
481
473
482 self.keys = [k.lower() for k in self.twoDDict]
474 self.keys = [k.lower() for k in self.twoDDict]
483 if 'range' in self.keys:
475 if 'range' in self.keys:
General Comments 0
You need to be logged in to leave comments. Login now