##// END OF EJS Templates
schain-jc
RSS
Fix Madrigal HDF5 reader (v2 & v3)
Juan C. Espinoza -
r1208:05917071ce48
parent child
Show More
Side by Side Unified
Context file:
r1208:05917071ce48 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1290,7 +1290,7 class JRODataReader(JRODataIO):
1290 1290 format=None,
1291 1291 oneDDict=None,
1292 1292 twoDDict=None,
1293 ind2DList=None):
1293 independentParam=None):
1294 1294 if server is not None:
1295 1295 if 'tcp://' in server:
1296 1296 address = server
@@ -1485,7 +1485,7 class JRODataReader(JRODataIO):
1485 1485 format=None,
1486 1486 oneDDict=None,
1487 1487 twoDDict=None,
1488 ind2DList=None, **kwargs):
1488 independentParam=None, **kwargs):
1489 1489
1490 1490 if not(self.isConfig):
1491 1491 self.setup(path=path,
@@ -1512,7 +1512,7 class JRODataReader(JRODataIO):
1512 1512 format=format,
1513 1513 oneDDict=oneDDict,
1514 1514 twoDDict=twoDDict,
1515 ind2DList=ind2DList)
1515 independentParam=independentParam)
1516 1516 self.isConfig = True
1517 1517 if server is None:
1518 1518 self.getData()
Show file before | Show file after | Hide comments
@@ -25,6 +25,11 except:
25 25 'You should install "madrigal library" module if you want to read/write Madrigal data'
26 26 )
27 27
28 try:
29 basestring
30 except:
31 basestring = str
32
28 33 DEF_CATALOG = {
29 34 'principleInvestigator': 'Marco Milla',
30 35 'expPurpose': '',
@@ -66,10 +71,10 def load_json(obj):
66 71 iterable = obj
67 72
68 73 if isinstance(iterable, dict):
69 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, (str,unicode)) else v
74 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, basestring) else v
70 75 for k, v in list(iterable.items())}
71 76 elif isinstance(iterable, (list, tuple)):
72 return [str(v) if isinstance(v, str) else v for v in iterable]
77 return [str(v) if isinstance(v, basestring) else v for v in iterable]
73 78
74 79 return iterable
75 80
@@ -107,8 +112,8 class MADReader(JRODataReader, ProcessingUnit):
107 112 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
108 113 self.twoDDict = load_json(kwargs.get('twoDDict',
109 114 "{\"GDALT\": \"heightList\"}"))
110 self.ind2DList = load_json(kwargs.get('ind2DList',
111 "[\"GDALT\"]"))
115 self.independentParam = 'GDALT'
116
112 117 if self.path is None:
113 118 raise ValueError('The path is not valid')
114 119
@@ -139,7 +144,6 class MADReader(JRODataReader, ProcessingUnit):
139 144 '''
140 145
141 146 log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
142 foldercounter = 0
143 147 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
144 148 fileList0.sort()
145 149
@@ -181,24 +185,18 class MADReader(JRODataReader, ProcessingUnit):
181 185 self.version = '2'
182 186 s_parameters = None
183 187 if self.ext == '.txt':
184 self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
188 self.parameters = [s.strip().lower() for s in self.fp.readline().decode().strip().split(' ') if s]
185 189 elif self.ext == '.hdf5':
186 metadata = self.fp['Metadata']
187 data = self.fp['Data']['Array Layout']
188 if 'Independent Spatial Parameters' in metadata:
189 s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
190 self.metadata = self.fp['Metadata']
191 if '_record_layout' in self.metadata:
192 s_parameters = [s[0].lower().decode() for s in self.metadata['Independent Spatial Parameters']]
190 193 self.version = '3'
191 one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
192 one_d = [1 for s in one]
193 two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
194 two_d = [2 for s in two]
195 self.parameters = one + two
196 self.parameters_d = one_d + two_d
194 self.parameters = [s[0].lower().decode() for s in self.metadata['Data Parameters']]
197 195
198 196 log.success('Parameters found: {}'.format(self.parameters),
199 197 'MADReader')
200 198 if s_parameters:
201 log.success('Spatial parameters: {}'.format(','.join(str(s_parameters))),
199 log.success('Spatial parameters found: {}'.format(s_parameters),
202 200 'MADReader')
203 201
204 202 for param in list(self.oneDDict.keys()):
@@ -220,7 +218,7 class MADReader(JRODataReader, ProcessingUnit):
220 218 if isinstance(value, list):
221 219 if value[0] not in self.output:
222 220 self.output[value[0]] = []
223 self.output[value[0]].append(None)
221 self.output[value[0]].append([])
224 222
225 223 def parseData(self):
226 224 '''
@@ -229,11 +227,14 class MADReader(JRODataReader, ProcessingUnit):
229 227 if self.ext == '.txt':
230 228 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
231 229 self.nrecords = self.data.shape[0]
232 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
230 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.independentParam.lower())])
231 self.counter_records = 0
233 232 elif self.ext == '.hdf5':
234 self.data = self.fp['Data']['Array Layout']
235 self.nrecords = len(self.data['timestamps'].value)
236 self.ranges = self.data['range'].value
233 self.data = self.fp['Data']
234 self.ranges = numpy.unique(self.data['Table Layout'][self.independentParam.lower()])
235 self.times = numpy.unique(self.data['Table Layout']['ut1_unix'])
236 self.counter_records = int(self.data['Table Layout']['recno'][0])
237 self.nrecords = int(self.data['Table Layout']['recno'][-1])
237 238
238 239 def setNextFile(self):
239 240 '''
@@ -267,7 +268,6 class MADReader(JRODataReader, ProcessingUnit):
267 268 self.parseHeader()
268 269 self.parseData()
269 270 self.sizeOfFile = os.path.getsize(self.filename)
270 self.counter_records = 0
271 271 self.flagIsNewFile = 0
272 272 self.fileId += 1
273 273
@@ -326,17 +326,8 class MADReader(JRODataReader, ProcessingUnit):
326 326 break
327 327 elif self.ext == '.hdf5':
328 328 datatime = datetime.datetime.utcfromtimestamp(
329 self.data['timestamps'][self.counter_records])
330 nHeights = len(self.ranges)
331 for n, param in enumerate(self.parameters):
332 if self.parameters_d[n] == 1:
333 dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
334 else:
335 if self.version == '2':
336 dum.append(self.data['2D Parameters'][param][self.counter_records])
337 else:
338 tmp = self.data['2D Parameters'][param].value.T
339 dum.append(tmp[self.counter_records])
329 self.times[self.counter_records])
330 dum = self.data['Table Layout'][self.data['Table Layout']['recno']==self.counter_records]
340 331 self.intervals.add((datatime-self.datatime).seconds)
341 332 if datatime.date()>self.datatime.date():
342 333 self.flagDiscontinuousBlock = 1
@@ -360,25 +351,26 class MADReader(JRODataReader, ProcessingUnit):
360 351 setattr(self.dataOut, attr, self.buffer[0][x])
361 352
362 353 for param, value in list(self.twoDDict.items()):
363 x = self.parameters.index(param.lower())
354 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
364 355 if self.ext == '.txt':
365 y = self.parameters.index(self.ind2DList[0].lower())
356 x = self.parameters.index(param.lower())
357 y = self.parameters.index(self.independentParam.lower())
366 358 ranges = self.buffer[:,y]
367 359 #if self.ranges.size == ranges.size:
368 360 # continue
369 361 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
370 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
371 362 dummy[index] = self.buffer[:,x]
372 363 else:
373 dummy = self.buffer[x]
364 ranges = self.buffer[self.independentParam.lower()]
365 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
366 dummy[index] = self.buffer[param.lower()]
374 367
375 368 if isinstance(value, str):
376 if value not in self.ind2DList:
369 if value not in self.independentParam:
377 370 setattr(self.dataOut, value, dummy.reshape(1,-1))
378 371 elif isinstance(value, list):
379 372 self.output[value[0]][value[1]] = dummy
380 373 parameters[value[1]] = param
381
382 374 for key, value in list(self.output.items()):
383 375 setattr(self.dataOut, key, numpy.array(value))
384 376
@@ -422,7 +414,7 class MADWriter(Operation):
422 414 self.path = None
423 415 self.fp = None
424 416
425 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
417 def run(self, dataOut, path, oneDDict, independentParam='[]', twoDDict='{}',
426 418 metadata='{}', format='cedar', **kwargs):
427 419 '''
428 420 Inputs:
@@ -434,8 +426,8 class MADWriter(Operation):
434 426 'gdlonr': 'lon',
435 427 'gdlat2':'lat',
436 428 'glon2':'lon'}
437 ind2DList - list of independent spatial two-dimensional parameters e.g:
438 ['heighList']
429 independentParam - list of independent spatial two-dimensional parameters e.g:
430 ['heigthList']
439 431 twoDDict - json of two-dimensional parameters in record where keys
440 432 are Madrigal codes (integers or mnemonics) and values the corresponding
441 433 dataOut attribute if multidimensional array specify as tupple
@@ -449,14 +441,14 class MADWriter(Operation):
449 441 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
450 442 '''
451 443 if not self.isConfig:
452 self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
444 self.setup(path, oneDDict, independentParam, twoDDict, metadata, format, **kwargs)
453 445 self.isConfig = True
454 446
455 447 self.dataOut = dataOut
456 448 self.putData()
457 449 return 1
458 450
459 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
451 def setup(self, path, oneDDict, independentParam, twoDDict, metadata, format, **kwargs):
460 452 '''
461 453 Configure Operation
462 454 '''
@@ -466,7 +458,7 class MADWriter(Operation):
466 458 self.counter = 0
467 459 self.oneDDict = load_json(oneDDict)
468 460 self.twoDDict = load_json(twoDDict)
469 self.ind2DList = load_json(ind2DList)
461 self.independentParam = load_json(independentParam)
470 462 meta = load_json(metadata)
471 463 self.kinst = meta.get('kinst')
472 464 self.kindat = meta.get('kindat')
@@ -477,7 +469,7 class MADWriter(Operation):
477 469 self.extra_args = {}
478 470 elif format == 'hdf5':
479 471 self.ext = '.hdf5'
480 self.extra_args = {'ind2DList': self.ind2DList}
472 self.extra_args = {'independentParam': self.independentParam}
481 473
482 474 self.keys = [k.lower() for k in self.twoDDict]
483 475 if 'range' in self.keys:
General Comments 0
You need to be logged in to leave comments. Login now