##// END OF EJS Templates
schain-jc
RSS
Writing Unit for Madrigal decorated (just for python 2x)
George Yong -
r1206:59caf7a2130e
parent child
Show More
Side by Side Unified
Context file:
r1206:59caf7a2130e Loading diff...:
Show file before | Show file after | Hide comments
@@ -13,8 +13,7 import datetime
13
13
14 import numpy
14 import numpy
15 import h5py
15 import h5py
16
16 from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
17 from schainpy.model.io.jroIO_base import JRODataReader
18 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
17 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
19 from schainpy.model.data.jrodata import Parameters
18 from schainpy.model.data.jrodata import Parameters
20 from schainpy.utils import log
19 from schainpy.utils import log
@@ -28,18 +27,20 except:
28
27
29 DEF_CATALOG = {
28 DEF_CATALOG = {
30 'principleInvestigator': 'Marco Milla',
29 'principleInvestigator': 'Marco Milla',
31 'expPurpose': None,
30 'expPurpose': '',
32 'cycleTime': None,
31 'cycleTime': '',
33 'correlativeExp': None,
32 'correlativeExp': '',
34 'sciRemarks': None,
33 'sciRemarks': '',
35 'instRemarks': None
34 'instRemarks': ''
36 }
35 }
36
37 DEF_HEADER = {
37 DEF_HEADER = {
38 'kindatDesc': None,
38 'kindatDesc': '',
39 'analyst': 'Jicamarca User',
39 'analyst': 'Jicamarca User',
40 'comments': None,
40 'comments': '',
41 'history': None
41 'history': ''
42 }
42 }
43
43 MNEMONICS = {
44 MNEMONICS = {
44 10: 'jro',
45 10: 'jro',
45 11: 'jbr',
46 11: 'jbr',
@@ -48,6 +49,8 MNEMONICS = {
48 1000: 'pbr',
49 1000: 'pbr',
49 1001: 'hbr',
50 1001: 'hbr',
50 1002: 'obr',
51 1002: 'obr',
52 400: 'clr'
53
51 }
54 }
52
55
53 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
56 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
@@ -63,7 +66,7 def load_json(obj):
63 iterable = obj
66 iterable = obj
64
67
65 if isinstance(iterable, dict):
68 if isinstance(iterable, dict):
66 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, str) else v
69 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, (str,unicode)) else v
67 for k, v in list(iterable.items())}
70 for k, v in list(iterable.items())}
68 elif isinstance(iterable, (list, tuple)):
71 elif isinstance(iterable, (list, tuple)):
69 return [str(v) if isinstance(v, str) else v for v in iterable]
72 return [str(v) if isinstance(v, str) else v for v in iterable]
@@ -192,7 +195,7 class MADReader(JRODataReader, ProcessingUnit):
192 self.parameters = one + two
195 self.parameters = one + two
193 self.parameters_d = one_d + two_d
196 self.parameters_d = one_d + two_d
194
197
195 log.success('Parameters found: {}'.format(','.join(str(self.parameters))),
198 log.success('Parameters found: {}'.format(self.parameters),
196 'MADReader')
199 'MADReader')
197 if s_parameters:
200 if s_parameters:
198 log.success('Spatial parameters: {}'.format(','.join(str(s_parameters))),
201 log.success('Spatial parameters: {}'.format(','.join(str(s_parameters))),
@@ -361,8 +364,8 class MADReader(JRODataReader, ProcessingUnit):
361 if self.ext == '.txt':
364 if self.ext == '.txt':
362 y = self.parameters.index(self.ind2DList[0].lower())
365 y = self.parameters.index(self.ind2DList[0].lower())
363 ranges = self.buffer[:,y]
366 ranges = self.buffer[:,y]
364 if self.ranges.size == ranges.size:
367 #if self.ranges.size == ranges.size:
365 continue
368 # continue
366 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
369 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
367 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
370 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
368 dummy[index] = self.buffer[:,x]
371 dummy[index] = self.buffer[:,x]
@@ -406,14 +409,14 class MADReader(JRODataReader, ProcessingUnit):
406
409
407 return 1
410 return 1
408
411
409
412 @MPDecorator
410 class MADWriter(Operation):
413 class MADWriter(Operation):
411
414
412 missing = -32767
415 missing = -32767
413
416
414 def __init__(self, **kwargs):
417 def __init__(self):
415
418
416 Operation.__init__(self, **kwargs)
419 Operation.__init__(self)
417 self.dataOut = Parameters()
420 self.dataOut = Parameters()
418 self.counter = 0
421 self.counter = 0
419 self.path = None
422 self.path = None
@@ -451,7 +454,7 class MADWriter(Operation):
451
454
452 self.dataOut = dataOut
455 self.dataOut = dataOut
453 self.putData()
456 self.putData()
454 return
457 return 1
455
458
456 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
459 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
457 '''
460 '''
@@ -596,7 +599,7 class MADWriter(Operation):
596 self.fp.append(rec)
599 self.fp.append(rec)
597 if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
600 if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
598 self.fp.dump()
601 self.fp.dump()
599 if self.counter % 100 == 0 and self.counter > 0:
602 if self.counter % 20 == 0 and self.counter > 0:
600 log.log(
603 log.log(
601 'Writing {} records'.format(
604 'Writing {} records'.format(
602 self.counter),
605 self.counter),
Show file before | Show file after | Hide comments
@@ -294,11 +294,17 def MPDecorator(BaseClass):
294 elif optype == 'other' and not self.dataOut.flagNoData:
294 elif optype == 'other' and not self.dataOut.flagNoData:
295 self.dataOut = op.run(self.dataOut, **kwargs)
295 self.dataOut = op.run(self.dataOut, **kwargs)
296 elif optype == 'external' and not self.dataOut.flagNoData:
296 elif optype == 'external' and not self.dataOut.flagNoData:
297 if not self.dataOut.flagNoData or self.dataOut.error:
298 self.publish(self.dataOut, opId)
297 self.publish(self.dataOut, opId)
299
298
300 if not self.dataOut.flagNoData or self.dataOut.error:
299 if not self.dataOut.flagNoData or self.dataOut.error:
301 self.publish(self.dataOut, self.id)
300 self.publish(self.dataOut, self.id)
301 for op, optype, opId, kwargs in self.operations:
302 if optype == 'self' and self.dataOut.error:
303 op(**kwargs)
304 elif optype == 'other' and self.dataOut.error:
305 self.dataOut = op.run(self.dataOut, **kwargs)
306 elif optype == 'external' and self.dataOut.error:
307 self.publish(self.dataOut, opId)
302
308
303 if self.dataIn.error:
309 if self.dataIn.error:
304 break
310 break
Show file before | Show file after | Hide comments
@@ -94,6 +94,7 class ParametersProc(ProcessingUnit):
94 self.dataOut.heightList = self.dataIn.getHeiRange()
94 self.dataOut.heightList = self.dataIn.getHeiRange()
95 self.dataOut.frequency = self.dataIn.frequency
95 self.dataOut.frequency = self.dataIn.frequency
96 # self.dataOut.noise = self.dataIn.noise
96 # self.dataOut.noise = self.dataIn.noise
97 self.dataOut.error = self.dataIn.error
97
98
98 def run(self):
99 def run(self):
99
100
General Comments 0
You need to be logged in to leave comments. Login now