##// END OF EJS Templates
schain-jc
RSS
Reading Unit for Madrigal decorated (just for python2X)
George Yong -
r1189:67ce1e63c533
parent child
Show More
Side by Side Unified
Context file:
r1189:67ce1e63c533 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,642 +1,642
1 '''
1 '''
2 Created on Aug 1, 2017
2 Created on Aug 1, 2017
3
3
4 @author: Juan C. Espinoza
4 @author: Juan C. Espinoza
5 '''
5 '''
6
6
7 import os
7 import os
8 import sys
8 import sys
9 import time
9 import time
10 import json
10 import json
11 import glob
11 import glob
12 import datetime
12 import datetime
13
13
14 import numpy
14 import numpy
15 import h5py
15 import h5py
16
16
17 from schainpy.model.io.jroIO_base import JRODataReader
17 from schainpy.model.io.jroIO_base import JRODataReader
18 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
18 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
19 from schainpy.model.data.jrodata import Parameters
19 from schainpy.model.data.jrodata import Parameters
20 from schainpy.utils import log
20 from schainpy.utils import log
21
21
22 try:
22 try:
23 import madrigal.cedar
23 import madrigal.cedar
24 except:
24 except:
25 log.warning(
25 log.warning(
26 'You should install "madrigal library" module if you want to read/write Madrigal data'
26 'You should install "madrigal library" module if you want to read/write Madrigal data'
27 )
27 )
28
28
29 DEF_CATALOG = {
29 DEF_CATALOG = {
30 'principleInvestigator': 'Marco Milla',
30 'principleInvestigator': 'Marco Milla',
31 'expPurpose': None,
31 'expPurpose': None,
32 'cycleTime': None,
32 'cycleTime': None,
33 'correlativeExp': None,
33 'correlativeExp': None,
34 'sciRemarks': None,
34 'sciRemarks': None,
35 'instRemarks': None
35 'instRemarks': None
36 }
36 }
37 DEF_HEADER = {
37 DEF_HEADER = {
38 'kindatDesc': None,
38 'kindatDesc': None,
39 'analyst': 'Jicamarca User',
39 'analyst': 'Jicamarca User',
40 'comments': None,
40 'comments': None,
41 'history': None
41 'history': None
42 }
42 }
43 MNEMONICS = {
43 MNEMONICS = {
44 10: 'jro',
44 10: 'jro',
45 11: 'jbr',
45 11: 'jbr',
46 840: 'jul',
46 840: 'jul',
47 13: 'jas',
47 13: 'jas',
48 1000: 'pbr',
48 1000: 'pbr',
49 1001: 'hbr',
49 1001: 'hbr',
50 1002: 'obr',
50 1002: 'obr',
51 }
51 }
52
52
53 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
53 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
54
54
55 def load_json(obj):
55 def load_json(obj):
56 '''
56 '''
57 Parse json as string instead of unicode
57 Parse json as string instead of unicode
58 '''
58 '''
59
59
60 if isinstance(obj, str):
60 if isinstance(obj, str):
61 iterable = json.loads(obj)
61 iterable = json.loads(obj)
62 else:
62 else:
63 iterable = obj
63 iterable = obj
64
64
65 if isinstance(iterable, dict):
65 if isinstance(iterable, dict):
66 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, str) else v
66 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, str) else v
67 for k, v in list(iterable.items())}
67 for k, v in list(iterable.items())}
68 elif isinstance(iterable, (list, tuple)):
68 elif isinstance(iterable, (list, tuple)):
69 return [str(v) if isinstance(v, str) else v for v in iterable]
69 return [str(v) if isinstance(v, str) else v for v in iterable]
70
70
71 return iterable
71 return iterable
72
72
73
73 @MPDecorator
74 class MADReader(JRODataReader, ProcessingUnit):
74 class MADReader(JRODataReader, ProcessingUnit):
75
75
76 def __init__(self, **kwargs):
76 def __init__(self):
77
77
78 ProcessingUnit.__init__(self, **kwargs)
78 ProcessingUnit.__init__(self)
79
79
80 self.dataOut = Parameters()
80 self.dataOut = Parameters()
81 self.counter_records = 0
81 self.counter_records = 0
82 self.nrecords = None
82 self.nrecords = None
83 self.flagNoMoreFiles = 0
83 self.flagNoMoreFiles = 0
84 self.isConfig = False
84 self.isConfig = False
85 self.filename = None
85 self.filename = None
86 self.intervals = set()
86 self.intervals = set()
87
87
88 def setup(self,
88 def setup(self,
89 path=None,
89 path=None,
90 startDate=None,
90 startDate=None,
91 endDate=None,
91 endDate=None,
92 format=None,
92 format=None,
93 startTime=datetime.time(0, 0, 0),
93 startTime=datetime.time(0, 0, 0),
94 endTime=datetime.time(23, 59, 59),
94 endTime=datetime.time(23, 59, 59),
95 **kwargs):
95 **kwargs):
96
96
97 self.path = path
97 self.path = path
98 self.startDate = startDate
98 self.startDate = startDate
99 self.endDate = endDate
99 self.endDate = endDate
100 self.startTime = startTime
100 self.startTime = startTime
101 self.endTime = endTime
101 self.endTime = endTime
102 self.datatime = datetime.datetime(1900,1,1)
102 self.datatime = datetime.datetime(1900,1,1)
103 self.oneDDict = load_json(kwargs.get('oneDDict',
103 self.oneDDict = load_json(kwargs.get('oneDDict',
104 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
104 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
105 self.twoDDict = load_json(kwargs.get('twoDDict',
105 self.twoDDict = load_json(kwargs.get('twoDDict',
106 "{\"GDALT\": \"heightList\"}"))
106 "{\"GDALT\": \"heightList\"}"))
107 self.ind2DList = load_json(kwargs.get('ind2DList',
107 self.ind2DList = load_json(kwargs.get('ind2DList',
108 "[\"GDALT\"]"))
108 "[\"GDALT\"]"))
109 if self.path is None:
109 if self.path is None:
110 raise ValueError('The path is not valid')
110 raise ValueError('The path is not valid')
111
111
112 if format is None:
112 if format is None:
113 raise ValueError('The format is not valid choose simple or hdf5')
113 raise ValueError('The format is not valid choose simple or hdf5')
114 elif format.lower() in ('simple', 'txt'):
114 elif format.lower() in ('simple', 'txt'):
115 self.ext = '.txt'
115 self.ext = '.txt'
116 elif format.lower() in ('cedar',):
116 elif format.lower() in ('cedar',):
117 self.ext = '.001'
117 self.ext = '.001'
118 else:
118 else:
119 self.ext = '.hdf5'
119 self.ext = '.hdf5'
120
120
121 self.search_files(self.path)
121 self.search_files(self.path)
122 self.fileId = 0
122 self.fileId = 0
123
123
124 if not self.fileList:
124 if not self.fileList:
125 raise Warning('There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path))
125 raise Warning('There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path))
126
126
127 self.setNextFile()
127 self.setNextFile()
128
128
129 def search_files(self, path):
129 def search_files(self, path):
130 '''
130 '''
131 Searching for madrigal files in path
131 Searching for madrigal files in path
132 Creating a list of files to procces included in [startDate,endDate]
132 Creating a list of files to procces included in [startDate,endDate]
133
133
134 Input:
134 Input:
135 path - Path to find files
135 path - Path to find files
136 '''
136 '''
137
137
138 log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
138 log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
139 foldercounter = 0
139 foldercounter = 0
140 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
140 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
141 fileList0.sort()
141 fileList0.sort()
142
142
143 self.fileList = []
143 self.fileList = []
144 self.dateFileList = []
144 self.dateFileList = []
145
145
146 startDate = self.startDate - datetime.timedelta(1)
146 startDate = self.startDate - datetime.timedelta(1)
147 endDate = self.endDate + datetime.timedelta(1)
147 endDate = self.endDate + datetime.timedelta(1)
148
148
149 for thisFile in fileList0:
149 for thisFile in fileList0:
150 year = thisFile[3:7]
150 year = thisFile[3:7]
151 if not year.isdigit():
151 if not year.isdigit():
152 continue
152 continue
153
153
154 month = thisFile[7:9]
154 month = thisFile[7:9]
155 if not month.isdigit():
155 if not month.isdigit():
156 continue
156 continue
157
157
158 day = thisFile[9:11]
158 day = thisFile[9:11]
159 if not day.isdigit():
159 if not day.isdigit():
160 continue
160 continue
161
161
162 year, month, day = int(year), int(month), int(day)
162 year, month, day = int(year), int(month), int(day)
163 dateFile = datetime.date(year, month, day)
163 dateFile = datetime.date(year, month, day)
164
164
165 if (startDate > dateFile) or (endDate < dateFile):
165 if (startDate > dateFile) or (endDate < dateFile):
166 continue
166 continue
167
167
168 self.fileList.append(thisFile)
168 self.fileList.append(thisFile)
169 self.dateFileList.append(dateFile)
169 self.dateFileList.append(dateFile)
170
170
171 return
171 return
172
172
173 def parseHeader(self):
173 def parseHeader(self):
174 '''
174 '''
175 '''
175 '''
176
176
177 self.output = {}
177 self.output = {}
178 self.version = '2'
178 self.version = '2'
179 s_parameters = None
179 s_parameters = None
180 if self.ext == '.txt':
180 if self.ext == '.txt':
181 self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
181 self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
182 elif self.ext == '.hdf5':
182 elif self.ext == '.hdf5':
183 metadata = self.fp['Metadata']
183 metadata = self.fp['Metadata']
184 data = self.fp['Data']['Array Layout']
184 data = self.fp['Data']['Array Layout']
185 if 'Independent Spatial Parameters' in metadata:
185 if 'Independent Spatial Parameters' in metadata:
186 s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
186 s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
187 self.version = '3'
187 self.version = '3'
188 one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
188 one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
189 one_d = [1 for s in one]
189 one_d = [1 for s in one]
190 two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
190 two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
191 two_d = [2 for s in two]
191 two_d = [2 for s in two]
192 self.parameters = one + two
192 self.parameters = one + two
193 self.parameters_d = one_d + two_d
193 self.parameters_d = one_d + two_d
194
194
195 log.success('Parameters found: {}'.format(','.join(self.parameters)),
195 log.success('Parameters found: {}'.format(','.join(self.parameters)),
196 'MADReader')
196 'MADReader')
197 if s_parameters:
197 if s_parameters:
198 log.success('Spatial parameters: {}'.format(','.join(s_parameters)),
198 log.success('Spatial parameters: {}'.format(','.join(s_parameters)),
199 'MADReader')
199 'MADReader')
200
200
201 for param in list(self.oneDDict.keys()):
201 for param in list(self.oneDDict.keys()):
202 if param.lower() not in self.parameters:
202 if param.lower() not in self.parameters:
203 log.warning(
203 log.warning(
204 'Parameter {} not found will be ignored'.format(
204 'Parameter {} not found will be ignored'.format(
205 param),
205 param),
206 'MADReader')
206 'MADReader')
207 self.oneDDict.pop(param, None)
207 self.oneDDict.pop(param, None)
208
208
209 for param, value in list(self.twoDDict.items()):
209 for param, value in list(self.twoDDict.items()):
210 if param.lower() not in self.parameters:
210 if param.lower() not in self.parameters:
211 log.warning(
211 log.warning(
212 'Parameter {} not found, it will be ignored'.format(
212 'Parameter {} not found, it will be ignored'.format(
213 param),
213 param),
214 'MADReader')
214 'MADReader')
215 self.twoDDict.pop(param, None)
215 self.twoDDict.pop(param, None)
216 continue
216 continue
217 if isinstance(value, list):
217 if isinstance(value, list):
218 if value[0] not in self.output:
218 if value[0] not in self.output:
219 self.output[value[0]] = []
219 self.output[value[0]] = []
220 self.output[value[0]].append(None)
220 self.output[value[0]].append(None)
221
221
222 def parseData(self):
222 def parseData(self):
223 '''
223 '''
224 '''
224 '''
225
225
226 if self.ext == '.txt':
226 if self.ext == '.txt':
227 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
227 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
228 self.nrecords = self.data.shape[0]
228 self.nrecords = self.data.shape[0]
229 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
229 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
230 elif self.ext == '.hdf5':
230 elif self.ext == '.hdf5':
231 self.data = self.fp['Data']['Array Layout']
231 self.data = self.fp['Data']['Array Layout']
232 self.nrecords = len(self.data['timestamps'].value)
232 self.nrecords = len(self.data['timestamps'].value)
233 self.ranges = self.data['range'].value
233 self.ranges = self.data['range'].value
234
234
235 def setNextFile(self):
235 def setNextFile(self):
236 '''
236 '''
237 '''
237 '''
238
238
239 file_id = self.fileId
239 file_id = self.fileId
240
240
241 if file_id == len(self.fileList):
241 if file_id == len(self.fileList):
242 log.success('No more files', 'MADReader')
242 log.success('No more files', 'MADReader')
243 self.flagNoMoreFiles = 1
243 self.flagNoMoreFiles = 1
244 return 0
244 return 0
245
245
246 log.success(
246 log.success(
247 'Opening: {}'.format(self.fileList[file_id]),
247 'Opening: {}'.format(self.fileList[file_id]),
248 'MADReader'
248 'MADReader'
249 )
249 )
250
250
251 filename = os.path.join(self.path, self.fileList[file_id])
251 filename = os.path.join(self.path, self.fileList[file_id])
252
252
253 if self.filename is not None:
253 if self.filename is not None:
254 self.fp.close()
254 self.fp.close()
255
255
256 self.filename = filename
256 self.filename = filename
257 self.filedate = self.dateFileList[file_id]
257 self.filedate = self.dateFileList[file_id]
258
258
259 if self.ext=='.hdf5':
259 if self.ext=='.hdf5':
260 self.fp = h5py.File(self.filename, 'r')
260 self.fp = h5py.File(self.filename, 'r')
261 else:
261 else:
262 self.fp = open(self.filename, 'rb')
262 self.fp = open(self.filename, 'rb')
263
263
264 self.parseHeader()
264 self.parseHeader()
265 self.parseData()
265 self.parseData()
266 self.sizeOfFile = os.path.getsize(self.filename)
266 self.sizeOfFile = os.path.getsize(self.filename)
267 self.counter_records = 0
267 self.counter_records = 0
268 self.flagIsNewFile = 0
268 self.flagIsNewFile = 0
269 self.fileId += 1
269 self.fileId += 1
270
270
271 return 1
271 return 1
272
272
273 def readNextBlock(self):
273 def readNextBlock(self):
274
274
275 while True:
275 while True:
276 self.flagDiscontinuousBlock = 0
276 self.flagDiscontinuousBlock = 0
277 if self.flagIsNewFile:
277 if self.flagIsNewFile:
278 if not self.setNextFile():
278 if not self.setNextFile():
279 return 0
279 return 0
280
280
281 self.readBlock()
281 self.readBlock()
282
282
283 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
283 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
284 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
284 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
285 log.warning(
285 log.warning(
286 'Reading Record No. {}/{} -> {} [Skipping]'.format(
286 'Reading Record No. {}/{} -> {} [Skipping]'.format(
287 self.counter_records,
287 self.counter_records,
288 self.nrecords,
288 self.nrecords,
289 self.datatime.ctime()),
289 self.datatime.ctime()),
290 'MADReader')
290 'MADReader')
291 continue
291 continue
292 break
292 break
293
293
294 log.log(
294 log.log(
295 'Reading Record No. {}/{} -> {}'.format(
295 'Reading Record No. {}/{} -> {}'.format(
296 self.counter_records,
296 self.counter_records,
297 self.nrecords,
297 self.nrecords,
298 self.datatime.ctime()),
298 self.datatime.ctime()),
299 'MADReader')
299 'MADReader')
300
300
301 return 1
301 return 1
302
302
303 def readBlock(self):
303 def readBlock(self):
304 '''
304 '''
305 '''
305 '''
306 dum = []
306 dum = []
307 if self.ext == '.txt':
307 if self.ext == '.txt':
308 dt = self.data[self.counter_records][:6].astype(int)
308 dt = self.data[self.counter_records][:6].astype(int)
309 if datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]).date() > self.datatime.date():
309 if datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]).date() > self.datatime.date():
310 self.flagDiscontinuousBlock = 1
310 self.flagDiscontinuousBlock = 1
311 self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
311 self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
312 while True:
312 while True:
313 dt = self.data[self.counter_records][:6].astype(int)
313 dt = self.data[self.counter_records][:6].astype(int)
314 datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
314 datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
315 if datatime == self.datatime:
315 if datatime == self.datatime:
316 dum.append(self.data[self.counter_records])
316 dum.append(self.data[self.counter_records])
317 self.counter_records += 1
317 self.counter_records += 1
318 if self.counter_records == self.nrecords:
318 if self.counter_records == self.nrecords:
319 self.flagIsNewFile = True
319 self.flagIsNewFile = True
320 break
320 break
321 continue
321 continue
322 self.intervals.add((datatime-self.datatime).seconds)
322 self.intervals.add((datatime-self.datatime).seconds)
323 break
323 break
324 elif self.ext == '.hdf5':
324 elif self.ext == '.hdf5':
325 datatime = datetime.datetime.utcfromtimestamp(
325 datatime = datetime.datetime.utcfromtimestamp(
326 self.data['timestamps'][self.counter_records])
326 self.data['timestamps'][self.counter_records])
327 nHeights = len(self.ranges)
327 nHeights = len(self.ranges)
328 for n, param in enumerate(self.parameters):
328 for n, param in enumerate(self.parameters):
329 if self.parameters_d[n] == 1:
329 if self.parameters_d[n] == 1:
330 dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
330 dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
331 else:
331 else:
332 if self.version == '2':
332 if self.version == '2':
333 dum.append(self.data['2D Parameters'][param][self.counter_records])
333 dum.append(self.data['2D Parameters'][param][self.counter_records])
334 else:
334 else:
335 tmp = self.data['2D Parameters'][param].value.T
335 tmp = self.data['2D Parameters'][param].value.T
336 dum.append(tmp[self.counter_records])
336 dum.append(tmp[self.counter_records])
337 self.intervals.add((datatime-self.datatime).seconds)
337 self.intervals.add((datatime-self.datatime).seconds)
338 if datatime.date()>self.datatime.date():
338 if datatime.date()>self.datatime.date():
339 self.flagDiscontinuousBlock = 1
339 self.flagDiscontinuousBlock = 1
340 self.datatime = datatime
340 self.datatime = datatime
341 self.counter_records += 1
341 self.counter_records += 1
342 if self.counter_records == self.nrecords:
342 if self.counter_records == self.nrecords:
343 self.flagIsNewFile = True
343 self.flagIsNewFile = True
344
344
345 self.buffer = numpy.array(dum)
345 self.buffer = numpy.array(dum)
346 return
346 return
347
347
348 def set_output(self):
348 def set_output(self):
349 '''
349 '''
350 Storing data from buffer to dataOut object
350 Storing data from buffer to dataOut object
351 '''
351 '''
352
352
353 parameters = [None for __ in self.parameters]
353 parameters = [None for __ in self.parameters]
354
354
355 for param, attr in list(self.oneDDict.items()):
355 for param, attr in list(self.oneDDict.items()):
356 x = self.parameters.index(param.lower())
356 x = self.parameters.index(param.lower())
357 setattr(self.dataOut, attr, self.buffer[0][x])
357 setattr(self.dataOut, attr, self.buffer[0][x])
358
358
359 for param, value in list(self.twoDDict.items()):
359 for param, value in list(self.twoDDict.items()):
360 x = self.parameters.index(param.lower())
360 x = self.parameters.index(param.lower())
361 if self.ext == '.txt':
361 if self.ext == '.txt':
362 y = self.parameters.index(self.ind2DList[0].lower())
362 y = self.parameters.index(self.ind2DList[0].lower())
363 ranges = self.buffer[:,y]
363 ranges = self.buffer[:,y]
364 if self.ranges.size == ranges.size:
364 if self.ranges.size == ranges.size:
365 continue
365 continue
366 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
366 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
367 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
367 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
368 dummy[index] = self.buffer[:,x]
368 dummy[index] = self.buffer[:,x]
369 else:
369 else:
370 dummy = self.buffer[x]
370 dummy = self.buffer[x]
371
371
372 if isinstance(value, str):
372 if isinstance(value, str):
373 if value not in self.ind2DList:
373 if value not in self.ind2DList:
374 setattr(self.dataOut, value, dummy.reshape(1,-1))
374 setattr(self.dataOut, value, dummy.reshape(1,-1))
375 elif isinstance(value, list):
375 elif isinstance(value, list):
376 self.output[value[0]][value[1]] = dummy
376 self.output[value[0]][value[1]] = dummy
377 parameters[value[1]] = param
377 parameters[value[1]] = param
378
378
379 for key, value in list(self.output.items()):
379 for key, value in list(self.output.items()):
380 setattr(self.dataOut, key, numpy.array(value))
380 setattr(self.dataOut, key, numpy.array(value))
381
381
382 self.dataOut.parameters = [s for s in parameters if s]
382 self.dataOut.parameters = [s for s in parameters if s]
383 self.dataOut.heightList = self.ranges
383 self.dataOut.heightList = self.ranges
384 self.dataOut.utctime = (self.datatime - datetime.datetime(1970, 1, 1)).total_seconds()
384 self.dataOut.utctime = (self.datatime - datetime.datetime(1970, 1, 1)).total_seconds()
385 self.dataOut.utctimeInit = self.dataOut.utctime
385 self.dataOut.utctimeInit = self.dataOut.utctime
386 self.dataOut.paramInterval = min(self.intervals)
386 self.dataOut.paramInterval = min(self.intervals)
387 self.dataOut.useLocalTime = False
387 self.dataOut.useLocalTime = False
388 self.dataOut.flagNoData = False
388 self.dataOut.flagNoData = False
389 self.dataOut.nrecords = self.nrecords
389 self.dataOut.nrecords = self.nrecords
390 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
390 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
391
391
392 def getData(self):
392 def getData(self):
393 '''
393 '''
394 Storing data from databuffer to dataOut object
394 Storing data from databuffer to dataOut object
395 '''
395 '''
396 if self.flagNoMoreFiles:
396 if self.flagNoMoreFiles:
397 self.dataOut.flagNoData = True
397 self.dataOut.flagNoData = True
398 log.error('No file left to process', 'MADReader')
398 log.error('No file left to process', 'MADReader')
399 return 0
399 return 0
400
400
401 if not self.readNextBlock():
401 if not self.readNextBlock():
402 self.dataOut.flagNoData = True
402 self.dataOut.flagNoData = True
403 return 0
403 return 0
404
404
405 self.set_output()
405 self.set_output()
406
406
407 return 1
407 return 1
408
408
409
409
410 class MADWriter(Operation):
410 class MADWriter(Operation):
411
411
412 missing = -32767
412 missing = -32767
413
413
414 def __init__(self, **kwargs):
414 def __init__(self, **kwargs):
415
415
416 Operation.__init__(self, **kwargs)
416 Operation.__init__(self, **kwargs)
417 self.dataOut = Parameters()
417 self.dataOut = Parameters()
418 self.counter = 0
418 self.counter = 0
419 self.path = None
419 self.path = None
420 self.fp = None
420 self.fp = None
421
421
422 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
422 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
423 metadata='{}', format='cedar', **kwargs):
423 metadata='{}', format='cedar', **kwargs):
424 '''
424 '''
425 Inputs:
425 Inputs:
426 path - path where files will be created
426 path - path where files will be created
427 oneDDict - json of one-dimensional parameters in record where keys
427 oneDDict - json of one-dimensional parameters in record where keys
428 are Madrigal codes (integers or mnemonics) and values the corresponding
428 are Madrigal codes (integers or mnemonics) and values the corresponding
429 dataOut attribute e.g: {
429 dataOut attribute e.g: {
430 'gdlatr': 'lat',
430 'gdlatr': 'lat',
431 'gdlonr': 'lon',
431 'gdlonr': 'lon',
432 'gdlat2':'lat',
432 'gdlat2':'lat',
433 'glon2':'lon'}
433 'glon2':'lon'}
434 ind2DList - list of independent spatial two-dimensional parameters e.g:
434 ind2DList - list of independent spatial two-dimensional parameters e.g:
435 ['heighList']
435 ['heighList']
436 twoDDict - json of two-dimensional parameters in record where keys
436 twoDDict - json of two-dimensional parameters in record where keys
437 are Madrigal codes (integers or mnemonics) and values the corresponding
437 are Madrigal codes (integers or mnemonics) and values the corresponding
438 dataOut attribute if multidimensional array specify as tupple
438 dataOut attribute if multidimensional array specify as tupple
439 ('attr', pos) e.g: {
439 ('attr', pos) e.g: {
440 'gdalt': 'heightList',
440 'gdalt': 'heightList',
441 'vn1p2': ('data_output', 0),
441 'vn1p2': ('data_output', 0),
442 'vn2p2': ('data_output', 1),
442 'vn2p2': ('data_output', 1),
443 'vn3': ('data_output', 2),
443 'vn3': ('data_output', 2),
444 'snl': ('data_SNR', 'db')
444 'snl': ('data_SNR', 'db')
445 }
445 }
446 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
446 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
447 '''
447 '''
448 if not self.isConfig:
448 if not self.isConfig:
449 self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
449 self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
450 self.isConfig = True
450 self.isConfig = True
451
451
452 self.dataOut = dataOut
452 self.dataOut = dataOut
453 self.putData()
453 self.putData()
454 return
454 return
455
455
456 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
456 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
457 '''
457 '''
458 Configure Operation
458 Configure Operation
459 '''
459 '''
460
460
461 self.path = path
461 self.path = path
462 self.blocks = kwargs.get('blocks', None)
462 self.blocks = kwargs.get('blocks', None)
463 self.counter = 0
463 self.counter = 0
464 self.oneDDict = load_json(oneDDict)
464 self.oneDDict = load_json(oneDDict)
465 self.twoDDict = load_json(twoDDict)
465 self.twoDDict = load_json(twoDDict)
466 self.ind2DList = load_json(ind2DList)
466 self.ind2DList = load_json(ind2DList)
467 meta = load_json(metadata)
467 meta = load_json(metadata)
468 self.kinst = meta.get('kinst')
468 self.kinst = meta.get('kinst')
469 self.kindat = meta.get('kindat')
469 self.kindat = meta.get('kindat')
470 self.catalog = meta.get('catalog', DEF_CATALOG)
470 self.catalog = meta.get('catalog', DEF_CATALOG)
471 self.header = meta.get('header', DEF_HEADER)
471 self.header = meta.get('header', DEF_HEADER)
472 if format == 'cedar':
472 if format == 'cedar':
473 self.ext = '.dat'
473 self.ext = '.dat'
474 self.extra_args = {}
474 self.extra_args = {}
475 elif format == 'hdf5':
475 elif format == 'hdf5':
476 self.ext = '.hdf5'
476 self.ext = '.hdf5'
477 self.extra_args = {'ind2DList': self.ind2DList}
477 self.extra_args = {'ind2DList': self.ind2DList}
478
478
479 self.keys = [k.lower() for k in self.twoDDict]
479 self.keys = [k.lower() for k in self.twoDDict]
480 if 'range' in self.keys:
480 if 'range' in self.keys:
481 self.keys.remove('range')
481 self.keys.remove('range')
482 if 'gdalt' in self.keys:
482 if 'gdalt' in self.keys:
483 self.keys.remove('gdalt')
483 self.keys.remove('gdalt')
484
484
485 def setFile(self):
485 def setFile(self):
486 '''
486 '''
487 Create new cedar file object
487 Create new cedar file object
488 '''
488 '''
489
489
490 self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
490 self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
491 date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
491 date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
492
492
493 filename = '{}{}{}'.format(self.mnemonic,
493 filename = '{}{}{}'.format(self.mnemonic,
494 date.strftime('%Y%m%d_%H%M%S'),
494 date.strftime('%Y%m%d_%H%M%S'),
495 self.ext)
495 self.ext)
496
496
497 self.fullname = os.path.join(self.path, filename)
497 self.fullname = os.path.join(self.path, filename)
498
498
499 if os.path.isfile(self.fullname) :
499 if os.path.isfile(self.fullname) :
500 log.warning(
500 log.warning(
501 'Destination file {} already exists, previous file deleted.'.format(
501 'Destination file {} already exists, previous file deleted.'.format(
502 self.fullname),
502 self.fullname),
503 'MADWriter')
503 'MADWriter')
504 os.remove(self.fullname)
504 os.remove(self.fullname)
505
505
506 try:
506 try:
507 log.success(
507 log.success(
508 'Creating file: {}'.format(self.fullname),
508 'Creating file: {}'.format(self.fullname),
509 'MADWriter')
509 'MADWriter')
510 self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
510 self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
511 except ValueError as e:
511 except ValueError as e:
512 log.error(
512 log.error(
513 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
513 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
514 'MADWriter')
514 'MADWriter')
515 return
515 return
516
516
517 return 1
517 return 1
518
518
519 def writeBlock(self):
519 def writeBlock(self):
520 '''
520 '''
521 Add data records to cedar file taking data from oneDDict and twoDDict
521 Add data records to cedar file taking data from oneDDict and twoDDict
522 attributes.
522 attributes.
523 Allowed parameters in: parcodes.tab
523 Allowed parameters in: parcodes.tab
524 '''
524 '''
525
525
526 startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
526 startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
527 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
527 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
528 heights = self.dataOut.heightList
528 heights = self.dataOut.heightList
529
529
530 if self.ext == '.dat':
530 if self.ext == '.dat':
531 for key, value in list(self.twoDDict.items()):
531 for key, value in list(self.twoDDict.items()):
532 if isinstance(value, str):
532 if isinstance(value, str):
533 data = getattr(self.dataOut, value)
533 data = getattr(self.dataOut, value)
534 invalid = numpy.isnan(data)
534 invalid = numpy.isnan(data)
535 data[invalid] = self.missing
535 data[invalid] = self.missing
536 elif isinstance(value, (tuple, list)):
536 elif isinstance(value, (tuple, list)):
537 attr, key = value
537 attr, key = value
538 data = getattr(self.dataOut, attr)
538 data = getattr(self.dataOut, attr)
539 invalid = numpy.isnan(data)
539 invalid = numpy.isnan(data)
540 data[invalid] = self.missing
540 data[invalid] = self.missing
541
541
542 out = {}
542 out = {}
543 for key, value in list(self.twoDDict.items()):
543 for key, value in list(self.twoDDict.items()):
544 key = key.lower()
544 key = key.lower()
545 if isinstance(value, str):
545 if isinstance(value, str):
546 if 'db' in value.lower():
546 if 'db' in value.lower():
547 tmp = getattr(self.dataOut, value.replace('_db', ''))
547 tmp = getattr(self.dataOut, value.replace('_db', ''))
548 SNRavg = numpy.average(tmp, axis=0)
548 SNRavg = numpy.average(tmp, axis=0)
549 tmp = 10*numpy.log10(SNRavg)
549 tmp = 10*numpy.log10(SNRavg)
550 else:
550 else:
551 tmp = getattr(self.dataOut, value)
551 tmp = getattr(self.dataOut, value)
552 out[key] = tmp.flatten()
552 out[key] = tmp.flatten()
553 elif isinstance(value, (tuple, list)):
553 elif isinstance(value, (tuple, list)):
554 attr, x = value
554 attr, x = value
555 data = getattr(self.dataOut, attr)
555 data = getattr(self.dataOut, attr)
556 out[key] = data[int(x)]
556 out[key] = data[int(x)]
557
557
558 a = numpy.array([out[k] for k in self.keys])
558 a = numpy.array([out[k] for k in self.keys])
559 nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
559 nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
560 index = numpy.where(nrows == False)[0]
560 index = numpy.where(nrows == False)[0]
561
561
562 rec = madrigal.cedar.MadrigalDataRecord(
562 rec = madrigal.cedar.MadrigalDataRecord(
563 self.kinst,
563 self.kinst,
564 self.kindat,
564 self.kindat,
565 startTime.year,
565 startTime.year,
566 startTime.month,
566 startTime.month,
567 startTime.day,
567 startTime.day,
568 startTime.hour,
568 startTime.hour,
569 startTime.minute,
569 startTime.minute,
570 startTime.second,
570 startTime.second,
571 startTime.microsecond/10000,
571 startTime.microsecond/10000,
572 endTime.year,
572 endTime.year,
573 endTime.month,
573 endTime.month,
574 endTime.day,
574 endTime.day,
575 endTime.hour,
575 endTime.hour,
576 endTime.minute,
576 endTime.minute,
577 endTime.second,
577 endTime.second,
578 endTime.microsecond/10000,
578 endTime.microsecond/10000,
579 list(self.oneDDict.keys()),
579 list(self.oneDDict.keys()),
580 list(self.twoDDict.keys()),
580 list(self.twoDDict.keys()),
581 len(index),
581 len(index),
582 **self.extra_args
582 **self.extra_args
583 )
583 )
584
584
585 # Setting 1d values
585 # Setting 1d values
586 for key in self.oneDDict:
586 for key in self.oneDDict:
587 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
587 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
588
588
589 # Setting 2d values
589 # Setting 2d values
590 nrec = 0
590 nrec = 0
591 for n in index:
591 for n in index:
592 for key in out:
592 for key in out:
593 rec.set2D(key, nrec, out[key][n])
593 rec.set2D(key, nrec, out[key][n])
594 nrec += 1
594 nrec += 1
595
595
596 self.fp.append(rec)
596 self.fp.append(rec)
597 if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
597 if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
598 self.fp.dump()
598 self.fp.dump()
599 if self.counter % 100 == 0 and self.counter > 0:
599 if self.counter % 100 == 0 and self.counter > 0:
600 log.log(
600 log.log(
601 'Writing {} records'.format(
601 'Writing {} records'.format(
602 self.counter),
602 self.counter),
603 'MADWriter')
603 'MADWriter')
604
604
605 def setHeader(self):
605 def setHeader(self):
606 '''
606 '''
607 Create an add catalog and header to cedar file
607 Create an add catalog and header to cedar file
608 '''
608 '''
609
609
610 log.success('Closing file {}'.format(self.fullname), 'MADWriter')
610 log.success('Closing file {}'.format(self.fullname), 'MADWriter')
611
611
612 if self.ext == '.dat':
612 if self.ext == '.dat':
613 self.fp.write()
613 self.fp.write()
614 else:
614 else:
615 self.fp.dump()
615 self.fp.dump()
616 self.fp.close()
616 self.fp.close()
617
617
618 header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
618 header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
619 header.createCatalog(**self.catalog)
619 header.createCatalog(**self.catalog)
620 header.createHeader(**self.header)
620 header.createHeader(**self.header)
621 header.write()
621 header.write()
622
622
623 def putData(self):
623 def putData(self):
624
624
625 if self.dataOut.flagNoData:
625 if self.dataOut.flagNoData:
626 return 0
626 return 0
627
627
628 if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
628 if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
629 if self.counter > 0:
629 if self.counter > 0:
630 self.setHeader()
630 self.setHeader()
631 self.counter = 0
631 self.counter = 0
632
632
633 if self.counter == 0:
633 if self.counter == 0:
634 self.setFile()
634 self.setFile()
635
635
636 self.writeBlock()
636 self.writeBlock()
637 self.counter += 1
637 self.counter += 1
638
638
639 def close(self):
639 def close(self):
640
640
641 if self.counter > 0:
641 if self.counter > 0:
642 self.setHeader() No newline at end of file
642 self.setHeader()
General Comments 0
You need to be logged in to leave comments. Login now