##// END OF EJS Templates
schain-jc
RSS
Fix bugs in madrigal module
Juan C. Espinoza -
r1073:9fc44081e2dc
parent child
Show More
Side by Side Unified
Context file:
r1073:9fc44081e2dc Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,632 +1,641
1 '''
1 '''
2 Created on Aug 1, 2017
2 Created on Aug 1, 2017
3
3
4 @author: Juan C. Espinoza
4 @author: Juan C. Espinoza
5 '''
5 '''
6
6
7 import os
7 import os
8 import sys
8 import sys
9 import time
9 import time
10 import json
10 import json
11 import glob
11 import glob
12 import datetime
12 import datetime
13
13
14 import numpy
14 import numpy
15 import h5py
15 import h5py
16
16
17 from schainpy.model.io.jroIO_base import JRODataReader
17 from schainpy.model.io.jroIO_base import JRODataReader
18 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
18 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
19 from schainpy.model.data.jrodata import Parameters
19 from schainpy.model.data.jrodata import Parameters
20 from schainpy.utils import log
20 from schainpy.utils import log
21
21
22 try:
22 try:
23 import madrigal.cedar
23 import madrigal.cedar
24 except:
24 except:
25 log.warning(
25 log.warning(
26 'You should install "madrigal library" module if you want to read/write Madrigal data'
26 'You should install "madrigal library" module if you want to read/write Madrigal data'
27 )
27 )
28
28
29 DEF_CATALOG = {
29 DEF_CATALOG = {
30 'principleInvestigator': 'Marco Milla',
30 'principleInvestigator': 'Marco Milla',
31 'expPurpose': None,
31 'expPurpose': None,
32 'cycleTime': None,
32 'cycleTime': None,
33 'correlativeExp': None,
33 'correlativeExp': None,
34 'sciRemarks': None,
34 'sciRemarks': None,
35 'instRemarks': None
35 'instRemarks': None
36 }
36 }
37 DEF_HEADER = {
37 DEF_HEADER = {
38 'kindatDesc': None,
38 'kindatDesc': None,
39 'analyst': 'Jicamarca User',
39 'analyst': 'Jicamarca User',
40 'comments': None,
40 'comments': None,
41 'history': None
41 'history': None
42 }
42 }
43 MNEMONICS = {
43 MNEMONICS = {
44 10: 'jro',
44 10: 'jro',
45 11: 'jbr',
45 11: 'jbr',
46 840: 'jul',
46 840: 'jul',
47 13: 'jas',
47 13: 'jas',
48 1000: 'pbr',
48 1000: 'pbr',
49 1001: 'hbr',
49 1001: 'hbr',
50 1002: 'obr',
50 1002: 'obr',
51 }
51 }
52
52
53 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
53 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
54
54
55 def load_json(obj):
55 def load_json(obj):
56 '''
56 '''
57 Parse json as string instead of unicode
57 Parse json as string instead of unicode
58 '''
58 '''
59
59
60 if isinstance(obj, str):
60 if isinstance(obj, str):
61 iterable = json.loads(obj)
61 iterable = json.loads(obj)
62 else:
62 else:
63 iterable = obj
63 iterable = obj
64
64
65 if isinstance(iterable, dict):
65 if isinstance(iterable, dict):
66 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
66 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
67 for k, v in iterable.items()}
67 for k, v in iterable.items()}
68 elif isinstance(iterable, (list, tuple)):
68 elif isinstance(iterable, (list, tuple)):
69 return [str(v) if isinstance(v, unicode) else v for v in iterable]
69 return [str(v) if isinstance(v, unicode) else v for v in iterable]
70
70
71 return iterable
71 return iterable
72
72
73
73
74 class MADReader(JRODataReader, ProcessingUnit):
74 class MADReader(JRODataReader, ProcessingUnit):
75
75
76 def __init__(self, **kwargs):
76 def __init__(self, **kwargs):
77
77
78 ProcessingUnit.__init__(self, **kwargs)
78 ProcessingUnit.__init__(self, **kwargs)
79
79
80 self.dataOut = Parameters()
80 self.dataOut = Parameters()
81 self.counter_records = 0
81 self.counter_records = 0
82 self.nrecords = None
82 self.nrecords = None
83 self.flagNoMoreFiles = 0
83 self.flagNoMoreFiles = 0
84 self.isConfig = False
84 self.isConfig = False
85 self.filename = None
85 self.filename = None
86 self.intervals = set()
86 self.intervals = set()
87
87
88 def setup(self,
88 def setup(self,
89 path=None,
89 path=None,
90 startDate=None,
90 startDate=None,
91 endDate=None,
91 endDate=None,
92 format=None,
92 format=None,
93 startTime=datetime.time(0, 0, 0),
93 startTime=datetime.time(0, 0, 0),
94 endTime=datetime.time(23, 59, 59),
94 endTime=datetime.time(23, 59, 59),
95 **kwargs):
95 **kwargs):
96
96
97 self.path = path
97 self.path = path
98 self.startDate = startDate
98 self.startDate = startDate
99 self.endDate = endDate
99 self.endDate = endDate
100 self.startTime = startTime
100 self.startTime = startTime
101 self.endTime = endTime
101 self.endTime = endTime
102 self.datatime = datetime.datetime(1900,1,1)
102 self.datatime = datetime.datetime(1900,1,1)
103 self.oneDDict = load_json(kwargs.get('oneDDict',
103 self.oneDDict = load_json(kwargs.get('oneDDict',
104 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
104 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
105 self.twoDDict = load_json(kwargs.get('twoDDict',
105 self.twoDDict = load_json(kwargs.get('twoDDict',
106 "{\"GDALT\": \"heightList\"}"))
106 "{\"GDALT\": \"heightList\"}"))
107 self.ind2DList = load_json(kwargs.get('ind2DList',
107 self.ind2DList = load_json(kwargs.get('ind2DList',
108 "[\"GDALT\"]"))
108 "[\"GDALT\"]"))
109 if self.path is None:
109 if self.path is None:
110 raise ValueError, 'The path is not valid'
110 raise ValueError, 'The path is not valid'
111
111
112 if format is None:
112 if format is None:
113 raise ValueError, 'The format is not valid choose simple or hdf5'
113 raise ValueError, 'The format is not valid choose simple or hdf5'
114 elif format.lower() in ('simple', 'txt'):
114 elif format.lower() in ('simple', 'txt'):
115 self.ext = '.txt'
115 self.ext = '.txt'
116 elif format.lower() in ('cedar',):
116 elif format.lower() in ('cedar',):
117 self.ext = '.001'
117 self.ext = '.001'
118 else:
118 else:
119 self.ext = '.hdf5'
119 self.ext = '.hdf5'
120
120
121 self.search_files(self.path)
121 self.search_files(self.path)
122 self.fileId = 0
122 self.fileId = 0
123
123
124 if not self.fileList:
124 if not self.fileList:
125 raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
125 raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
126
126
127 self.setNextFile()
127 self.setNextFile()
128
128
129 def search_files(self, path):
129 def search_files(self, path):
130 '''
130 '''
131 Searching for madrigal files in path
131 Searching for madrigal files in path
132 Creating a list of files to procces included in [startDate,endDate]
132 Creating a list of files to procces included in [startDate,endDate]
133
133
134 Input:
134 Input:
135 path - Path to find files
135 path - Path to find files
136 '''
136 '''
137
137
138 log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
138 log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
139 foldercounter = 0
139 foldercounter = 0
140 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
140 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
141 fileList0.sort()
141 fileList0.sort()
142
142
143 self.fileList = []
143 self.fileList = []
144 self.dateFileList = []
144 self.dateFileList = []
145
145
146 startDate = self.startDate - datetime.timedelta(1)
146 startDate = self.startDate - datetime.timedelta(1)
147 endDate = self.endDate + datetime.timedelta(1)
147 endDate = self.endDate + datetime.timedelta(1)
148
148
149 for thisFile in fileList0:
149 for thisFile in fileList0:
150 year = thisFile[3:7]
150 year = thisFile[3:7]
151 if not year.isdigit():
151 if not year.isdigit():
152 continue
152 continue
153
153
154 month = thisFile[7:9]
154 month = thisFile[7:9]
155 if not month.isdigit():
155 if not month.isdigit():
156 continue
156 continue
157
157
158 day = thisFile[9:11]
158 day = thisFile[9:11]
159 if not day.isdigit():
159 if not day.isdigit():
160 continue
160 continue
161
161
162 year, month, day = int(year), int(month), int(day)
162 year, month, day = int(year), int(month), int(day)
163 dateFile = datetime.date(year, month, day)
163 dateFile = datetime.date(year, month, day)
164
164
165 if (startDate > dateFile) or (endDate < dateFile):
165 if (startDate > dateFile) or (endDate < dateFile):
166 continue
166 continue
167
167
168 self.fileList.append(thisFile)
168 self.fileList.append(thisFile)
169 self.dateFileList.append(dateFile)
169 self.dateFileList.append(dateFile)
170
170
171 return
171 return
172
172
173 def parseHeader(self):
173 def parseHeader(self):
174 '''
174 '''
175 '''
175 '''
176
176
177 self.output = {}
177 self.output = {}
178 self.version = '2'
178 self.version = '2'
179 s_parameters = None
179 s_parameters = None
180 if self.ext == '.txt':
180 if self.ext == '.txt':
181 self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
181 self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
182 elif self.ext == '.hdf5':
182 elif self.ext == '.hdf5':
183 metadata = self.fp['Metadata']
183 metadata = self.fp['Metadata']
184 data = self.fp['Data']['Array Layout']
184 data = self.fp['Data']['Array Layout']
185 if 'Independent Spatial Parameters' in metadata:
185 if 'Independent Spatial Parameters' in metadata:
186 s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
186 s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
187 self.version = '3'
187 self.version = '3'
188 one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
188 one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
189 one_d = [1 for s in one]
189 one_d = [1 for s in one]
190 two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
190 two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
191 two_d = [2 for s in two]
191 two_d = [2 for s in two]
192 self.parameters = one + two
192 self.parameters = one + two
193 self.parameters_d = one_d + two_d
193 self.parameters_d = one_d + two_d
194
194
195 log.success('Parameters found: {}'.format(','.join(self.parameters)),
195 log.success('Parameters found: {}'.format(','.join(self.parameters)),
196 'MADReader')
196 'MADReader')
197 if s_parameters:
197 if s_parameters:
198 log.success('Spatial parameters: {}'.format(','.join(s_parameters)),
198 log.success('Spatial parameters: {}'.format(','.join(s_parameters)),
199 'MADReader')
199 'MADReader')
200
200
201 for param in self.oneDDict.keys():
201 for param in self.oneDDict.keys():
202 if param.lower() not in self.parameters:
202 if param.lower() not in self.parameters:
203 log.warning(
203 log.warning(
204 'Parameter {} not found will be ignored'.format(
204 'Parameter {} not found will be ignored'.format(
205 param),
205 param),
206 'MADReader')
206 'MADReader')
207 self.oneDDict.pop(param, None)
207 self.oneDDict.pop(param, None)
208
208
209 for param, value in self.twoDDict.items():
209 for param, value in self.twoDDict.items():
210 if param.lower() not in self.parameters:
210 if param.lower() not in self.parameters:
211 log.warning(
211 log.warning(
212 'Parameter {} not found, it will be ignored'.format(
212 'Parameter {} not found, it will be ignored'.format(
213 param),
213 param),
214 'MADReader')
214 'MADReader')
215 self.twoDDict.pop(param, None)
215 self.twoDDict.pop(param, None)
216 continue
216 continue
217 if isinstance(value, list):
217 if isinstance(value, list):
218 if value[0] not in self.output:
218 if value[0] not in self.output:
219 self.output[value[0]] = []
219 self.output[value[0]] = []
220 self.output[value[0]].append(None)
220 self.output[value[0]].append(None)
221
221
222 def parseData(self):
222 def parseData(self):
223 '''
223 '''
224 '''
224 '''
225
225
226 if self.ext == '.txt':
226 if self.ext == '.txt':
227 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
227 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
228 self.nrecords = self.data.shape[0]
228 self.nrecords = self.data.shape[0]
229 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
229 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
230 elif self.ext == '.hdf5':
230 elif self.ext == '.hdf5':
231 self.data = self.fp['Data']['Array Layout']
231 self.data = self.fp['Data']['Array Layout']
232 self.nrecords = len(self.data['timestamps'].value)
232 self.nrecords = len(self.data['timestamps'].value)
233 self.ranges = self.data['range'].value
233 self.ranges = self.data['range'].value
234
234
235 def setNextFile(self):
235 def setNextFile(self):
236 '''
236 '''
237 '''
237 '''
238
238
239 file_id = self.fileId
239 file_id = self.fileId
240
240
241 if file_id == len(self.fileList):
241 if file_id == len(self.fileList):
242 log.success('No more files', 'MADReader')
242 log.success('No more files', 'MADReader')
243 self.flagNoMoreFiles = 1
243 self.flagNoMoreFiles = 1
244 return 0
244 return 0
245
245
246 log.success(
246 log.success(
247 'Opening: {}'.format(self.fileList[file_id]),
247 'Opening: {}'.format(self.fileList[file_id]),
248 'MADReader'
248 'MADReader'
249 )
249 )
250
250
251 filename = os.path.join(self.path, self.fileList[file_id])
251 filename = os.path.join(self.path, self.fileList[file_id])
252
252
253 if self.filename is not None:
253 if self.filename is not None:
254 self.fp.close()
254 self.fp.close()
255
255
256 self.filename = filename
256 self.filename = filename
257 self.filedate = self.dateFileList[file_id]
257 self.filedate = self.dateFileList[file_id]
258
258
259 if self.ext=='.hdf5':
259 if self.ext=='.hdf5':
260 self.fp = h5py.File(self.filename, 'r')
260 self.fp = h5py.File(self.filename, 'r')
261 else:
261 else:
262 self.fp = open(self.filename, 'rb')
262 self.fp = open(self.filename, 'rb')
263
263
264 self.parseHeader()
264 self.parseHeader()
265 self.parseData()
265 self.parseData()
266 self.sizeOfFile = os.path.getsize(self.filename)
266 self.sizeOfFile = os.path.getsize(self.filename)
267 self.counter_records = 0
267 self.counter_records = 0
268 self.flagIsNewFile = 0
268 self.flagIsNewFile = 0
269 self.fileId += 1
269 self.fileId += 1
270
270
271 return 1
271 return 1
272
272
273 def readNextBlock(self):
273 def readNextBlock(self):
274
274
275 while True:
275 while True:
276 self.flagDiscontinuousBlock = 0
276 self.flagDiscontinuousBlock = 0
277 if self.flagIsNewFile:
277 if self.flagIsNewFile:
278 if not self.setNextFile():
278 if not self.setNextFile():
279 return 0
279 return 0
280
280
281 self.readBlock()
281 self.readBlock()
282
282
283 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
283 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
284 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
284 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
285 log.warning(
285 log.warning(
286 'Reading Record No. {}/{} -> {} [Skipping]'.format(
286 'Reading Record No. {}/{} -> {} [Skipping]'.format(
287 self.counter_records,
287 self.counter_records,
288 self.nrecords,
288 self.nrecords,
289 self.datatime.ctime()),
289 self.datatime.ctime()),
290 'MADReader')
290 'MADReader')
291 continue
291 continue
292 break
292 break
293
293
294 log.log(
294 log.log(
295 'Reading Record No. {}/{} -> {}'.format(
295 'Reading Record No. {}/{} -> {}'.format(
296 self.counter_records,
296 self.counter_records,
297 self.nrecords,
297 self.nrecords,
298 self.datatime.ctime()),
298 self.datatime.ctime()),
299 'MADReader')
299 'MADReader')
300
300
301 return 1
301 return 1
302
302
303 def readBlock(self):
303 def readBlock(self):
304 '''
304 '''
305 '''
305 '''
306 dum = []
306 dum = []
307 if self.ext == '.txt':
307 if self.ext == '.txt':
308 dt = self.data[self.counter_records][:6].astype(int)
308 dt = self.data[self.counter_records][:6].astype(int)
309 if datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]).date() > self.datatime.date():
310 self.flagDiscontinuousBlock = 1
309 self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
311 self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
310 while True:
312 while True:
311 dt = self.data[self.counter_records][:6].astype(int)
313 dt = self.data[self.counter_records][:6].astype(int)
312 datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
314 datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
313 if datatime == self.datatime:
315 if datatime == self.datatime:
314 dum.append(self.data[self.counter_records])
316 dum.append(self.data[self.counter_records])
315 self.counter_records += 1
317 self.counter_records += 1
316 if self.counter_records == self.nrecords:
318 if self.counter_records == self.nrecords:
317 self.flagIsNewFile = True
319 self.flagIsNewFile = True
318 break
320 break
319 continue
321 continue
320 self.intervals.add((datatime-self.datatime).seconds)
322 self.intervals.add((datatime-self.datatime).seconds)
321 if datatime.date() > self.datatime.date():
322 self.flagDiscontinuousBlock = 1
323 break
323 break
324 elif self.ext == '.hdf5':
324 elif self.ext == '.hdf5':
325 datatime = datetime.datetime.utcfromtimestamp(
325 datatime = datetime.datetime.utcfromtimestamp(
326 self.data['timestamps'][self.counter_records])
326 self.data['timestamps'][self.counter_records])
327 nHeights = len(self.ranges)
327 nHeights = len(self.ranges)
328 for n, param in enumerate(self.parameters):
328 for n, param in enumerate(self.parameters):
329 if self.parameters_d[n] == 1:
329 if self.parameters_d[n] == 1:
330 dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
330 dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
331 else:
331 else:
332 if self.version == '2':
332 if self.version == '2':
333 dum.append(self.data['2D Parameters'][param][self.counter_records])
333 dum.append(self.data['2D Parameters'][param][self.counter_records])
334 else:
334 else:
335 tmp = self.data['2D Parameters'][param].value.T
335 tmp = self.data['2D Parameters'][param].value.T
336 dum.append(tmp[self.counter_records])
336 dum.append(tmp[self.counter_records])
337 self.intervals.add((datatime-self.datatime).seconds)
337 self.intervals.add((datatime-self.datatime).seconds)
338 if datatime.date()>self.datatime.date():
338 if datatime.date()>self.datatime.date():
339 self.flagDiscontinuousBlock = 1
339 self.flagDiscontinuousBlock = 1
340 self.datatime = datatime
340 self.datatime = datatime
341 self.counter_records += 1
341 self.counter_records += 1
342 if self.counter_records == self.nrecords:
342 if self.counter_records == self.nrecords:
343 self.flagIsNewFile = True
343 self.flagIsNewFile = True
344
344
345 self.buffer = numpy.array(dum)
345 self.buffer = numpy.array(dum)
346 return
346 return
347
347
348 def set_output(self):
348 def set_output(self):
349 '''
349 '''
350 Storing data from buffer to dataOut object
350 Storing data from buffer to dataOut object
351 '''
351 '''
352
352
353 parameters = [None for __ in self.parameters]
353 parameters = [None for __ in self.parameters]
354
354
355 for param, attr in self.oneDDict.items():
355 for param, attr in self.oneDDict.items():
356 x = self.parameters.index(param.lower())
356 x = self.parameters.index(param.lower())
357 setattr(self.dataOut, attr, self.buffer[0][x])
357 setattr(self.dataOut, attr, self.buffer[0][x])
358
358
359 for param, value in self.twoDDict.items():
359 for param, value in self.twoDDict.items():
360 x = self.parameters.index(param.lower())
360 x = self.parameters.index(param.lower())
361 if self.ext == '.txt':
361 if self.ext == '.txt':
362 y = self.parameters.index(self.ind2DList[0].lower())
362 y = self.parameters.index(self.ind2DList[0].lower())
363 ranges = self.buffer[:,y]
363 ranges = self.buffer[:,y]
364 if self.ranges.size == ranges.size:
364 if self.ranges.size == ranges.size:
365 continue
365 continue
366 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
366 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
367 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
367 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
368 dummy[index] = self.buffer[:,x]
368 dummy[index] = self.buffer[:,x]
369 else:
369 else:
370 dummy = self.buffer[x]
370 dummy = self.buffer[x]
371
371
372 if isinstance(value, str):
372 if isinstance(value, str):
373 if value not in self.ind2DList:
373 if value not in self.ind2DList:
374 setattr(self.dataOut, value, dummy.reshape(1,-1))
374 setattr(self.dataOut, value, dummy.reshape(1,-1))
375 elif isinstance(value, list):
375 elif isinstance(value, list):
376 self.output[value[0]][value[1]] = dummy
376 self.output[value[0]][value[1]] = dummy
377 parameters[value[1]] = param
377 parameters[value[1]] = param
378
378
379 for key, value in self.output.items():
379 for key, value in self.output.items():
380 setattr(self.dataOut, key, numpy.array(value))
380 setattr(self.dataOut, key, numpy.array(value))
381
381
382 self.dataOut.parameters = [s for s in parameters if s]
382 self.dataOut.parameters = [s for s in parameters if s]
383 self.dataOut.heightList = self.ranges
383 self.dataOut.heightList = self.ranges
384 self.dataOut.utctime = (self.datatime - UT1970).total_seconds()
384 self.dataOut.utctime = (self.datatime - UT1970).total_seconds()
385 self.dataOut.utctimeInit = self.dataOut.utctime
385 self.dataOut.utctimeInit = self.dataOut.utctime
386 self.dataOut.paramInterval = min(self.intervals)
386 self.dataOut.paramInterval = min(self.intervals)
387 self.dataOut.useLocalTime = False
387 self.dataOut.useLocalTime = False
388 self.dataOut.flagNoData = False
388 self.dataOut.flagNoData = False
389 self.dataOut.nrecords = self.nrecords
389 self.dataOut.nrecords = self.nrecords
390 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
390 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
391
391
392 def getData(self):
392 def getData(self):
393 '''
393 '''
394 Storing data from databuffer to dataOut object
394 Storing data from databuffer to dataOut object
395 '''
395 '''
396 if self.flagNoMoreFiles:
396 if self.flagNoMoreFiles:
397 self.dataOut.flagNoData = True
397 self.dataOut.flagNoData = True
398 log.error('No file left to process', 'MADReader')
398 log.error('No file left to process', 'MADReader')
399 return 0
399 return 0
400
400
401 if not self.readNextBlock():
401 if not self.readNextBlock():
402 self.dataOut.flagNoData = True
402 self.dataOut.flagNoData = True
403 return 0
403 return 0
404
404
405 self.set_output()
405 self.set_output()
406
406
407 return 1
407 return 1
408
408
409
409
410 class MADWriter(Operation):
410 class MADWriter(Operation):
411
411
412 missing = -32767
412 missing = -32767
413
413
414 def __init__(self, **kwargs):
414 def __init__(self, **kwargs):
415
415
416 Operation.__init__(self, **kwargs)
416 Operation.__init__(self, **kwargs)
417 self.dataOut = Parameters()
417 self.dataOut = Parameters()
418 self.path = None
418 self.path = None
419 self.fp = None
419 self.fp = None
420
420
421 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
421 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
422 metadata='{}', format='cedar', **kwargs):
422 metadata='{}', format='cedar', **kwargs):
423 '''
423 '''
424 Inputs:
424 Inputs:
425 path - path where files will be created
425 path - path where files will be created
426 oneDDict - json of one-dimensional parameters in record where keys
426 oneDDict - json of one-dimensional parameters in record where keys
427 are Madrigal codes (integers or mnemonics) and values the corresponding
427 are Madrigal codes (integers or mnemonics) and values the corresponding
428 dataOut attribute e.g: {
428 dataOut attribute e.g: {
429 'gdlatr': 'lat',
429 'gdlatr': 'lat',
430 'gdlonr': 'lon',
430 'gdlonr': 'lon',
431 'gdlat2':'lat',
431 'gdlat2':'lat',
432 'glon2':'lon'}
432 'glon2':'lon'}
433 ind2DList - list of independent spatial two-dimensional parameters e.g:
433 ind2DList - list of independent spatial two-dimensional parameters e.g:
434 ['heighList']
434 ['heighList']
435 twoDDict - json of two-dimensional parameters in record where keys
435 twoDDict - json of two-dimensional parameters in record where keys
436 are Madrigal codes (integers or mnemonics) and values the corresponding
436 are Madrigal codes (integers or mnemonics) and values the corresponding
437 dataOut attribute if multidimensional array specify as tupple
437 dataOut attribute if multidimensional array specify as tupple
438 ('attr', pos) e.g: {
438 ('attr', pos) e.g: {
439 'gdalt': 'heightList',
439 'gdalt': 'heightList',
440 'vn1p2': ('data_output', 0),
440 'vn1p2': ('data_output', 0),
441 'vn2p2': ('data_output', 1),
441 'vn2p2': ('data_output', 1),
442 'vn3': ('data_output', 2),
442 'vn3': ('data_output', 2),
443 'snl': ('data_SNR', 'db')
443 'snl': ('data_SNR', 'db')
444 }
444 }
445 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
445 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
446 '''
446 '''
447 if not self.isConfig:
447 if not self.isConfig:
448 self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
448 self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
449 self.isConfig = True
449 self.isConfig = True
450
450
451 self.dataOut = dataOut
451 self.dataOut = dataOut
452 self.putData()
452 self.putData()
453 return
453 return
454
454
455 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
455 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
456 '''
456 '''
457 Configure Operation
457 Configure Operation
458 '''
458 '''
459
459
460 self.path = path
460 self.path = path
461 self.blocks = kwargs.get('blocks', None)
461 self.blocks = kwargs.get('blocks', None)
462 self.counter = 0
462 self.counter = 0
463 self.oneDDict = load_json(oneDDict)
463 self.oneDDict = load_json(oneDDict)
464 self.twoDDict = load_json(twoDDict)
464 self.twoDDict = load_json(twoDDict)
465 self.ind2DList = load_json(ind2DList)
465 self.ind2DList = load_json(ind2DList)
466 meta = load_json(metadata)
466 meta = load_json(metadata)
467 self.kinst = meta.get('kinst')
467 self.kinst = meta.get('kinst')
468 self.kindat = meta.get('kindat')
468 self.kindat = meta.get('kindat')
469 self.catalog = meta.get('catalog', DEF_CATALOG)
469 self.catalog = meta.get('catalog', DEF_CATALOG)
470 self.header = meta.get('header', DEF_HEADER)
470 self.header = meta.get('header', DEF_HEADER)
471 if format == 'cedar':
471 if format == 'cedar':
472 self.ext = '.dat'
472 self.ext = '.dat'
473 self.extra_args = {}
473 self.extra_args = {}
474 elif format == 'hdf5':
474 elif format == 'hdf5':
475 self.ext = '.hdf5'
475 self.ext = '.hdf5'
476 self.extra_args = {'ind2DList': self.ind2DList}
476 self.extra_args = {'ind2DList': self.ind2DList}
477
477
478 self.keys = [k.lower() for k in self.twoDDict]
478 self.keys = [k.lower() for k in self.twoDDict]
479 if 'range' in self.keys:
479 if 'range' in self.keys:
480 self.keys.remove('range')
480 self.keys.remove('range')
481 if 'gdalt' in self.keys:
481 if 'gdalt' in self.keys:
482 self.keys.remove('gdalt')
482 self.keys.remove('gdalt')
483
483
484 def setFile(self):
484 def setFile(self):
485 '''
485 '''
486 Create new cedar file object
486 Create new cedar file object
487 '''
487 '''
488
488
489 self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
489 self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
490 date = datetime.datetime.fromtimestamp(self.dataOut.utctime)
490 date = datetime.datetime.fromtimestamp(self.dataOut.utctime)
491
491
492 filename = '{}{}{}'.format(self.mnemonic,
492 filename = '{}{}{}'.format(self.mnemonic,
493 date.strftime('%Y%m%d_%H%M%S'),
493 date.strftime('%Y%m%d_%H%M%S'),
494 self.ext)
494 self.ext)
495
495
496 self.fullname = os.path.join(self.path, filename)
496 self.fullname = os.path.join(self.path, filename)
497
497
498 if os.path.isfile(self.fullname) :
498 if os.path.isfile(self.fullname) :
499 log.warning(
499 log.warning(
500 'Destination path {} already exists. Previous file deleted.'.format(
500 'Destination path {} already exists. Previous file deleted.'.format(
501 self.fullname),
501 self.fullname),
502 'MADWriter')
502 'MADWriter')
503 os.remove(self.fullname)
503 os.remove(self.fullname)
504
504
505 try:
505 try:
506 log.success(
506 log.success(
507 'Creating file: {}'.format(self.fullname),
507 'Creating file: {}'.format(self.fullname),
508 'MADWriter')
508 'MADWriter')
509 self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
509 self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
510 except ValueError, e:
510 except ValueError, e:
511 log.error(
511 log.error(
512 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
512 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
513 'MADWriter')
513 'MADWriter')
514 return
514 return
515
515
516 return 1
516 return 1
517
517
518 def writeBlock(self):
518 def writeBlock(self):
519 '''
519 '''
520 Add data records to cedar file taking data from oneDDict and twoDDict
520 Add data records to cedar file taking data from oneDDict and twoDDict
521 attributes.
521 attributes.
522 Allowed parameters in: parcodes.tab
522 Allowed parameters in: parcodes.tab
523 '''
523 '''
524
524
525 startTime = datetime.datetime.fromtimestamp(self.dataOut.utctime)
525 startTime = datetime.datetime.fromtimestamp(self.dataOut.utctime)
526 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
526 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
527 heights = self.dataOut.heightList
527 heights = self.dataOut.heightList
528
528
529 if self.ext == '.dat':
529 if self.ext == '.dat':
530 invalid = numpy.isnan(self.dataOut.data_output)
530 for key, value in self.twoDDict.items():
531 self.dataOut.data_output[invalid] = self.missing
531 if isinstance(value, str):
532 out = {}
532 data = getattr(self.dataOut, value)
533 invalid = numpy.isnan(data)
534 data[invalid] = self.missing
535 elif isinstance(value, (tuple, list)):
536 attr, key = value
537 data = getattr(self.dataOut, attr)
538 invalid = numpy.isnan(data)
539 data[invalid] = self.missing
540
541 out = {}
533 for key, value in self.twoDDict.items():
542 for key, value in self.twoDDict.items():
534 key = key.lower()
543 key = key.lower()
535 if isinstance(value, str):
544 if isinstance(value, str):
536 if 'db' in value.lower():
545 if 'db' in value.lower():
537 tmp = getattr(self.dataOut, value.replace('_db', ''))
546 tmp = getattr(self.dataOut, value.replace('_db', ''))
538 SNRavg = numpy.average(tmp, axis=0)
547 SNRavg = numpy.average(tmp, axis=0)
539 tmp = 10*numpy.log10(SNRavg)
548 tmp = 10*numpy.log10(SNRavg)
540 else:
549 else:
541 tmp = getattr(self.dataOut, value)
550 tmp = getattr(self.dataOut, value)
542 out[key] = tmp.flatten()
551 out[key] = tmp.flatten()
543 elif isinstance(value, (tuple, list)):
552 elif isinstance(value, (tuple, list)):
544 attr, x = value
553 attr, x = value
545 data = getattr(self.dataOut, attr)
554 data = getattr(self.dataOut, attr)
546 out[key] = data[int(x)]
555 out[key] = data[int(x)]
547
556
548 a = numpy.array([out[k] for k in self.keys])
557 a = numpy.array([out[k] for k in self.keys])
549 nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
558 nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
550 index = numpy.where(nrows == False)[0]
559 index = numpy.where(nrows == False)[0]
551
560
552 rec = madrigal.cedar.MadrigalDataRecord(
561 rec = madrigal.cedar.MadrigalDataRecord(
553 self.kinst,
562 self.kinst,
554 self.kindat,
563 self.kindat,
555 startTime.year,
564 startTime.year,
556 startTime.month,
565 startTime.month,
557 startTime.day,
566 startTime.day,
558 startTime.hour,
567 startTime.hour,
559 startTime.minute,
568 startTime.minute,
560 startTime.second,
569 startTime.second,
561 startTime.microsecond/10000,
570 startTime.microsecond/10000,
562 endTime.year,
571 endTime.year,
563 endTime.month,
572 endTime.month,
564 endTime.day,
573 endTime.day,
565 endTime.hour,
574 endTime.hour,
566 endTime.minute,
575 endTime.minute,
567 endTime.second,
576 endTime.second,
568 endTime.microsecond/10000,
577 endTime.microsecond/10000,
569 self.oneDDict.keys(),
578 self.oneDDict.keys(),
570 self.twoDDict.keys(),
579 self.twoDDict.keys(),
571 len(index),
580 len(index),
572 **self.extra_args
581 **self.extra_args
573 )
582 )
574
583
575 # Setting 1d values
584 # Setting 1d values
576 for key in self.oneDDict:
585 for key in self.oneDDict:
577 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
586 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
578
587
579 # Setting 2d values
588 # Setting 2d values
580 nrec = 0
589 nrec = 0
581 for n in index:
590 for n in index:
582 for key in out:
591 for key in out:
583 rec.set2D(key, nrec, out[key][n])
592 rec.set2D(key, nrec, out[key][n])
584 nrec += 1
593 nrec += 1
585
594
586 self.fp.append(rec)
595 self.fp.append(rec)
587 if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
596 if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
588 self.fp.dump()
597 self.fp.dump()
589 if self.counter % 10 == 0 and self.counter > 0:
598 if self.counter % 100 == 0 and self.counter > 0:
590 log.log(
599 log.log(
591 'Writing {} records'.format(
600 'Writing {} records'.format(
592 self.counter),
601 self.counter),
593 'MADWriter')
602 'MADWriter')
594
603
595 def setHeader(self):
604 def setHeader(self):
596 '''
605 '''
597 Create an add catalog and header to cedar file
606 Create an add catalog and header to cedar file
598 '''
607 '''
599
608
600 log.success('Closing file {}'.format(self.fullname), 'MADWriter')
609 log.success('Closing file {}'.format(self.fullname), 'MADWriter')
601
610
602 if self.ext == '.dat':
611 if self.ext == '.dat':
603 self.fp.write()
612 self.fp.write()
604 else:
613 else:
605 self.fp.dump()
614 self.fp.dump()
606 self.fp.close()
615 self.fp.close()
607
616
608 header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
617 header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
609 header.createCatalog(**self.catalog)
618 header.createCatalog(**self.catalog)
610 header.createHeader(**self.header)
619 header.createHeader(**self.header)
611 header.write()
620 header.write()
612
621
613 def putData(self):
622 def putData(self):
614
623
615 if self.dataOut.flagNoData:
624 if self.dataOut.flagNoData:
616 return 0
625 return 0
617
626
618 if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
627 if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
619 if self.counter > 0:
628 if self.counter > 0:
620 self.setHeader()
629 self.setHeader()
621 self.counter = 0
630 self.counter = 0
622
631
623 if self.counter == 0:
632 if self.counter == 0:
624 self.setFile()
633 self.setFile()
625
634
626 self.writeBlock()
635 self.writeBlock()
627 self.counter += 1
636 self.counter += 1
628
637
629 def close(self):
638 def close(self):
630
639
631 if self.counter > 0:
640 if self.counter > 0:
632 self.setHeader()
641 self.setHeader()
General Comments 0
You need to be logged in to leave comments. Login now