##// END OF EJS Templates
schain-jc
RSS
Madrigal 3 reading/writing task #1154
jespinoza -
r1070:d6c1e5e8fd34
parent child
Show More
Side by Side Unified
Context file:
r1070:d6c1e5e8fd34 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,580 +1,632
1 '''
1 '''
2 Created on Aug 1, 2017
2 Created on Aug 1, 2017
3
3
4 @author: Juan C. Espinoza
4 @author: Juan C. Espinoza
5 '''
5 '''
6
6
7 import os
7 import os
8 import sys
8 import sys
9 import time
9 import time
10 import json
10 import json
11 import glob
11 import glob
12 import datetime
12 import datetime
13
13
14 import numpy
14 import numpy
15 import h5py
15 import h5py
16
16
17 try:
17 from schainpy.model.io.jroIO_base import JRODataReader
18 import madrigal
19 import madrigal.cedar
20 except:
21 print 'You should install "madrigal library" module if you want to read/write Madrigal data'
22
23 from schainpy.model.io.jroIO_base import JRODataReader
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
18 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 from schainpy.model.data.jrodata import Parameters
19 from schainpy.model.data.jrodata import Parameters
26 from schainpy.utils import log
20 from schainpy.utils import log
27
21
22 try:
23 import madrigal.cedar
24 except:
25 log.warning(
26 'You should install "madrigal library" module if you want to read/write Madrigal data'
27 )
28
28
29 DEF_CATALOG = {
29 DEF_CATALOG = {
30 'principleInvestigator': 'Marco Milla',
30 'principleInvestigator': 'Marco Milla',
31 'expPurpose': None,
31 'expPurpose': None,
32 'expMode': None,
33 'cycleTime': None,
32 'cycleTime': None,
34 'correlativeExp': None,
33 'correlativeExp': None,
35 'sciRemarks': None,
34 'sciRemarks': None,
36 'instRemarks': None
35 'instRemarks': None
37 }
36 }
38 DEF_HEADER = {
37 DEF_HEADER = {
39 'kindatDesc': None,
38 'kindatDesc': None,
40 'analyst': 'Jicamarca User',
39 'analyst': 'Jicamarca User',
41 'comments': None,
40 'comments': None,
42 'history': None
41 'history': None
43 }
42 }
44 MNEMONICS = {
43 MNEMONICS = {
45 10: 'jro',
44 10: 'jro',
46 11: 'jbr',
45 11: 'jbr',
47 840: 'jul',
46 840: 'jul',
48 13: 'jas',
47 13: 'jas',
49 1000: 'pbr',
48 1000: 'pbr',
50 1001: 'hbr',
49 1001: 'hbr',
51 1002: 'obr',
50 1002: 'obr',
52 }
51 }
53
52
54 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
53 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
55
54
56 def load_json(obj):
55 def load_json(obj):
57 '''
56 '''
58 Parse json as string instead of unicode
57 Parse json as string instead of unicode
59 '''
58 '''
60
59
61 if isinstance(obj, str):
60 if isinstance(obj, str):
62 iterable = json.loads(obj)
61 iterable = json.loads(obj)
62 else:
63 iterable = obj
63
64
64 if isinstance(iterable, dict):
65 if isinstance(iterable, dict):
65 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
66 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
66 for k, v in iterable.items()}
67 for k, v in iterable.items()}
67 elif isinstance(iterable, (list, tuple)):
68 elif isinstance(iterable, (list, tuple)):
68 return [str(v) if isinstance(v, unicode) else v for v in iterable]
69 return [str(v) if isinstance(v, unicode) else v for v in iterable]
69
70
70 return iterable
71 return iterable
71
72
72
73
73 class MADReader(JRODataReader, ProcessingUnit):
74 class MADReader(JRODataReader, ProcessingUnit):
74
75
75 def __init__(self, **kwargs):
76 def __init__(self, **kwargs):
76
77
77 ProcessingUnit.__init__(self, **kwargs)
78 ProcessingUnit.__init__(self, **kwargs)
78
79
79 self.dataOut = Parameters()
80 self.dataOut = Parameters()
80 self.counter_records = 0
81 self.counter_records = 0
81 self.nrecords = None
82 self.nrecords = None
82 self.flagNoMoreFiles = 0
83 self.flagNoMoreFiles = 0
83 self.isConfig = False
84 self.isConfig = False
84 self.filename = None
85 self.filename = None
85 self.intervals = set()
86 self.intervals = set()
86
87
87 def setup(self,
88 def setup(self,
88 path=None,
89 path=None,
89 startDate=None,
90 startDate=None,
90 endDate=None,
91 endDate=None,
91 format=None,
92 format=None,
92 startTime=datetime.time(0, 0, 0),
93 startTime=datetime.time(0, 0, 0),
93 endTime=datetime.time(23, 59, 59),
94 endTime=datetime.time(23, 59, 59),
94 **kwargs):
95 **kwargs):
95
96
96 self.started = True
97 self.path = path
97 self.path = path
98 self.startDate = startDate
98 self.startDate = startDate
99 self.endDate = endDate
99 self.endDate = endDate
100 self.startTime = startTime
100 self.startTime = startTime
101 self.endTime = endTime
101 self.endTime = endTime
102 self.datatime = datetime.datetime(1900,1,1)
102 self.datatime = datetime.datetime(1900,1,1)
103 self.oneDDict = load_json(kwargs.get('oneDDict',
103 self.oneDDict = load_json(kwargs.get('oneDDict',
104 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
104 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
105 self.twoDDict = load_json(kwargs.get('twoDDict',
105 self.twoDDict = load_json(kwargs.get('twoDDict',
106 "{\"GDALT\": \"heightList\"}"))
106 "{\"GDALT\": \"heightList\"}"))
107 self.ind2DList = load_json(kwargs.get('ind2DList',
107 self.ind2DList = load_json(kwargs.get('ind2DList',
108 "[\"GDALT\"]"))
108 "[\"GDALT\"]"))
109 if self.path is None:
109 if self.path is None:
110 raise ValueError, 'The path is not valid'
110 raise ValueError, 'The path is not valid'
111
111
112 if format is None:
112 if format is None:
113 raise ValueError, 'The format is not valid choose simple or hdf5'
113 raise ValueError, 'The format is not valid choose simple or hdf5'
114 elif format.lower() in ('simple', 'txt'):
114 elif format.lower() in ('simple', 'txt'):
115 self.ext = '.txt'
115 self.ext = '.txt'
116 elif format.lower() in ('cedar',):
116 elif format.lower() in ('cedar',):
117 self.ext = '.001'
117 self.ext = '.001'
118 else:
118 else:
119 self.ext = '.hdf5'
119 self.ext = '.hdf5'
120
120
121 self.search_files(self.path)
121 self.search_files(self.path)
122 self.fileId = 0
122 self.fileId = 0
123
123
124 if not self.fileList:
124 if not self.fileList:
125 raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
125 raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
126
126
127 self.setNextFile()
127 self.setNextFile()
128
128
129 def search_files(self, path):
129 def search_files(self, path):
130 '''
130 '''
131 Searching for madrigal files in path
131 Searching for madrigal files in path
132 Creating a list of files to procces included in [startDate,endDate]
132 Creating a list of files to procces included in [startDate,endDate]
133
133
134 Input:
134 Input:
135 path - Path to find files
135 path - Path to find files
136 '''
136 '''
137
137
138 print 'Searching files {} in {} '.format(self.ext, path)
138 log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
139 foldercounter = 0
139 foldercounter = 0
140 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
140 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
141 fileList0.sort()
141 fileList0.sort()
142
142
143 self.fileList = []
143 self.fileList = []
144 self.dateFileList = []
144 self.dateFileList = []
145
145
146 startDate = self.startDate - datetime.timedelta(1)
146 startDate = self.startDate - datetime.timedelta(1)
147 endDate = self.endDate + datetime.timedelta(1)
147 endDate = self.endDate + datetime.timedelta(1)
148
148
149 for thisFile in fileList0:
149 for thisFile in fileList0:
150 year = thisFile[3:7]
150 year = thisFile[3:7]
151 if not year.isdigit():
151 if not year.isdigit():
152 continue
152 continue
153
153
154 month = thisFile[7:9]
154 month = thisFile[7:9]
155 if not month.isdigit():
155 if not month.isdigit():
156 continue
156 continue
157
157
158 day = thisFile[9:11]
158 day = thisFile[9:11]
159 if not day.isdigit():
159 if not day.isdigit():
160 continue
160 continue
161
161
162 year, month, day = int(year), int(month), int(day)
162 year, month, day = int(year), int(month), int(day)
163 dateFile = datetime.date(year, month, day)
163 dateFile = datetime.date(year, month, day)
164
164
165 if (startDate > dateFile) or (endDate < dateFile):
165 if (startDate > dateFile) or (endDate < dateFile):
166 continue
166 continue
167
167
168 self.fileList.append(thisFile)
168 self.fileList.append(thisFile)
169 self.dateFileList.append(dateFile)
169 self.dateFileList.append(dateFile)
170
170
171 return
171 return
172
172
173 def parseHeader(self):
173 def parseHeader(self):
174 '''
174 '''
175 '''
175 '''
176
176
177 self.output = {}
177 self.output = {}
178 self.version = '2'
178 self.version = '2'
179 s_parameters = None
179 s_parameters = None
180 if self.ext == '.txt':
180 if self.ext == '.txt':
181 self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
181 self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
182 elif self.ext == '.hdf5':
182 elif self.ext == '.hdf5':
183 metadata = self.fp['Metadata']
183 metadata = self.fp['Metadata']
184 data = self.fp['Data']['Array Layout']
184 data = self.fp['Data']['Array Layout']
185 if 'Independent Spatial Parameters' in metadata:
185 if 'Independent Spatial Parameters' in metadata:
186 s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
186 s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
187 self.version = '3'
187 self.version = '3'
188 one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
188 one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
189 one_d = [1 for s in one]
189 one_d = [1 for s in one]
190 two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
190 two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
191 two_d = [2 for s in two]
191 two_d = [2 for s in two]
192 self.parameters = one + two
192 self.parameters = one + two
193 self.parameters_d = one_d + two_d
193 self.parameters_d = one_d + two_d
194
194
195 log.success('Parameters found: {}'.format(','.join(self.parameters)),
195 log.success('Parameters found: {}'.format(','.join(self.parameters)),
196 'MADReader')
196 'MADReader')
197 if s_parameters:
197 if s_parameters:
198 log.success('Spatial parameters: {}'.format(','.join(s_parameters)),
198 log.success('Spatial parameters: {}'.format(','.join(s_parameters)),
199 'MADReader')
199 'MADReader')
200
200
201 for param in self.oneDDict.keys():
201 for param in self.oneDDict.keys():
202 if param.lower() not in self.parameters:
202 if param.lower() not in self.parameters:
203 print('\x1b[33m[Warning]\x1b[0m Parameter \x1b[1;32m{}\x1b[0m not found will be ignored'.format(
203 log.warning(
204 param
204 'Parameter {} not found will be ignored'.format(
205 ))
205 param),
206 'MADReader')
206 self.oneDDict.pop(param, None)
207 self.oneDDict.pop(param, None)
207
208
208 for param, value in self.twoDDict.items():
209 for param, value in self.twoDDict.items():
209 if param.lower() not in self.parameters:
210 if param.lower() not in self.parameters:
210 print('\x1b[33m[Warning]\x1b[0m Parameter \x1b[1;32m{}\x1b[0m not found will be ignored'.format(
211 log.warning(
211 param
212 'Parameter {} not found, it will be ignored'.format(
212 ))
213 param),
214 'MADReader')
213 self.twoDDict.pop(param, None)
215 self.twoDDict.pop(param, None)
214 continue
216 continue
215 if isinstance(value, list):
217 if isinstance(value, list):
216 if value[0] not in self.output:
218 if value[0] not in self.output:
217 self.output[value[0]] = []
219 self.output[value[0]] = []
218 self.output[value[0]].append(None)
220 self.output[value[0]].append(None)
219
221
220 def parseData(self):
222 def parseData(self):
221 '''
223 '''
222 '''
224 '''
223
225
224 if self.ext == '.txt':
226 if self.ext == '.txt':
225 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
227 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
226 self.nrecords = self.data.shape[0]
228 self.nrecords = self.data.shape[0]
227 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
229 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
228 elif self.ext == '.hdf5':
230 elif self.ext == '.hdf5':
229 self.data = self.fp['Data']['Array Layout']
231 self.data = self.fp['Data']['Array Layout']
230 self.nrecords = len(self.data['timestamps'].value)
232 self.nrecords = len(self.data['timestamps'].value)
231 self.ranges = self.data['range'].value
233 self.ranges = self.data['range'].value
232
234
233 def setNextFile(self):
235 def setNextFile(self):
234 '''
236 '''
235 '''
237 '''
236
238
237 file_id = self.fileId
239 file_id = self.fileId
238
240
239 if file_id == len(self.fileList):
241 if file_id == len(self.fileList):
240 print '\nNo more files in the folder'
242 log.success('No more files', 'MADReader')
241 print 'Total number of file(s) read : {}'.format(self.fileId)
242 self.flagNoMoreFiles = 1
243 self.flagNoMoreFiles = 1
243 return 0
244 return 0
244
245
245 print('\x1b[32m[Info]\x1b[0m Opening: {}'.format(
246 log.success(
246 self.fileList[file_id]
247 'Opening: {}'.format(self.fileList[file_id]),
247 ))
248 'MADReader'
249 )
250
248 filename = os.path.join(self.path, self.fileList[file_id])
251 filename = os.path.join(self.path, self.fileList[file_id])
249
252
250 if self.filename is not None:
253 if self.filename is not None:
251 self.fp.close()
254 self.fp.close()
252
255
253 self.filename = filename
256 self.filename = filename
254 self.filedate = self.dateFileList[file_id]
257 self.filedate = self.dateFileList[file_id]
255
258
256 if self.ext=='.hdf5':
259 if self.ext=='.hdf5':
257 self.fp = h5py.File(self.filename, 'r')
260 self.fp = h5py.File(self.filename, 'r')
258 else:
261 else:
259 self.fp = open(self.filename, 'rb')
262 self.fp = open(self.filename, 'rb')
260
263
261 self.parseHeader()
264 self.parseHeader()
262 self.parseData()
265 self.parseData()
263 self.sizeOfFile = os.path.getsize(self.filename)
266 self.sizeOfFile = os.path.getsize(self.filename)
264 self.counter_records = 0
267 self.counter_records = 0
265 self.flagIsNewFile = 0
268 self.flagIsNewFile = 0
266 self.fileId += 1
269 self.fileId += 1
267
270
268 return 1
271 return 1
269
272
270 def readNextBlock(self):
273 def readNextBlock(self):
271
274
272 while True:
275 while True:
273
276 self.flagDiscontinuousBlock = 0
274 if self.flagIsNewFile:
277 if self.flagIsNewFile:
275 if not self.setNextFile():
278 if not self.setNextFile():
276 return 0
279 return 0
277
280
278 self.readBlock()
281 self.readBlock()
279
282
280 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
283 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
281 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
284 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
282 print "\x1b[32m[Reading]\x1b[0m Record No. %d/%d -> %s \x1b[33m[Skipping]\x1b[0m" %(
285 log.warning(
283 self.counter_records,
286 'Reading Record No. {}/{} -> {} [Skipping]'.format(
284 self.nrecords,
287 self.counter_records,
285 self.datatime.ctime())
288 self.nrecords,
289 self.datatime.ctime()),
290 'MADReader')
286 continue
291 continue
287 break
292 break
288
293
289 print "\x1b[32m[Reading]\x1b[0m Record No. %d/%d -> %s" %(
294 log.log(
290 self.counter_records,
295 'Reading Record No. {}/{} -> {}'.format(
291 self.nrecords,
296 self.counter_records,
292 self.datatime.ctime())
297 self.nrecords,
298 self.datatime.ctime()),
299 'MADReader')
293
300
294 return 1
301 return 1
295
302
296 def readBlock(self):
303 def readBlock(self):
297 '''
304 '''
298 '''
305 '''
299 dum = []
306 dum = []
300 if self.ext == '.txt':
307 if self.ext == '.txt':
301 dt = self.data[self.counter_records][:6].astype(int)
308 dt = self.data[self.counter_records][:6].astype(int)
302 self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
309 self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
303 while True:
310 while True:
304 dt = self.data[self.counter_records][:6].astype(int)
311 dt = self.data[self.counter_records][:6].astype(int)
305 datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
312 datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
306 if datatime == self.datatime:
313 if datatime == self.datatime:
307 dum.append(self.data[self.counter_records])
314 dum.append(self.data[self.counter_records])
308 self.counter_records += 1
315 self.counter_records += 1
309 if self.counter_records == self.nrecords:
316 if self.counter_records == self.nrecords:
310 self.flagIsNewFile = True
317 self.flagIsNewFile = True
311 break
318 break
312 continue
319 continue
313 self.intervals.add((datatime-self.datatime).seconds)
320 self.intervals.add((datatime-self.datatime).seconds)
321 if datatime.date() > self.datatime.date():
322 self.flagDiscontinuousBlock = 1
314 break
323 break
315 elif self.ext == '.hdf5':
324 elif self.ext == '.hdf5':
316 datatime = datetime.datetime.utcfromtimestamp(
325 datatime = datetime.datetime.utcfromtimestamp(
317 self.data['timestamps'][self.counter_records])
326 self.data['timestamps'][self.counter_records])
318 nHeights = len(self.ranges)
327 nHeights = len(self.ranges)
319 for n, param in enumerate(self.parameters):
328 for n, param in enumerate(self.parameters):
320 if self.parameters_d[n] == 1:
329 if self.parameters_d[n] == 1:
321 dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
330 dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
322 else:
331 else:
323 if self.version == '2':
332 if self.version == '2':
324 dum.append(self.data['2D Parameters'][param][self.counter_records])
333 dum.append(self.data['2D Parameters'][param][self.counter_records])
325 else:
334 else:
326 tmp = self.data['2D Parameters'][param].value.T
335 tmp = self.data['2D Parameters'][param].value.T
327 dum.append(tmp[self.counter_records])
336 dum.append(tmp[self.counter_records])
328 self.intervals.add((datatime-self.datatime).seconds)
337 self.intervals.add((datatime-self.datatime).seconds)
338 if datatime.date()>self.datatime.date():
339 self.flagDiscontinuousBlock = 1
329 self.datatime = datatime
340 self.datatime = datatime
330 self.counter_records += 1
341 self.counter_records += 1
331 if self.counter_records == self.nrecords:
342 if self.counter_records == self.nrecords:
332 self.flagIsNewFile = True
343 self.flagIsNewFile = True
333
344
334 self.buffer = numpy.array(dum)
345 self.buffer = numpy.array(dum)
335 return
346 return
336
347
337 def set_output(self):
348 def set_output(self):
338 '''
349 '''
339 Storing data from buffer to dataOut object
350 Storing data from buffer to dataOut object
340 '''
351 '''
341
352
342 parameters = [None for __ in self.parameters]
353 parameters = [None for __ in self.parameters]
343
354
344 for param, attr in self.oneDDict.items():
355 for param, attr in self.oneDDict.items():
345 x = self.parameters.index(param.lower())
356 x = self.parameters.index(param.lower())
346 setattr(self.dataOut, attr, self.buffer[0][x])
357 setattr(self.dataOut, attr, self.buffer[0][x])
347
358
348 for param, value in self.twoDDict.items():
359 for param, value in self.twoDDict.items():
349 x = self.parameters.index(param.lower())
360 x = self.parameters.index(param.lower())
350 if self.ext == '.txt':
361 if self.ext == '.txt':
351 y = self.parameters.index(self.ind2DList[0].lower())
362 y = self.parameters.index(self.ind2DList[0].lower())
352 ranges = self.buffer[:,y]
363 ranges = self.buffer[:,y]
353 if self.ranges.size == ranges.size:
364 if self.ranges.size == ranges.size:
354 continue
365 continue
355 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
366 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
356 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
367 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
357 dummy[index] = self.buffer[:,x]
368 dummy[index] = self.buffer[:,x]
358 else:
369 else:
359
370 dummy = self.buffer[x]
360 dummy = self.buffer[x]
371
361
362 if isinstance(value, str):
372 if isinstance(value, str):
363 if value not in self.ind2DList:
373 if value not in self.ind2DList:
364 setattr(self.dataOut, value, dummy.reshape(1,-1))
374 setattr(self.dataOut, value, dummy.reshape(1,-1))
365 elif isinstance(value, list):
375 elif isinstance(value, list):
366 self.output[value[0]][value[1]] = dummy
376 self.output[value[0]][value[1]] = dummy
367 parameters[value[1]] = param
377 parameters[value[1]] = param
368
378
369 for key, value in self.output.items():
379 for key, value in self.output.items():
370 setattr(self.dataOut, key, numpy.array(value))
380 setattr(self.dataOut, key, numpy.array(value))
371
381
372 self.dataOut.parameters = [s for s in parameters if s]
382 self.dataOut.parameters = [s for s in parameters if s]
373 self.dataOut.heightList = self.ranges
383 self.dataOut.heightList = self.ranges
374 self.dataOut.utctime = (self.datatime - UT1970).total_seconds()
384 self.dataOut.utctime = (self.datatime - UT1970).total_seconds()
375 self.dataOut.utctimeInit = self.dataOut.utctime
385 self.dataOut.utctimeInit = self.dataOut.utctime
376 self.dataOut.paramInterval = min(self.intervals)
386 self.dataOut.paramInterval = min(self.intervals)
377 self.dataOut.useLocalTime = False
387 self.dataOut.useLocalTime = False
378 self.dataOut.flagNoData = False
388 self.dataOut.flagNoData = False
379 self.dataOut.started = self.started
389 self.dataOut.nrecords = self.nrecords
390 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
380
391
381 def getData(self):
392 def getData(self):
382 '''
393 '''
383 Storing data from databuffer to dataOut object
394 Storing data from databuffer to dataOut object
384 '''
395 '''
385 if self.flagNoMoreFiles:
396 if self.flagNoMoreFiles:
386 self.dataOut.flagNoData = True
397 self.dataOut.flagNoData = True
387 print 'No file left to process'
398 log.error('No file left to process', 'MADReader')
388 return 0
399 return 0
389
400
390 if not self.readNextBlock():
401 if not self.readNextBlock():
391 self.dataOut.flagNoData = True
402 self.dataOut.flagNoData = True
392 return 0
403 return 0
393
404
394 self.set_output()
405 self.set_output()
395
406
396 return 1
407 return 1
397
408
398
409
399 class MAD2Writer(Operation):
410 class MADWriter(Operation):
400
411
401 missing = -32767
412 missing = -32767
402 ext = '.dat'
403
413
404 def __init__(self, **kwargs):
414 def __init__(self, **kwargs):
405
415
406 Operation.__init__(self, **kwargs)
416 Operation.__init__(self, **kwargs)
407 self.dataOut = Parameters()
417 self.dataOut = Parameters()
408 self.path = None
418 self.path = None
409 self.dataOut = None
419 self.fp = None
410
420
411 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}', metadata='{}', **kwargs):
421 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
422 metadata='{}', format='cedar', **kwargs):
412 '''
423 '''
413 Inputs:
424 Inputs:
414 path - path where files will be created
425 path - path where files will be created
415 oneDDict - json of one-dimensional parameters in record where keys
426 oneDDict - json of one-dimensional parameters in record where keys
416 are Madrigal codes (integers or mnemonics) and values the corresponding
427 are Madrigal codes (integers or mnemonics) and values the corresponding
417 dataOut attribute e.g: {
428 dataOut attribute e.g: {
418 'gdlatr': 'lat',
429 'gdlatr': 'lat',
419 'gdlonr': 'lon',
430 'gdlonr': 'lon',
420 'gdlat2':'lat',
431 'gdlat2':'lat',
421 'glon2':'lon'}
432 'glon2':'lon'}
422 ind2DList - list of independent spatial two-dimensional parameters e.g:
433 ind2DList - list of independent spatial two-dimensional parameters e.g:
423 ['heighList']
434 ['heighList']
424 twoDDict - json of two-dimensional parameters in record where keys
435 twoDDict - json of two-dimensional parameters in record where keys
425 are Madrigal codes (integers or mnemonics) and values the corresponding
436 are Madrigal codes (integers or mnemonics) and values the corresponding
426 dataOut attribute if multidimensional array specify as tupple
437 dataOut attribute if multidimensional array specify as tupple
427 ('attr', pos) e.g: {
438 ('attr', pos) e.g: {
428 'gdalt': 'heightList',
439 'gdalt': 'heightList',
429 'vn1p2': ('data_output', 0),
440 'vn1p2': ('data_output', 0),
430 'vn2p2': ('data_output', 1),
441 'vn2p2': ('data_output', 1),
431 'vn3': ('data_output', 2),
442 'vn3': ('data_output', 2),
432 'snl': ('data_SNR', 'db')
443 'snl': ('data_SNR', 'db')
433 }
444 }
434 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
445 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
435 '''
446 '''
436 if not self.isConfig:
447 if not self.isConfig:
437 self.setup(dataOut, path, oneDDict, ind2DList, twoDDict, metadata, **kwargs)
448 self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
438 self.isConfig = True
449 self.isConfig = True
439
450
451 self.dataOut = dataOut
440 self.putData()
452 self.putData()
441 return
453 return
442
454
443 def setup(self, dataOut, path, oneDDict, ind2DList, twoDDict, metadata, **kwargs):
455 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
444 '''
456 '''
445 Configure Operation
457 Configure Operation
446 '''
458 '''
447
459
448 self.dataOut = dataOut
449 self.nmodes = self.dataOut.nmodes
450 self.path = path
460 self.path = path
451 self.blocks = kwargs.get('blocks', None)
461 self.blocks = kwargs.get('blocks', None)
452 self.counter = 0
462 self.counter = 0
453 self.oneDDict = load_json(oneDDict)
463 self.oneDDict = load_json(oneDDict)
454 self.twoDDict = load_json(twoDDict)
464 self.twoDDict = load_json(twoDDict)
455 self.ind2DList = load_json(ind2DList)
465 self.ind2DList = load_json(ind2DList)
456 meta = load_json(metadata)
466 meta = load_json(metadata)
457 self.kinst = meta.get('kinst')
467 self.kinst = meta.get('kinst')
458 self.kindat = meta.get('kindat')
468 self.kindat = meta.get('kindat')
459 self.catalog = meta.get('catalog', DEF_CATALOG)
469 self.catalog = meta.get('catalog', DEF_CATALOG)
460 self.header = meta.get('header', DEF_HEADER)
470 self.header = meta.get('header', DEF_HEADER)
461
471 if format == 'cedar':
462 return
472 self.ext = '.dat'
473 self.extra_args = {}
474 elif format == 'hdf5':
475 self.ext = '.hdf5'
476 self.extra_args = {'ind2DList': self.ind2DList}
477
478 self.keys = [k.lower() for k in self.twoDDict]
479 if 'range' in self.keys:
480 self.keys.remove('range')
481 if 'gdalt' in self.keys:
482 self.keys.remove('gdalt')
463
483
464 def setFile(self):
484 def setFile(self):
465 '''
485 '''
466 Create new cedar file object
486 Create new cedar file object
467 '''
487 '''
468
488
469 self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
489 self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
470 date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
490 date = datetime.datetime.fromtimestamp(self.dataOut.utctime)
471
491
472 filename = '%s%s_%s%s' % (self.mnemonic,
492 filename = '{}{}{}'.format(self.mnemonic,
473 date.strftime('%Y%m%d_%H%M%S'),
493 date.strftime('%Y%m%d_%H%M%S'),
474 self.dataOut.mode,
494 self.ext)
475 self.ext)
476
495
477 self.fullname = os.path.join(self.path, filename)
496 self.fullname = os.path.join(self.path, filename)
478
497
479 if os.path.isfile(self.fullname) :
498 if os.path.isfile(self.fullname) :
480 print "Destination path '%s' already exists. Previous file deleted. " %self.fullname
499 log.warning(
500 'Destination path {} already exists. Previous file deleted.'.format(
501 self.fullname),
502 'MADWriter')
481 os.remove(self.fullname)
503 os.remove(self.fullname)
482
504
483 try:
505 try:
484 print '[Writing] creating file : %s' % (self.fullname)
506 log.success(
485 self.cedarObj = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
507 'Creating file: {}'.format(self.fullname),
508 'MADWriter')
509 self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
486 except ValueError, e:
510 except ValueError, e:
487 print '[Error]: Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile" '
511 log.error(
512 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
513 'MADWriter')
488 return
514 return
489
515
490 return 1
516 return 1
491
517
492 def writeBlock(self):
518 def writeBlock(self):
493 '''
519 '''
494 Add data records to cedar file taking data from oneDDict and twoDDict
520 Add data records to cedar file taking data from oneDDict and twoDDict
495 attributes.
521 attributes.
496 Allowed parameters in: parcodes.tab
522 Allowed parameters in: parcodes.tab
497 '''
523 '''
498
524
499 startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
525 startTime = datetime.datetime.fromtimestamp(self.dataOut.utctime)
500 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
526 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
501 nrows = len(getattr(self.dataOut, self.ind2DList))
527 heights = self.dataOut.heightList
502
528
529 if self.ext == '.dat':
530 invalid = numpy.isnan(self.dataOut.data_output)
531 self.dataOut.data_output[invalid] = self.missing
532 out = {}
533 for key, value in self.twoDDict.items():
534 key = key.lower()
535 if isinstance(value, str):
536 if 'db' in value.lower():
537 tmp = getattr(self.dataOut, value.replace('_db', ''))
538 SNRavg = numpy.average(tmp, axis=0)
539 tmp = 10*numpy.log10(SNRavg)
540 else:
541 tmp = getattr(self.dataOut, value)
542 out[key] = tmp.flatten()
543 elif isinstance(value, (tuple, list)):
544 attr, x = value
545 data = getattr(self.dataOut, attr)
546 out[key] = data[int(x)]
547
548 a = numpy.array([out[k] for k in self.keys])
549 nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
550 index = numpy.where(nrows == False)[0]
551
503 rec = madrigal.cedar.MadrigalDataRecord(
552 rec = madrigal.cedar.MadrigalDataRecord(
504 self.kinst,
553 self.kinst,
505 self.kindat,
554 self.kindat,
506 startTime.year,
555 startTime.year,
507 startTime.month,
556 startTime.month,
508 startTime.day,
557 startTime.day,
509 startTime.hour,
558 startTime.hour,
510 startTime.minute,
559 startTime.minute,
511 startTime.second,
560 startTime.second,
512 startTime.microsecond/10000,
561 startTime.microsecond/10000,
513 endTime.year,
562 endTime.year,
514 endTime.month,
563 endTime.month,
515 endTime.day,
564 endTime.day,
516 endTime.hour,
565 endTime.hour,
517 endTime.minute,
566 endTime.minute,
518 endTime.second,
567 endTime.second,
519 endTime.microsecond/10000,
568 endTime.microsecond/10000,
520 self.oneDDict.keys(),
569 self.oneDDict.keys(),
521 self.twoDDict.keys(),
570 self.twoDDict.keys(),
522 nrows
571 len(index),
523 )
572 **self.extra_args
524
573 )
574
525 # Setting 1d values
575 # Setting 1d values
526 for key in self.oneDDict:
576 for key in self.oneDDict:
527 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
577 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
528
578
529 # Setting 2d values
579 # Setting 2d values
530 invalid = numpy.isnan(self.dataOut.data_output)
580 nrec = 0
531 self.dataOut.data_output[invalid] = self.missing
581 for n in index:
532 out = {}
533 for key, value in self.twoDDict.items():
534 if isinstance(value, str):
535 out[key] = getattr(self.dataOut, value)
536 elif isinstance(value, tuple):
537 attr, x = value
538 if isinstance(x, (int, float)):
539 out[key] = getattr(self.dataOut, attr)[int(x)]
540 elif x.lower()=='db':
541 tmp = getattr(self.dataOut, attr)
542 SNRavg = numpy.average(tmp, axis=0)
543 out[key] = 10*numpy.log10(SNRavg)
544
545 for n in range(nrows):
546 for key in out:
582 for key in out:
547 rec.set2D(key, n, out[key][n])
583 rec.set2D(key, nrec, out[key][n])
548
584 nrec += 1
549 self.cedarObj.append(rec)
585
550 self.cedarObj.dump()
586 self.fp.append(rec)
551 print '[Writing] Record No. {} (mode {}).'.format(
587 if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
552 self.counter,
588 self.fp.dump()
553 self.dataOut.mode
589 if self.counter % 10 == 0 and self.counter > 0:
554 )
590 log.log(
591 'Writing {} records'.format(
592 self.counter),
593 'MADWriter')
555
594
556 def setHeader(self):
595 def setHeader(self):
557 '''
596 '''
558 Create an add catalog and header to cedar file
597 Create an add catalog and header to cedar file
559 '''
598 '''
560
599
600 log.success('Closing file {}'.format(self.fullname), 'MADWriter')
601
602 if self.ext == '.dat':
603 self.fp.write()
604 else:
605 self.fp.dump()
606 self.fp.close()
607
561 header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
608 header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
562 header.createCatalog(**self.catalog)
609 header.createCatalog(**self.catalog)
563 header.createHeader(**self.header)
610 header.createHeader(**self.header)
564 header.write()
611 header.write()
565
612
566 def putData(self):
613 def putData(self):
567
614
568 if self.dataOut.flagNoData:
615 if self.dataOut.flagNoData:
569 return 0
616 return 0
570
617
618 if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
619 if self.counter > 0:
620 self.setHeader()
621 self.counter = 0
622
571 if self.counter == 0:
623 if self.counter == 0:
572 self.setFile()
624 self.setFile()
573
625
574 if self.counter <= self.dataOut.nrecords:
626 self.writeBlock()
575 self.writeBlock()
627 self.counter += 1
576 self.counter += 1
628
629 def close(self):
577
630
578 if self.counter == self.dataOut.nrecords or self.counter == self.blocks:
631 if self.counter > 0:
579 self.setHeader()
632 self.setHeader()
580 self.counter = 0
General Comments 0
You need to be logged in to leave comments. Login now