##// END OF EJS Templates
schain-jc
RSS
Fix utc time in MAdrigal modules
Juan C. Espinoza -
r1086:828386118e15
parent child
Show More
Side by Side Unified
Context file:
r1086:828386118e15 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,641 +1,642
1 '''
1 '''
2 Created on Aug 1, 2017
2 Created on Aug 1, 2017
3
3
4 @author: Juan C. Espinoza
4 @author: Juan C. Espinoza
5 '''
5 '''
6
6
7 import os
7 import os
8 import sys
8 import sys
9 import time
9 import time
10 import json
10 import json
11 import glob
11 import glob
12 import datetime
12 import datetime
13
13
14 import numpy
14 import numpy
15 import h5py
15 import h5py
16
16
17 from schainpy.model.io.jroIO_base import JRODataReader
17 from schainpy.model.io.jroIO_base import JRODataReader
18 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
18 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
19 from schainpy.model.data.jrodata import Parameters
19 from schainpy.model.data.jrodata import Parameters
20 from schainpy.utils import log
20 from schainpy.utils import log
21
21
22 try:
22 try:
23 import madrigal.cedar
23 import madrigal.cedar
24 except:
24 except:
25 log.warning(
25 log.warning(
26 'You should install "madrigal library" module if you want to read/write Madrigal data'
26 'You should install "madrigal library" module if you want to read/write Madrigal data'
27 )
27 )
28
28
29 DEF_CATALOG = {
29 DEF_CATALOG = {
30 'principleInvestigator': 'Marco Milla',
30 'principleInvestigator': 'Marco Milla',
31 'expPurpose': None,
31 'expPurpose': None,
32 'cycleTime': None,
32 'cycleTime': None,
33 'correlativeExp': None,
33 'correlativeExp': None,
34 'sciRemarks': None,
34 'sciRemarks': None,
35 'instRemarks': None
35 'instRemarks': None
36 }
36 }
37 DEF_HEADER = {
37 DEF_HEADER = {
38 'kindatDesc': None,
38 'kindatDesc': None,
39 'analyst': 'Jicamarca User',
39 'analyst': 'Jicamarca User',
40 'comments': None,
40 'comments': None,
41 'history': None
41 'history': None
42 }
42 }
43 MNEMONICS = {
43 MNEMONICS = {
44 10: 'jro',
44 10: 'jro',
45 11: 'jbr',
45 11: 'jbr',
46 840: 'jul',
46 840: 'jul',
47 13: 'jas',
47 13: 'jas',
48 1000: 'pbr',
48 1000: 'pbr',
49 1001: 'hbr',
49 1001: 'hbr',
50 1002: 'obr',
50 1002: 'obr',
51 }
51 }
52
52
53 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
53 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
54
54
55 def load_json(obj):
55 def load_json(obj):
56 '''
56 '''
57 Parse json as string instead of unicode
57 Parse json as string instead of unicode
58 '''
58 '''
59
59
60 if isinstance(obj, str):
60 if isinstance(obj, str):
61 iterable = json.loads(obj)
61 iterable = json.loads(obj)
62 else:
62 else:
63 iterable = obj
63 iterable = obj
64
64
65 if isinstance(iterable, dict):
65 if isinstance(iterable, dict):
66 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
66 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
67 for k, v in iterable.items()}
67 for k, v in iterable.items()}
68 elif isinstance(iterable, (list, tuple)):
68 elif isinstance(iterable, (list, tuple)):
69 return [str(v) if isinstance(v, unicode) else v for v in iterable]
69 return [str(v) if isinstance(v, unicode) else v for v in iterable]
70
70
71 return iterable
71 return iterable
72
72
73
73
74 class MADReader(JRODataReader, ProcessingUnit):
74 class MADReader(JRODataReader, ProcessingUnit):
75
75
76 def __init__(self, **kwargs):
76 def __init__(self, **kwargs):
77
77
78 ProcessingUnit.__init__(self, **kwargs)
78 ProcessingUnit.__init__(self, **kwargs)
79
79
80 self.dataOut = Parameters()
80 self.dataOut = Parameters()
81 self.counter_records = 0
81 self.counter_records = 0
82 self.nrecords = None
82 self.nrecords = None
83 self.flagNoMoreFiles = 0
83 self.flagNoMoreFiles = 0
84 self.isConfig = False
84 self.isConfig = False
85 self.filename = None
85 self.filename = None
86 self.intervals = set()
86 self.intervals = set()
87
87
88 def setup(self,
88 def setup(self,
89 path=None,
89 path=None,
90 startDate=None,
90 startDate=None,
91 endDate=None,
91 endDate=None,
92 format=None,
92 format=None,
93 startTime=datetime.time(0, 0, 0),
93 startTime=datetime.time(0, 0, 0),
94 endTime=datetime.time(23, 59, 59),
94 endTime=datetime.time(23, 59, 59),
95 **kwargs):
95 **kwargs):
96
96
97 self.path = path
97 self.path = path
98 self.startDate = startDate
98 self.startDate = startDate
99 self.endDate = endDate
99 self.endDate = endDate
100 self.startTime = startTime
100 self.startTime = startTime
101 self.endTime = endTime
101 self.endTime = endTime
102 self.datatime = datetime.datetime(1900,1,1)
102 self.datatime = datetime.datetime(1900,1,1)
103 self.oneDDict = load_json(kwargs.get('oneDDict',
103 self.oneDDict = load_json(kwargs.get('oneDDict',
104 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
104 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
105 self.twoDDict = load_json(kwargs.get('twoDDict',
105 self.twoDDict = load_json(kwargs.get('twoDDict',
106 "{\"GDALT\": \"heightList\"}"))
106 "{\"GDALT\": \"heightList\"}"))
107 self.ind2DList = load_json(kwargs.get('ind2DList',
107 self.ind2DList = load_json(kwargs.get('ind2DList',
108 "[\"GDALT\"]"))
108 "[\"GDALT\"]"))
109 if self.path is None:
109 if self.path is None:
110 raise ValueError, 'The path is not valid'
110 raise ValueError, 'The path is not valid'
111
111
112 if format is None:
112 if format is None:
113 raise ValueError, 'The format is not valid choose simple or hdf5'
113 raise ValueError, 'The format is not valid choose simple or hdf5'
114 elif format.lower() in ('simple', 'txt'):
114 elif format.lower() in ('simple', 'txt'):
115 self.ext = '.txt'
115 self.ext = '.txt'
116 elif format.lower() in ('cedar',):
116 elif format.lower() in ('cedar',):
117 self.ext = '.001'
117 self.ext = '.001'
118 else:
118 else:
119 self.ext = '.hdf5'
119 self.ext = '.hdf5'
120
120
121 self.search_files(self.path)
121 self.search_files(self.path)
122 self.fileId = 0
122 self.fileId = 0
123
123
124 if not self.fileList:
124 if not self.fileList:
125 raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
125 raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
126
126
127 self.setNextFile()
127 self.setNextFile()
128
128
129 def search_files(self, path):
129 def search_files(self, path):
130 '''
130 '''
131 Searching for madrigal files in path
131 Searching for madrigal files in path
132 Creating a list of files to procces included in [startDate,endDate]
132 Creating a list of files to procces included in [startDate,endDate]
133
133
134 Input:
134 Input:
135 path - Path to find files
135 path - Path to find files
136 '''
136 '''
137
137
138 log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
138 log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
139 foldercounter = 0
139 foldercounter = 0
140 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
140 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
141 fileList0.sort()
141 fileList0.sort()
142
142
143 self.fileList = []
143 self.fileList = []
144 self.dateFileList = []
144 self.dateFileList = []
145
145
146 startDate = self.startDate - datetime.timedelta(1)
146 startDate = self.startDate - datetime.timedelta(1)
147 endDate = self.endDate + datetime.timedelta(1)
147 endDate = self.endDate + datetime.timedelta(1)
148
148
149 for thisFile in fileList0:
149 for thisFile in fileList0:
150 year = thisFile[3:7]
150 year = thisFile[3:7]
151 if not year.isdigit():
151 if not year.isdigit():
152 continue
152 continue
153
153
154 month = thisFile[7:9]
154 month = thisFile[7:9]
155 if not month.isdigit():
155 if not month.isdigit():
156 continue
156 continue
157
157
158 day = thisFile[9:11]
158 day = thisFile[9:11]
159 if not day.isdigit():
159 if not day.isdigit():
160 continue
160 continue
161
161
162 year, month, day = int(year), int(month), int(day)
162 year, month, day = int(year), int(month), int(day)
163 dateFile = datetime.date(year, month, day)
163 dateFile = datetime.date(year, month, day)
164
164
165 if (startDate > dateFile) or (endDate < dateFile):
165 if (startDate > dateFile) or (endDate < dateFile):
166 continue
166 continue
167
167
168 self.fileList.append(thisFile)
168 self.fileList.append(thisFile)
169 self.dateFileList.append(dateFile)
169 self.dateFileList.append(dateFile)
170
170
171 return
171 return
172
172
173 def parseHeader(self):
173 def parseHeader(self):
174 '''
174 '''
175 '''
175 '''
176
176
177 self.output = {}
177 self.output = {}
178 self.version = '2'
178 self.version = '2'
179 s_parameters = None
179 s_parameters = None
180 if self.ext == '.txt':
180 if self.ext == '.txt':
181 self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
181 self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
182 elif self.ext == '.hdf5':
182 elif self.ext == '.hdf5':
183 metadata = self.fp['Metadata']
183 metadata = self.fp['Metadata']
184 data = self.fp['Data']['Array Layout']
184 data = self.fp['Data']['Array Layout']
185 if 'Independent Spatial Parameters' in metadata:
185 if 'Independent Spatial Parameters' in metadata:
186 s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
186 s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
187 self.version = '3'
187 self.version = '3'
188 one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
188 one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
189 one_d = [1 for s in one]
189 one_d = [1 for s in one]
190 two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
190 two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
191 two_d = [2 for s in two]
191 two_d = [2 for s in two]
192 self.parameters = one + two
192 self.parameters = one + two
193 self.parameters_d = one_d + two_d
193 self.parameters_d = one_d + two_d
194
194
195 log.success('Parameters found: {}'.format(','.join(self.parameters)),
195 log.success('Parameters found: {}'.format(','.join(self.parameters)),
196 'MADReader')
196 'MADReader')
197 if s_parameters:
197 if s_parameters:
198 log.success('Spatial parameters: {}'.format(','.join(s_parameters)),
198 log.success('Spatial parameters: {}'.format(','.join(s_parameters)),
199 'MADReader')
199 'MADReader')
200
200
201 for param in self.oneDDict.keys():
201 for param in self.oneDDict.keys():
202 if param.lower() not in self.parameters:
202 if param.lower() not in self.parameters:
203 log.warning(
203 log.warning(
204 'Parameter {} not found will be ignored'.format(
204 'Parameter {} not found will be ignored'.format(
205 param),
205 param),
206 'MADReader')
206 'MADReader')
207 self.oneDDict.pop(param, None)
207 self.oneDDict.pop(param, None)
208
208
209 for param, value in self.twoDDict.items():
209 for param, value in self.twoDDict.items():
210 if param.lower() not in self.parameters:
210 if param.lower() not in self.parameters:
211 log.warning(
211 log.warning(
212 'Parameter {} not found, it will be ignored'.format(
212 'Parameter {} not found, it will be ignored'.format(
213 param),
213 param),
214 'MADReader')
214 'MADReader')
215 self.twoDDict.pop(param, None)
215 self.twoDDict.pop(param, None)
216 continue
216 continue
217 if isinstance(value, list):
217 if isinstance(value, list):
218 if value[0] not in self.output:
218 if value[0] not in self.output:
219 self.output[value[0]] = []
219 self.output[value[0]] = []
220 self.output[value[0]].append(None)
220 self.output[value[0]].append(None)
221
221
222 def parseData(self):
222 def parseData(self):
223 '''
223 '''
224 '''
224 '''
225
225
226 if self.ext == '.txt':
226 if self.ext == '.txt':
227 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
227 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
228 self.nrecords = self.data.shape[0]
228 self.nrecords = self.data.shape[0]
229 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
229 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
230 elif self.ext == '.hdf5':
230 elif self.ext == '.hdf5':
231 self.data = self.fp['Data']['Array Layout']
231 self.data = self.fp['Data']['Array Layout']
232 self.nrecords = len(self.data['timestamps'].value)
232 self.nrecords = len(self.data['timestamps'].value)
233 self.ranges = self.data['range'].value
233 self.ranges = self.data['range'].value
234
234
235 def setNextFile(self):
235 def setNextFile(self):
236 '''
236 '''
237 '''
237 '''
238
238
239 file_id = self.fileId
239 file_id = self.fileId
240
240
241 if file_id == len(self.fileList):
241 if file_id == len(self.fileList):
242 log.success('No more files', 'MADReader')
242 log.success('No more files', 'MADReader')
243 self.flagNoMoreFiles = 1
243 self.flagNoMoreFiles = 1
244 return 0
244 return 0
245
245
246 log.success(
246 log.success(
247 'Opening: {}'.format(self.fileList[file_id]),
247 'Opening: {}'.format(self.fileList[file_id]),
248 'MADReader'
248 'MADReader'
249 )
249 )
250
250
251 filename = os.path.join(self.path, self.fileList[file_id])
251 filename = os.path.join(self.path, self.fileList[file_id])
252
252
253 if self.filename is not None:
253 if self.filename is not None:
254 self.fp.close()
254 self.fp.close()
255
255
256 self.filename = filename
256 self.filename = filename
257 self.filedate = self.dateFileList[file_id]
257 self.filedate = self.dateFileList[file_id]
258
258
259 if self.ext=='.hdf5':
259 if self.ext=='.hdf5':
260 self.fp = h5py.File(self.filename, 'r')
260 self.fp = h5py.File(self.filename, 'r')
261 else:
261 else:
262 self.fp = open(self.filename, 'rb')
262 self.fp = open(self.filename, 'rb')
263
263
264 self.parseHeader()
264 self.parseHeader()
265 self.parseData()
265 self.parseData()
266 self.sizeOfFile = os.path.getsize(self.filename)
266 self.sizeOfFile = os.path.getsize(self.filename)
267 self.counter_records = 0
267 self.counter_records = 0
268 self.flagIsNewFile = 0
268 self.flagIsNewFile = 0
269 self.fileId += 1
269 self.fileId += 1
270
270
271 return 1
271 return 1
272
272
273 def readNextBlock(self):
273 def readNextBlock(self):
274
274
275 while True:
275 while True:
276 self.flagDiscontinuousBlock = 0
276 self.flagDiscontinuousBlock = 0
277 if self.flagIsNewFile:
277 if self.flagIsNewFile:
278 if not self.setNextFile():
278 if not self.setNextFile():
279 return 0
279 return 0
280
280
281 self.readBlock()
281 self.readBlock()
282
282
283 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
283 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
284 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
284 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
285 log.warning(
285 log.warning(
286 'Reading Record No. {}/{} -> {} [Skipping]'.format(
286 'Reading Record No. {}/{} -> {} [Skipping]'.format(
287 self.counter_records,
287 self.counter_records,
288 self.nrecords,
288 self.nrecords,
289 self.datatime.ctime()),
289 self.datatime.ctime()),
290 'MADReader')
290 'MADReader')
291 continue
291 continue
292 break
292 break
293
293
294 log.log(
294 log.log(
295 'Reading Record No. {}/{} -> {}'.format(
295 'Reading Record No. {}/{} -> {}'.format(
296 self.counter_records,
296 self.counter_records,
297 self.nrecords,
297 self.nrecords,
298 self.datatime.ctime()),
298 self.datatime.ctime()),
299 'MADReader')
299 'MADReader')
300
300
301 return 1
301 return 1
302
302
303 def readBlock(self):
303 def readBlock(self):
304 '''
304 '''
305 '''
305 '''
306 dum = []
306 dum = []
307 if self.ext == '.txt':
307 if self.ext == '.txt':
308 dt = self.data[self.counter_records][:6].astype(int)
308 dt = self.data[self.counter_records][:6].astype(int)
309 if datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]).date() > self.datatime.date():
309 if datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]).date() > self.datatime.date():
310 self.flagDiscontinuousBlock = 1
310 self.flagDiscontinuousBlock = 1
311 self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
311 self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
312 while True:
312 while True:
313 dt = self.data[self.counter_records][:6].astype(int)
313 dt = self.data[self.counter_records][:6].astype(int)
314 datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
314 datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
315 if datatime == self.datatime:
315 if datatime == self.datatime:
316 dum.append(self.data[self.counter_records])
316 dum.append(self.data[self.counter_records])
317 self.counter_records += 1
317 self.counter_records += 1
318 if self.counter_records == self.nrecords:
318 if self.counter_records == self.nrecords:
319 self.flagIsNewFile = True
319 self.flagIsNewFile = True
320 break
320 break
321 continue
321 continue
322 self.intervals.add((datatime-self.datatime).seconds)
322 self.intervals.add((datatime-self.datatime).seconds)
323 break
323 break
324 elif self.ext == '.hdf5':
324 elif self.ext == '.hdf5':
325 datatime = datetime.datetime.utcfromtimestamp(
325 datatime = datetime.datetime.utcfromtimestamp(
326 self.data['timestamps'][self.counter_records])
326 self.data['timestamps'][self.counter_records])
327 nHeights = len(self.ranges)
327 nHeights = len(self.ranges)
328 for n, param in enumerate(self.parameters):
328 for n, param in enumerate(self.parameters):
329 if self.parameters_d[n] == 1:
329 if self.parameters_d[n] == 1:
330 dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
330 dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
331 else:
331 else:
332 if self.version == '2':
332 if self.version == '2':
333 dum.append(self.data['2D Parameters'][param][self.counter_records])
333 dum.append(self.data['2D Parameters'][param][self.counter_records])
334 else:
334 else:
335 tmp = self.data['2D Parameters'][param].value.T
335 tmp = self.data['2D Parameters'][param].value.T
336 dum.append(tmp[self.counter_records])
336 dum.append(tmp[self.counter_records])
337 self.intervals.add((datatime-self.datatime).seconds)
337 self.intervals.add((datatime-self.datatime).seconds)
338 if datatime.date()>self.datatime.date():
338 if datatime.date()>self.datatime.date():
339 self.flagDiscontinuousBlock = 1
339 self.flagDiscontinuousBlock = 1
340 self.datatime = datatime
340 self.datatime = datatime
341 self.counter_records += 1
341 self.counter_records += 1
342 if self.counter_records == self.nrecords:
342 if self.counter_records == self.nrecords:
343 self.flagIsNewFile = True
343 self.flagIsNewFile = True
344
344
345 self.buffer = numpy.array(dum)
345 self.buffer = numpy.array(dum)
346 return
346 return
347
347
348 def set_output(self):
348 def set_output(self):
349 '''
349 '''
350 Storing data from buffer to dataOut object
350 Storing data from buffer to dataOut object
351 '''
351 '''
352
352
353 parameters = [None for __ in self.parameters]
353 parameters = [None for __ in self.parameters]
354
354
355 for param, attr in self.oneDDict.items():
355 for param, attr in self.oneDDict.items():
356 x = self.parameters.index(param.lower())
356 x = self.parameters.index(param.lower())
357 setattr(self.dataOut, attr, self.buffer[0][x])
357 setattr(self.dataOut, attr, self.buffer[0][x])
358
358
359 for param, value in self.twoDDict.items():
359 for param, value in self.twoDDict.items():
360 x = self.parameters.index(param.lower())
360 x = self.parameters.index(param.lower())
361 if self.ext == '.txt':
361 if self.ext == '.txt':
362 y = self.parameters.index(self.ind2DList[0].lower())
362 y = self.parameters.index(self.ind2DList[0].lower())
363 ranges = self.buffer[:,y]
363 ranges = self.buffer[:,y]
364 if self.ranges.size == ranges.size:
364 if self.ranges.size == ranges.size:
365 continue
365 continue
366 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
366 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
367 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
367 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
368 dummy[index] = self.buffer[:,x]
368 dummy[index] = self.buffer[:,x]
369 else:
369 else:
370 dummy = self.buffer[x]
370 dummy = self.buffer[x]
371
371
372 if isinstance(value, str):
372 if isinstance(value, str):
373 if value not in self.ind2DList:
373 if value not in self.ind2DList:
374 setattr(self.dataOut, value, dummy.reshape(1,-1))
374 setattr(self.dataOut, value, dummy.reshape(1,-1))
375 elif isinstance(value, list):
375 elif isinstance(value, list):
376 self.output[value[0]][value[1]] = dummy
376 self.output[value[0]][value[1]] = dummy
377 parameters[value[1]] = param
377 parameters[value[1]] = param
378
378
379 for key, value in self.output.items():
379 for key, value in self.output.items():
380 setattr(self.dataOut, key, numpy.array(value))
380 setattr(self.dataOut, key, numpy.array(value))
381
381
382 self.dataOut.parameters = [s for s in parameters if s]
382 self.dataOut.parameters = [s for s in parameters if s]
383 self.dataOut.heightList = self.ranges
383 self.dataOut.heightList = self.ranges
384 self.dataOut.utctime = (self.datatime - UT1970).total_seconds()
384 self.dataOut.utctime = (self.datatime - datetime.datetime(1970, 1, 1)).total_seconds()
385 self.dataOut.utctimeInit = self.dataOut.utctime
385 self.dataOut.utctimeInit = self.dataOut.utctime
386 self.dataOut.paramInterval = min(self.intervals)
386 self.dataOut.paramInterval = min(self.intervals)
387 self.dataOut.useLocalTime = False
387 self.dataOut.useLocalTime = False
388 self.dataOut.flagNoData = False
388 self.dataOut.flagNoData = False
389 self.dataOut.nrecords = self.nrecords
389 self.dataOut.nrecords = self.nrecords
390 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
390 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
391
391
392 def getData(self):
392 def getData(self):
393 '''
393 '''
394 Storing data from databuffer to dataOut object
394 Storing data from databuffer to dataOut object
395 '''
395 '''
396 if self.flagNoMoreFiles:
396 if self.flagNoMoreFiles:
397 self.dataOut.flagNoData = True
397 self.dataOut.flagNoData = True
398 log.error('No file left to process', 'MADReader')
398 log.error('No file left to process', 'MADReader')
399 return 0
399 return 0
400
400
401 if not self.readNextBlock():
401 if not self.readNextBlock():
402 self.dataOut.flagNoData = True
402 self.dataOut.flagNoData = True
403 return 0
403 return 0
404
404
405 self.set_output()
405 self.set_output()
406
406
407 return 1
407 return 1
408
408
409
409
410 class MADWriter(Operation):
410 class MADWriter(Operation):
411
411
412 missing = -32767
412 missing = -32767
413
413
414 def __init__(self, **kwargs):
414 def __init__(self, **kwargs):
415
415
416 Operation.__init__(self, **kwargs)
416 Operation.__init__(self, **kwargs)
417 self.dataOut = Parameters()
417 self.dataOut = Parameters()
418 self.counter = 0
418 self.path = None
419 self.path = None
419 self.fp = None
420 self.fp = None
420
421
421 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
422 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
422 metadata='{}', format='cedar', **kwargs):
423 metadata='{}', format='cedar', **kwargs):
423 '''
424 '''
424 Inputs:
425 Inputs:
425 path - path where files will be created
426 path - path where files will be created
426 oneDDict - json of one-dimensional parameters in record where keys
427 oneDDict - json of one-dimensional parameters in record where keys
427 are Madrigal codes (integers or mnemonics) and values the corresponding
428 are Madrigal codes (integers or mnemonics) and values the corresponding
428 dataOut attribute e.g: {
429 dataOut attribute e.g: {
429 'gdlatr': 'lat',
430 'gdlatr': 'lat',
430 'gdlonr': 'lon',
431 'gdlonr': 'lon',
431 'gdlat2':'lat',
432 'gdlat2':'lat',
432 'glon2':'lon'}
433 'glon2':'lon'}
433 ind2DList - list of independent spatial two-dimensional parameters e.g:
434 ind2DList - list of independent spatial two-dimensional parameters e.g:
434 ['heighList']
435 ['heighList']
435 twoDDict - json of two-dimensional parameters in record where keys
436 twoDDict - json of two-dimensional parameters in record where keys
436 are Madrigal codes (integers or mnemonics) and values the corresponding
437 are Madrigal codes (integers or mnemonics) and values the corresponding
437 dataOut attribute if multidimensional array specify as tupple
438 dataOut attribute if multidimensional array specify as tupple
438 ('attr', pos) e.g: {
439 ('attr', pos) e.g: {
439 'gdalt': 'heightList',
440 'gdalt': 'heightList',
440 'vn1p2': ('data_output', 0),
441 'vn1p2': ('data_output', 0),
441 'vn2p2': ('data_output', 1),
442 'vn2p2': ('data_output', 1),
442 'vn3': ('data_output', 2),
443 'vn3': ('data_output', 2),
443 'snl': ('data_SNR', 'db')
444 'snl': ('data_SNR', 'db')
444 }
445 }
445 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
446 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
446 '''
447 '''
447 if not self.isConfig:
448 if not self.isConfig:
448 self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
449 self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
449 self.isConfig = True
450 self.isConfig = True
450
451
451 self.dataOut = dataOut
452 self.dataOut = dataOut
452 self.putData()
453 self.putData()
453 return
454 return
454
455
455 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
456 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
456 '''
457 '''
457 Configure Operation
458 Configure Operation
458 '''
459 '''
459
460
460 self.path = path
461 self.path = path
461 self.blocks = kwargs.get('blocks', None)
462 self.blocks = kwargs.get('blocks', None)
462 self.counter = 0
463 self.counter = 0
463 self.oneDDict = load_json(oneDDict)
464 self.oneDDict = load_json(oneDDict)
464 self.twoDDict = load_json(twoDDict)
465 self.twoDDict = load_json(twoDDict)
465 self.ind2DList = load_json(ind2DList)
466 self.ind2DList = load_json(ind2DList)
466 meta = load_json(metadata)
467 meta = load_json(metadata)
467 self.kinst = meta.get('kinst')
468 self.kinst = meta.get('kinst')
468 self.kindat = meta.get('kindat')
469 self.kindat = meta.get('kindat')
469 self.catalog = meta.get('catalog', DEF_CATALOG)
470 self.catalog = meta.get('catalog', DEF_CATALOG)
470 self.header = meta.get('header', DEF_HEADER)
471 self.header = meta.get('header', DEF_HEADER)
471 if format == 'cedar':
472 if format == 'cedar':
472 self.ext = '.dat'
473 self.ext = '.dat'
473 self.extra_args = {}
474 self.extra_args = {}
474 elif format == 'hdf5':
475 elif format == 'hdf5':
475 self.ext = '.hdf5'
476 self.ext = '.hdf5'
476 self.extra_args = {'ind2DList': self.ind2DList}
477 self.extra_args = {'ind2DList': self.ind2DList}
477
478
478 self.keys = [k.lower() for k in self.twoDDict]
479 self.keys = [k.lower() for k in self.twoDDict]
479 if 'range' in self.keys:
480 if 'range' in self.keys:
480 self.keys.remove('range')
481 self.keys.remove('range')
481 if 'gdalt' in self.keys:
482 if 'gdalt' in self.keys:
482 self.keys.remove('gdalt')
483 self.keys.remove('gdalt')
483
484
484 def setFile(self):
485 def setFile(self):
485 '''
486 '''
486 Create new cedar file object
487 Create new cedar file object
487 '''
488 '''
488
489
489 self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
490 self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
490 date = datetime.datetime.fromtimestamp(self.dataOut.utctime)
491 date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
491
492
492 filename = '{}{}{}'.format(self.mnemonic,
493 filename = '{}{}{}'.format(self.mnemonic,
493 date.strftime('%Y%m%d_%H%M%S'),
494 date.strftime('%Y%m%d_%H%M%S'),
494 self.ext)
495 self.ext)
495
496
496 self.fullname = os.path.join(self.path, filename)
497 self.fullname = os.path.join(self.path, filename)
497
498
498 if os.path.isfile(self.fullname) :
499 if os.path.isfile(self.fullname) :
499 log.warning(
500 log.warning(
500 'Destination path {} already exists. Previous file deleted.'.format(
501 'Destination file {} already exists, previous file deleted.'.format(
501 self.fullname),
502 self.fullname),
502 'MADWriter')
503 'MADWriter')
503 os.remove(self.fullname)
504 os.remove(self.fullname)
504
505
505 try:
506 try:
506 log.success(
507 log.success(
507 'Creating file: {}'.format(self.fullname),
508 'Creating file: {}'.format(self.fullname),
508 'MADWriter')
509 'MADWriter')
509 self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
510 self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
510 except ValueError, e:
511 except ValueError, e:
511 log.error(
512 log.error(
512 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
513 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
513 'MADWriter')
514 'MADWriter')
514 return
515 return
515
516
516 return 1
517 return 1
517
518
518 def writeBlock(self):
519 def writeBlock(self):
519 '''
520 '''
520 Add data records to cedar file taking data from oneDDict and twoDDict
521 Add data records to cedar file taking data from oneDDict and twoDDict
521 attributes.
522 attributes.
522 Allowed parameters in: parcodes.tab
523 Allowed parameters in: parcodes.tab
523 '''
524 '''
524
525
525 startTime = datetime.datetime.fromtimestamp(self.dataOut.utctime)
526 startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
526 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
527 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
527 heights = self.dataOut.heightList
528 heights = self.dataOut.heightList
528
529
529 if self.ext == '.dat':
530 if self.ext == '.dat':
530 for key, value in self.twoDDict.items():
531 for key, value in self.twoDDict.items():
531 if isinstance(value, str):
532 if isinstance(value, str):
532 data = getattr(self.dataOut, value)
533 data = getattr(self.dataOut, value)
533 invalid = numpy.isnan(data)
534 invalid = numpy.isnan(data)
534 data[invalid] = self.missing
535 data[invalid] = self.missing
535 elif isinstance(value, (tuple, list)):
536 elif isinstance(value, (tuple, list)):
536 attr, key = value
537 attr, key = value
537 data = getattr(self.dataOut, attr)
538 data = getattr(self.dataOut, attr)
538 invalid = numpy.isnan(data)
539 invalid = numpy.isnan(data)
539 data[invalid] = self.missing
540 data[invalid] = self.missing
540
541
541 out = {}
542 out = {}
542 for key, value in self.twoDDict.items():
543 for key, value in self.twoDDict.items():
543 key = key.lower()
544 key = key.lower()
544 if isinstance(value, str):
545 if isinstance(value, str):
545 if 'db' in value.lower():
546 if 'db' in value.lower():
546 tmp = getattr(self.dataOut, value.replace('_db', ''))
547 tmp = getattr(self.dataOut, value.replace('_db', ''))
547 SNRavg = numpy.average(tmp, axis=0)
548 SNRavg = numpy.average(tmp, axis=0)
548 tmp = 10*numpy.log10(SNRavg)
549 tmp = 10*numpy.log10(SNRavg)
549 else:
550 else:
550 tmp = getattr(self.dataOut, value)
551 tmp = getattr(self.dataOut, value)
551 out[key] = tmp.flatten()
552 out[key] = tmp.flatten()
552 elif isinstance(value, (tuple, list)):
553 elif isinstance(value, (tuple, list)):
553 attr, x = value
554 attr, x = value
554 data = getattr(self.dataOut, attr)
555 data = getattr(self.dataOut, attr)
555 out[key] = data[int(x)]
556 out[key] = data[int(x)]
556
557
557 a = numpy.array([out[k] for k in self.keys])
558 a = numpy.array([out[k] for k in self.keys])
558 nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
559 nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
559 index = numpy.where(nrows == False)[0]
560 index = numpy.where(nrows == False)[0]
560
561
561 rec = madrigal.cedar.MadrigalDataRecord(
562 rec = madrigal.cedar.MadrigalDataRecord(
562 self.kinst,
563 self.kinst,
563 self.kindat,
564 self.kindat,
564 startTime.year,
565 startTime.year,
565 startTime.month,
566 startTime.month,
566 startTime.day,
567 startTime.day,
567 startTime.hour,
568 startTime.hour,
568 startTime.minute,
569 startTime.minute,
569 startTime.second,
570 startTime.second,
570 startTime.microsecond/10000,
571 startTime.microsecond/10000,
571 endTime.year,
572 endTime.year,
572 endTime.month,
573 endTime.month,
573 endTime.day,
574 endTime.day,
574 endTime.hour,
575 endTime.hour,
575 endTime.minute,
576 endTime.minute,
576 endTime.second,
577 endTime.second,
577 endTime.microsecond/10000,
578 endTime.microsecond/10000,
578 self.oneDDict.keys(),
579 self.oneDDict.keys(),
579 self.twoDDict.keys(),
580 self.twoDDict.keys(),
580 len(index),
581 len(index),
581 **self.extra_args
582 **self.extra_args
582 )
583 )
583
584
584 # Setting 1d values
585 # Setting 1d values
585 for key in self.oneDDict:
586 for key in self.oneDDict:
586 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
587 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
587
588
588 # Setting 2d values
589 # Setting 2d values
589 nrec = 0
590 nrec = 0
590 for n in index:
591 for n in index:
591 for key in out:
592 for key in out:
592 rec.set2D(key, nrec, out[key][n])
593 rec.set2D(key, nrec, out[key][n])
593 nrec += 1
594 nrec += 1
594
595
595 self.fp.append(rec)
596 self.fp.append(rec)
596 if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
597 if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
597 self.fp.dump()
598 self.fp.dump()
598 if self.counter % 100 == 0 and self.counter > 0:
599 if self.counter % 100 == 0 and self.counter > 0:
599 log.log(
600 log.log(
600 'Writing {} records'.format(
601 'Writing {} records'.format(
601 self.counter),
602 self.counter),
602 'MADWriter')
603 'MADWriter')
603
604
604 def setHeader(self):
605 def setHeader(self):
605 '''
606 '''
606 Create an add catalog and header to cedar file
607 Create an add catalog and header to cedar file
607 '''
608 '''
608
609
609 log.success('Closing file {}'.format(self.fullname), 'MADWriter')
610 log.success('Closing file {}'.format(self.fullname), 'MADWriter')
610
611
611 if self.ext == '.dat':
612 if self.ext == '.dat':
612 self.fp.write()
613 self.fp.write()
613 else:
614 else:
614 self.fp.dump()
615 self.fp.dump()
615 self.fp.close()
616 self.fp.close()
616
617
617 header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
618 header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
618 header.createCatalog(**self.catalog)
619 header.createCatalog(**self.catalog)
619 header.createHeader(**self.header)
620 header.createHeader(**self.header)
620 header.write()
621 header.write()
621
622
622 def putData(self):
623 def putData(self):
623
624
624 if self.dataOut.flagNoData:
625 if self.dataOut.flagNoData:
625 return 0
626 return 0
626
627
627 if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
628 if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
628 if self.counter > 0:
629 if self.counter > 0:
629 self.setHeader()
630 self.setHeader()
630 self.counter = 0
631 self.counter = 0
631
632
632 if self.counter == 0:
633 if self.counter == 0:
633 self.setFile()
634 self.setFile()
634
635
635 self.writeBlock()
636 self.writeBlock()
636 self.counter += 1
637 self.counter += 1
637
638
638 def close(self):
639 def close(self):
639
640
640 if self.counter > 0:
641 if self.counter > 0:
641 self.setHeader()
642 self.setHeader()
General Comments 0
You need to be logged in to leave comments. Login now