##// END OF EJS Templates
schain-jc
RSS
MADReader support for HDF5 (mad2 & mad3)
jespinoza -
r1065:9a4424b32cac
parent child
Show More
Side by Side Unified
Context file:
r1065:9a4424b32cac Loading diff...:
Show file before | Show file after | Hide comments
@@ -4,4 +4,4 Created on Feb 7, 2012
4 @author $Author$
4 @author $Author$
5 @version $Id$
5 @version $Id$
6 '''
6 '''
7 __version__ = "2.3"
7 __version__ = '2.3'
Show file before | Show file after | Hide comments
@@ -145,6 +145,7 class PlotData(Operation, Process):
145 ax.figure.add_axes(nax)
145 ax.figure.add_axes(nax)
146 return nax
146 return nax
147
147
148 self.setup()
148
149
149 def setup(self):
150 def setup(self):
150 '''
151 '''
@@ -335,7 +336,6 class PlotData(Operation, Process):
335 if self.data:
336 if self.data:
336 self.__plot()
337 self.__plot()
337
338
338
339 class PlotSpectraData(PlotData):
339 class PlotSpectraData(PlotData):
340 '''
340 '''
341 Plot for Spectra data
341 Plot for Spectra data
@@ -727,6 +727,7 class PlotParamData(PlotRTIData):
727 self.nplots = self.nrows
727 self.nplots = self.nrows
728 if self.showSNR:
728 if self.showSNR:
729 self.nrows += 1
729 self.nrows += 1
730 self.nplots += 1
730
731
731 self.ylabel = 'Height [Km]'
732 self.ylabel = 'Height [Km]'
732 self.titles = self.data.parameters \
733 self.titles = self.data.parameters \
@@ -779,4 +780,4 class PlotOuputData(PlotParamData):
779 '''
780 '''
780
781
781 CODE = 'output'
782 CODE = 'output'
782 colormap = 'seismic' No newline at end of file
783 colormap = 'seismic'
Show file before | Show file after | Hide comments
@@ -1115,25 +1115,23 class Parameters1Plot(Figure):
1115
1115
1116 x = dataOut.getTimeRange1(dataOut.paramInterval)
1116 x = dataOut.getTimeRange1(dataOut.paramInterval)
1117 y = dataOut.heightList
1117 y = dataOut.heightList
1118 z = data_param[channelIndexList,parameterIndex,:].copy()
1119
1118
1120 zRange = dataOut.abscissaList
1119 if dataOut.data_param.ndim == 3:
1121 # nChannels = z.shape[0] #Number of wind dimensions estimated
1120 z = dataOut.data_param[channelIndexList,parameterIndex,:]
1122 # thisDatetime = dataOut.datatime
1121 else:
1122 z = dataOut.data_param[channelIndexList,:]
1123
1123
1124 if dataOut.data_SNR is not None:
1124 if dataOut.data_SNR is not None:
1125 SNRarray = dataOut.data_SNR[channelIndexList,:]
1125 if dataOut.data_SNR.ndim == 2:
1126 SNRdB = 10*numpy.log10(SNRarray)
1126 SNRavg = numpy.average(dataOut.data_SNR, axis=0)
1127 # SNRavgdB = 10*numpy.log10(SNRavg)
1127 else:
1128 ind = numpy.where(SNRdB < 10**(SNRthresh/10))
1128 SNRavg = dataOut.data_SNR
1129 z[ind] = numpy.nan
1129 SNRdB = 10*numpy.log10(SNRavg)
1130
1130
1131 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1131 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1132 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
1132 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
1133 xlabel = ""
1133 xlabel = ""
1134 ylabel = "Range (Km)"
1134 ylabel = "Range (Km)"
1135
1136 if (SNR and not onlySNR): nplots = 2*nplots
1137
1135
1138 if onlyPositive:
1136 if onlyPositive:
1139 colormap = "jet"
1137 colormap = "jet"
@@ -1152,8 +1150,8 class Parameters1Plot(Figure):
1152
1150
1153 if ymin == None: ymin = numpy.nanmin(y)
1151 if ymin == None: ymin = numpy.nanmin(y)
1154 if ymax == None: ymax = numpy.nanmax(y)
1152 if ymax == None: ymax = numpy.nanmax(y)
1155 if zmin == None: zmin = numpy.nanmin(zRange)
1153 if zmin == None: zmin = numpy.nanmin(z)
1156 if zmax == None: zmax = numpy.nanmax(zRange)
1154 if zmax == None: zmax = numpy.nanmax(z)
1157
1155
1158 if SNR:
1156 if SNR:
1159 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
1157 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
@@ -1203,19 +1201,18 class Parameters1Plot(Figure):
1203 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
1201 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
1204 ticksize=9, cblabel=zlabel, cbsize="1%")
1202 ticksize=9, cblabel=zlabel, cbsize="1%")
1205
1203
1206 if SNR:
1204 if SNR:
1207 title = "Channel %d Signal Noise Ratio (SNR): %s" %(channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1205 title = "Channel %d Signal Noise Ratio (SNR): %s" %(channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1208 axes = self.axesList[(j)*self.__nsubplots]
1206 axes = self.axesList[(j)*self.__nsubplots]
1209 if not onlySNR:
1207 if not onlySNR:
1210 axes = self.axesList[(j + 1)*self.__nsubplots]
1208 axes = self.axesList[(j + 1)*self.__nsubplots]
1211
1212 axes = self.axesList[(j + nGraphsByChannel-1)]
1213
1209
1214 z1 = SNRdB[i,:].reshape((1,-1))
1210 axes = self.axesList[(j + nGraphsByChannel-1)]
1215 axes.pcolorbuffer(x, y, z1,
1211 z1 = SNRdB.reshape((1,-1))
1216 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1212 axes.pcolorbuffer(x, y, z1,
1217 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap="jet",
1213 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1218 ticksize=9, cblabel=zlabel, cbsize="1%")
1214 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap="jet",
1215 ticksize=9, cblabel=zlabel, cbsize="1%")
1219
1216
1220
1217
1221
1218
Show file before | Show file after | Hide comments
@@ -4,7 +4,7 import sys
4 import matplotlib
4 import matplotlib
5
5
6 if 'linux' in sys.platform:
6 if 'linux' in sys.platform:
7 matplotlib.use("TKAgg")
7 matplotlib.use("GTK3Agg")
8
8
9 if 'darwin' in sys.platform:
9 if 'darwin' in sys.platform:
10 matplotlib.use('TKAgg')
10 matplotlib.use('TKAgg')
Show file before | Show file after | Hide comments
@@ -18,4 +18,3 from jroIO_madrigal import *
18 from bltrIO_param import *
18 from bltrIO_param import *
19 from jroIO_bltr import *
19 from jroIO_bltr import *
20 from jroIO_mira35c import *
20 from jroIO_mira35c import *
21
Show file before | Show file after | Hide comments
@@ -1269,7 +1269,11 class JRODataReader(JRODataIO):
1269 cursor=None,
1269 cursor=None,
1270 warnings=True,
1270 warnings=True,
1271 verbose=True,
1271 verbose=True,
1272 server=None):
1272 server=None,
1273 format=None,
1274 oneDDict=None,
1275 twoDDict=None,
1276 ind2DList=None):
1273 if server is not None:
1277 if server is not None:
1274 if 'tcp://' in server:
1278 if 'tcp://' in server:
1275 address = server
1279 address = server
@@ -1457,7 +1461,11 class JRODataReader(JRODataIO):
1457 cursor=None,
1461 cursor=None,
1458 warnings=True,
1462 warnings=True,
1459 server=None,
1463 server=None,
1460 verbose=True, **kwargs):
1464 verbose=True,
1465 format=None,
1466 oneDDict=None,
1467 twoDDict=None,
1468 ind2DList=None, **kwargs):
1461
1469
1462 if not(self.isConfig):
1470 if not(self.isConfig):
1463 self.setup(path=path,
1471 self.setup(path=path,
@@ -1480,7 +1488,11 class JRODataReader(JRODataIO):
1480 cursor=cursor,
1488 cursor=cursor,
1481 warnings=warnings,
1489 warnings=warnings,
1482 server=server,
1490 server=server,
1483 verbose=verbose)
1491 verbose=verbose,
1492 format=format,
1493 oneDDict=oneDDict,
1494 twoDDict=twoDDict,
1495 ind2DList=ind2DList)
1484 self.isConfig = True
1496 self.isConfig = True
1485 if server is None:
1497 if server is None:
1486 self.getData()
1498 self.getData()
Show file before | Show file after | Hide comments
@@ -8,9 +8,11 import os
8 import sys
8 import sys
9 import time
9 import time
10 import json
10 import json
11 import glob
11 import datetime
12 import datetime
12
13
13 import numpy
14 import numpy
15 import h5py
14
16
15 try:
17 try:
16 import madrigal
18 import madrigal
@@ -18,10 +20,12 try:
18 except:
20 except:
19 print 'You should install "madrigal library" module if you want to read/write Madrigal data'
21 print 'You should install "madrigal library" module if you want to read/write Madrigal data'
20
22
21 from schainpy.model.proc.jroproc_base import Operation
23 from schainpy.model.io.jroIO_base import JRODataReader
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
22 from schainpy.model.data.jrodata import Parameters
25 from schainpy.model.data.jrodata import Parameters
26 from schainpy.utils import log
27
23
28
24 MISSING = -32767
25 DEF_CATALOG = {
29 DEF_CATALOG = {
26 'principleInvestigator': 'Marco Milla',
30 'principleInvestigator': 'Marco Milla',
27 'expPurpose': None,
31 'expPurpose': None,
@@ -47,19 +51,355 MNEMONICS = {
47 1002: 'obr',
51 1002: 'obr',
48 }
52 }
49
53
54 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
55
50 def load_json(obj):
56 def load_json(obj):
51 '''
57 '''
52 Parse json as string instead of unicode
58 Parse json as string instead of unicode
53 '''
59 '''
54
60
55 if isinstance(obj, str):
61 if isinstance(obj, str):
56 obj = json.loads(obj)
62 iterable = json.loads(obj)
63
64 if isinstance(iterable, dict):
65 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
66 for k, v in iterable.items()}
67 elif isinstance(iterable, (list, tuple)):
68 return [str(v) if isinstance(v, unicode) else v for v in iterable]
69
70 return iterable
71
72
73 class MADReader(JRODataReader, ProcessingUnit):
74
75 def __init__(self, **kwargs):
76
77 ProcessingUnit.__init__(self, **kwargs)
78
79 self.dataOut = Parameters()
80 self.counter_records = 0
81 self.nrecords = None
82 self.flagNoMoreFiles = 0
83 self.isConfig = False
84 self.filename = None
85 self.intervals = set()
86
87 def setup(self,
88 path=None,
89 startDate=None,
90 endDate=None,
91 format=None,
92 startTime=datetime.time(0, 0, 0),
93 endTime=datetime.time(23, 59, 59),
94 **kwargs):
95
96 self.started = True
97 self.path = path
98 self.startDate = startDate
99 self.endDate = endDate
100 self.startTime = startTime
101 self.endTime = endTime
102 self.datatime = datetime.datetime(1900,1,1)
103 self.oneDDict = load_json(kwargs.get('oneDDict',
104 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
105 self.twoDDict = load_json(kwargs.get('twoDDict',
106 "{\"GDALT\": \"heightList\"}"))
107 self.ind2DList = load_json(kwargs.get('ind2DList',
108 "[\"GDALT\"]"))
109 if self.path is None:
110 raise ValueError, 'The path is not valid'
111
112 if format is None:
113 raise ValueError, 'The format is not valid choose simple or hdf5'
114 elif format.lower() in ('simple', 'txt'):
115 self.ext = '.txt'
116 elif format.lower() in ('cedar',):
117 self.ext = '.001'
118 else:
119 self.ext = '.hdf5'
120
121 self.search_files(self.path)
122 self.fileId = 0
123
124 if not self.fileList:
125 raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
126
127 self.setNextFile()
128
129 def search_files(self, path):
130 '''
131 Searching for madrigal files in path
132 Creating a list of files to procces included in [startDate,endDate]
133
134 Input:
135 path - Path to find files
136 '''
137
138 print 'Searching files {} in {} '.format(self.ext, path)
139 foldercounter = 0
140 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
141 fileList0.sort()
142
143 self.fileList = []
144 self.dateFileList = []
57
145
58 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
146 startDate = self.startDate - datetime.timedelta(1)
59 for k, v in obj.items()}
147 endDate = self.endDate + datetime.timedelta(1)
148
149 for thisFile in fileList0:
150 year = thisFile[3:7]
151 if not year.isdigit():
152 continue
153
154 month = thisFile[7:9]
155 if not month.isdigit():
156 continue
157
158 day = thisFile[9:11]
159 if not day.isdigit():
160 continue
161
162 year, month, day = int(year), int(month), int(day)
163 dateFile = datetime.date(year, month, day)
164
165 if (startDate > dateFile) or (endDate < dateFile):
166 continue
167
168 self.fileList.append(thisFile)
169 self.dateFileList.append(dateFile)
170
171 return
172
173 def parseHeader(self):
174 '''
175 '''
176
177 self.output = {}
178 self.version = '2'
179 s_parameters = None
180 if self.ext == '.txt':
181 self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
182 elif self.ext == '.hdf5':
183 metadata = self.fp['Metadata']
184 data = self.fp['Data']['Array Layout']
185 if 'Independent Spatial Parameters' in metadata:
186 s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
187 self.version = '3'
188 one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
189 one_d = [1 for s in one]
190 two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
191 two_d = [2 for s in two]
192 self.parameters = one + two
193 self.parameters_d = one_d + two_d
194
195 log.success('Parameters found: {}'.format(','.join(self.parameters)),
196 'MADReader')
197 if s_parameters:
198 log.success('Spatial parameters: {}'.format(','.join(s_parameters)),
199 'MADReader')
200
201 for param in self.oneDDict.keys():
202 if param.lower() not in self.parameters:
203 print('\x1b[33m[Warning]\x1b[0m Parameter \x1b[1;32m{}\x1b[0m not found will be ignored'.format(
204 param
205 ))
206 self.oneDDict.pop(param, None)
207
208 for param, value in self.twoDDict.items():
209 if param.lower() not in self.parameters:
210 print('\x1b[33m[Warning]\x1b[0m Parameter \x1b[1;32m{}\x1b[0m not found will be ignored'.format(
211 param
212 ))
213 self.twoDDict.pop(param, None)
214 continue
215 if isinstance(value, list):
216 if value[0] not in self.output:
217 self.output[value[0]] = []
218 self.output[value[0]].append(None)
219
220 def parseData(self):
221 '''
222 '''
223
224 if self.ext == '.txt':
225 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
226 self.nrecords = self.data.shape[0]
227 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
228 elif self.ext == '.hdf5':
229 self.data = self.fp['Data']['Array Layout']
230 self.nrecords = len(self.data['timestamps'].value)
231 self.ranges = self.data['range'].value
232
233 def setNextFile(self):
234 '''
235 '''
236
237 file_id = self.fileId
238
239 if file_id == len(self.fileList):
240 print '\nNo more files in the folder'
241 print 'Total number of file(s) read : {}'.format(self.fileId)
242 self.flagNoMoreFiles = 1
243 return 0
244
245 print('\x1b[32m[Info]\x1b[0m Opening: {}'.format(
246 self.fileList[file_id]
247 ))
248 filename = os.path.join(self.path, self.fileList[file_id])
249
250 if self.filename is not None:
251 self.fp.close()
252
253 self.filename = filename
254 self.filedate = self.dateFileList[file_id]
255
256 if self.ext=='.hdf5':
257 self.fp = h5py.File(self.filename, 'r')
258 else:
259 self.fp = open(self.filename, 'rb')
260
261 self.parseHeader()
262 self.parseData()
263 self.sizeOfFile = os.path.getsize(self.filename)
264 self.counter_records = 0
265 self.flagIsNewFile = 0
266 self.fileId += 1
267
268 return 1
269
270 def readNextBlock(self):
271
272 while True:
273
274 if self.flagIsNewFile:
275 if not self.setNextFile():
276 return 0
277
278 self.readBlock()
279
280 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
281 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
282 print "\x1b[32m[Reading]\x1b[0m Record No. %d/%d -> %s \x1b[33m[Skipping]\x1b[0m" %(
283 self.counter_records,
284 self.nrecords,
285 self.datatime.ctime())
286 continue
287 break
288
289 print "\x1b[32m[Reading]\x1b[0m Record No. %d/%d -> %s" %(
290 self.counter_records,
291 self.nrecords,
292 self.datatime.ctime())
293
294 return 1
295
296 def readBlock(self):
297 '''
298 '''
299 dum = []
300 if self.ext == '.txt':
301 dt = self.data[self.counter_records][:6].astype(int)
302 self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
303 while True:
304 dt = self.data[self.counter_records][:6].astype(int)
305 datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
306 if datatime == self.datatime:
307 dum.append(self.data[self.counter_records])
308 self.counter_records += 1
309 if self.counter_records == self.nrecords:
310 self.flagIsNewFile = True
311 break
312 continue
313 self.intervals.add((datatime-self.datatime).seconds)
314 break
315 elif self.ext == '.hdf5':
316 datatime = datetime.datetime.utcfromtimestamp(
317 self.data['timestamps'][self.counter_records])
318 nHeights = len(self.ranges)
319 for n, param in enumerate(self.parameters):
320 if self.parameters_d[n] == 1:
321 dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
322 else:
323 if self.version == '2':
324 dum.append(self.data['2D Parameters'][param][self.counter_records])
325 else:
326 tmp = self.data['2D Parameters'][param].value.T
327 dum.append(tmp[self.counter_records])
328 self.intervals.add((datatime-self.datatime).seconds)
329 self.datatime = datatime
330 self.counter_records += 1
331 if self.counter_records == self.nrecords:
332 self.flagIsNewFile = True
333
334 self.buffer = numpy.array(dum)
335 return
336
337 def set_output(self):
338 '''
339 Storing data from buffer to dataOut object
340 '''
341
342 parameters = [None for __ in self.parameters]
343
344 for param, attr in self.oneDDict.items():
345 x = self.parameters.index(param.lower())
346 setattr(self.dataOut, attr, self.buffer[0][x])
347
348 for param, value in self.twoDDict.items():
349 x = self.parameters.index(param.lower())
350 if self.ext == '.txt':
351 y = self.parameters.index(self.ind2DList[0].lower())
352 ranges = self.buffer[:,y]
353 if self.ranges.size == ranges.size:
354 continue
355 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
356 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
357 dummy[index] = self.buffer[:,x]
358 else:
359
360 dummy = self.buffer[x]
361
362 if isinstance(value, str):
363 if value not in self.ind2DList:
364 setattr(self.dataOut, value, dummy.reshape(1,-1))
365 elif isinstance(value, list):
366 self.output[value[0]][value[1]] = dummy
367 parameters[value[1]] = param
368
369 for key, value in self.output.items():
370 setattr(self.dataOut, key, numpy.array(value))
371
372 self.dataOut.parameters = [s for s in parameters if s]
373 self.dataOut.heightList = self.ranges
374 self.dataOut.utctime = (self.datatime - UT1970).total_seconds()
375 self.dataOut.utctimeInit = self.dataOut.utctime
376 self.dataOut.paramInterval = min(self.intervals)
377 self.dataOut.useLocalTime = False
378 self.dataOut.flagNoData = False
379 self.dataOut.started = self.started
380
381 def getData(self):
382 '''
383 Storing data from databuffer to dataOut object
384 '''
385 if self.flagNoMoreFiles:
386 self.dataOut.flagNoData = True
387 print 'No file left to process'
388 return 0
389
390 if not self.readNextBlock():
391 self.dataOut.flagNoData = True
392 return 0
393
394 self.set_output()
395
396 return 1
60
397
61
398
62 class MAD2Writer(Operation):
399 class MAD2Writer(Operation):
400
401 missing = -32767
402 ext = '.dat'
63
403
64 def __init__(self, **kwargs):
404 def __init__(self, **kwargs):
65
405
@@ -67,24 +407,21 class MAD2Writer(Operation):
67 self.dataOut = Parameters()
407 self.dataOut = Parameters()
68 self.path = None
408 self.path = None
69 self.dataOut = None
409 self.dataOut = None
70 self.ext = '.dat'
71
72 return
73
410
74 def run(self, dataOut, path, oneDList, twoDParam='', twoDList='{}', metadata='{}', **kwargs):
411 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}', metadata='{}', **kwargs):
75 '''
412 '''
76 Inputs:
413 Inputs:
77 path - path where files will be created
414 path - path where files will be created
78 oneDList - json of one-dimensional parameters in record where keys
415 oneDDict - json of one-dimensional parameters in record where keys
79 are Madrigal codes (integers or mnemonics) and values the corresponding
416 are Madrigal codes (integers or mnemonics) and values the corresponding
80 dataOut attribute e.g: {
417 dataOut attribute e.g: {
81 'gdlatr': 'lat',
418 'gdlatr': 'lat',
82 'gdlonr': 'lon',
419 'gdlonr': 'lon',
83 'gdlat2':'lat',
420 'gdlat2':'lat',
84 'glon2':'lon'}
421 'glon2':'lon'}
85 twoDParam - independent parameter to get the number of rows e.g:
422 ind2DList - list of independent spatial two-dimensional parameters e.g:
86 heighList
423 ['heighList']
87 twoDList - json of two-dimensional parameters in record where keys
424 twoDDict - json of two-dimensional parameters in record where keys
88 are Madrigal codes (integers or mnemonics) and values the corresponding
425 are Madrigal codes (integers or mnemonics) and values the corresponding
89 dataOut attribute if multidimensional array specify as tupple
426 dataOut attribute if multidimensional array specify as tupple
90 ('attr', pos) e.g: {
427 ('attr', pos) e.g: {
@@ -97,13 +434,13 class MAD2Writer(Operation):
97 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
434 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
98 '''
435 '''
99 if not self.isConfig:
436 if not self.isConfig:
100 self.setup(dataOut, path, oneDList, twoDParam, twoDList, metadata, **kwargs)
437 self.setup(dataOut, path, oneDDict, ind2DList, twoDDict, metadata, **kwargs)
101 self.isConfig = True
438 self.isConfig = True
102
439
103 self.putData()
440 self.putData()
104 return
441 return
105
442
106 def setup(self, dataOut, path, oneDList, twoDParam, twoDList, metadata, **kwargs):
443 def setup(self, dataOut, path, oneDDict, ind2DList, twoDDict, metadata, **kwargs):
107 '''
444 '''
108 Configure Operation
445 Configure Operation
109 '''
446 '''
@@ -113,9 +450,9 class MAD2Writer(Operation):
113 self.path = path
450 self.path = path
114 self.blocks = kwargs.get('blocks', None)
451 self.blocks = kwargs.get('blocks', None)
115 self.counter = 0
452 self.counter = 0
116 self.oneDList = load_json(oneDList)
453 self.oneDDict = load_json(oneDDict)
117 self.twoDList = load_json(twoDList)
454 self.twoDDict = load_json(twoDDict)
118 self.twoDParam = twoDParam
455 self.ind2DList = load_json(ind2DList)
119 meta = load_json(metadata)
456 meta = load_json(metadata)
120 self.kinst = meta.get('kinst')
457 self.kinst = meta.get('kinst')
121 self.kindat = meta.get('kindat')
458 self.kindat = meta.get('kindat')
@@ -154,14 +491,14 class MAD2Writer(Operation):
154
491
155 def writeBlock(self):
492 def writeBlock(self):
156 '''
493 '''
157 Add data records to cedar file taking data from oneDList and twoDList
494 Add data records to cedar file taking data from oneDDict and twoDDict
158 attributes.
495 attributes.
159 Allowed parameters in: parcodes.tab
496 Allowed parameters in: parcodes.tab
160 '''
497 '''
161
498
162 startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
499 startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
163 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
500 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
164 nrows = len(getattr(self.dataOut, self.twoDParam))
501 nrows = len(getattr(self.dataOut, self.ind2DList))
165
502
166 rec = madrigal.cedar.MadrigalDataRecord(
503 rec = madrigal.cedar.MadrigalDataRecord(
167 self.kinst,
504 self.kinst,
@@ -180,20 +517,20 class MAD2Writer(Operation):
180 endTime.minute,
517 endTime.minute,
181 endTime.second,
518 endTime.second,
182 endTime.microsecond/10000,
519 endTime.microsecond/10000,
183 self.oneDList.keys(),
520 self.oneDDict.keys(),
184 self.twoDList.keys(),
521 self.twoDDict.keys(),
185 nrows
522 nrows
186 )
523 )
187
524
188 # Setting 1d values
525 # Setting 1d values
189 for key in self.oneDList:
526 for key in self.oneDDict:
190 rec.set1D(key, getattr(self.dataOut, self.oneDList[key]))
527 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
191
528
192 # Setting 2d values
529 # Setting 2d values
193 invalid = numpy.isnan(self.dataOut.data_output)
530 invalid = numpy.isnan(self.dataOut.data_output)
194 self.dataOut.data_output[invalid] = MISSING
531 self.dataOut.data_output[invalid] = self.missing
195 out = {}
532 out = {}
196 for key, value in self.twoDList.items():
533 for key, value in self.twoDDict.items():
197 if isinstance(value, str):
534 if isinstance(value, str):
198 out[key] = getattr(self.dataOut, value)
535 out[key] = getattr(self.dataOut, value)
199 elif isinstance(value, tuple):
536 elif isinstance(value, tuple):
Show file before | Show file after | Hide comments
@@ -256,15 +256,6 class SpectraReader(JRODataReader, ProcessingUnit):
256 Exceptions:
256 Exceptions:
257 Si un bloque leido no es un bloque valido
257 Si un bloque leido no es un bloque valido
258 """
258 """
259 print ' ======================================================== '
260 print ' '
261 print ' '
262 print self.processingHeaderObj.totalSpectra, 'TotalSpectra', type(self.processingHeaderObj.totalSpectra)
263 print self.processingHeaderObj.spectraComb, 'SpectraComb', type(self.processingHeaderObj.spectraComb)
264 print ' '
265 print ' '
266 print ' ======================================================== '
267
268
259
269 blockOk_flag = False
260 blockOk_flag = False
270 fpointer = self.fp.tell()
261 fpointer = self.fp.tell()
Show file before | Show file after | Hide comments
@@ -1401,6 +1401,9 class SpectralMoments(Operation):
1401
1401
1402 dataOut.data_param = data_param[:,1:,:]
1402 dataOut.data_param = data_param[:,1:,:]
1403 dataOut.data_SNR = data_param[:,0]
1403 dataOut.data_SNR = data_param[:,0]
1404 dataOut.data_DOP = data_param[:,1]
1405 dataOut.data_MEAN = data_param[:,2]
1406 dataOut.data_STD = data_param[:,3]
1404 return
1407 return
1405
1408
1406 def __calculateMoments(self, oldspec, oldfreq, n0,
1409 def __calculateMoments(self, oldspec, oldfreq, n0,
Show file before | Show file after | Hide comments
@@ -113,6 +113,8 class Data(object):
113 self.__heights = []
113 self.__heights = []
114 self.__all_heights = set()
114 self.__all_heights = set()
115 for plot in self.plottypes:
115 for plot in self.plottypes:
116 if 'snr' in plot:
117 plot = 'snr'
116 self.data[plot] = {}
118 self.data[plot] = {}
117
119
118 def shape(self, key):
120 def shape(self, key):
@@ -138,8 +140,9 class Data(object):
138 self.parameters = getattr(dataOut, 'parameters', [])
140 self.parameters = getattr(dataOut, 'parameters', [])
139 self.pairs = dataOut.pairsList
141 self.pairs = dataOut.pairsList
140 self.channels = dataOut.channelList
142 self.channels = dataOut.channelList
141 self.xrange = (dataOut.getFreqRange(1)/1000. , dataOut.getAcfRange(1) , dataOut.getVelRange(1))
142 self.interval = dataOut.getTimeInterval()
143 self.interval = dataOut.getTimeInterval()
144 if 'spc' in self.plottypes or 'cspc' in self.plottypes:
145 self.xrange = (dataOut.getFreqRange(1)/1000. , dataOut.getAcfRange(1) , dataOut.getVelRange(1))
143 self.__heights.append(dataOut.heightList)
146 self.__heights.append(dataOut.heightList)
144 self.__all_heights.update(dataOut.heightList)
147 self.__all_heights.update(dataOut.heightList)
145 self.__times.append(tm)
148 self.__times.append(tm)
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now