@@ -4,4 +4,4 Created on Feb 7, 2012 | |||
|
4 | 4 | @author $Author$ |
|
5 | 5 | @version $Id$ |
|
6 | 6 | ''' |
|
7 |
__version__ = |
|
|
7 | __version__ = '2.3' |
@@ -145,6 +145,7 class PlotData(Operation, Process): | |||
|
145 | 145 | ax.figure.add_axes(nax) |
|
146 | 146 | return nax |
|
147 | 147 | |
|
148 | self.setup() | |
|
148 | 149 | |
|
149 | 150 | def setup(self): |
|
150 | 151 | ''' |
@@ -335,7 +336,6 class PlotData(Operation, Process): | |||
|
335 | 336 | if self.data: |
|
336 | 337 | self.__plot() |
|
337 | 338 | |
|
338 | ||
|
339 | 339 | class PlotSpectraData(PlotData): |
|
340 | 340 | ''' |
|
341 | 341 | Plot for Spectra data |
@@ -727,6 +727,7 class PlotParamData(PlotRTIData): | |||
|
727 | 727 | self.nplots = self.nrows |
|
728 | 728 | if self.showSNR: |
|
729 | 729 | self.nrows += 1 |
|
730 | self.nplots += 1 | |
|
730 | 731 | |
|
731 | 732 | self.ylabel = 'Height [Km]' |
|
732 | 733 | self.titles = self.data.parameters \ |
@@ -1115,26 +1115,24 class Parameters1Plot(Figure): | |||
|
1115 | 1115 | |
|
1116 | 1116 | x = dataOut.getTimeRange1(dataOut.paramInterval) |
|
1117 | 1117 | y = dataOut.heightList |
|
1118 | z = data_param[channelIndexList,parameterIndex,:].copy() | |
|
1119 | 1118 | |
|
1120 | zRange = dataOut.abscissaList | |
|
1121 | # nChannels = z.shape[0] #Number of wind dimensions estimated | |
|
1122 | # thisDatetime = dataOut.datatime | |
|
1119 | if dataOut.data_param.ndim == 3: | |
|
1120 | z = dataOut.data_param[channelIndexList,parameterIndex,:] | |
|
1121 | else: | |
|
1122 | z = dataOut.data_param[channelIndexList,:] | |
|
1123 | 1123 | |
|
1124 | 1124 | if dataOut.data_SNR is not None: |
|
1125 |
|
|
|
1126 | SNRdB = 10*numpy.log10(SNRarray) | |
|
1127 | # SNRavgdB = 10*numpy.log10(SNRavg) | |
|
1128 | ind = numpy.where(SNRdB < 10**(SNRthresh/10)) | |
|
1129 | z[ind] = numpy.nan | |
|
1125 | if dataOut.data_SNR.ndim == 2: | |
|
1126 | SNRavg = numpy.average(dataOut.data_SNR, axis=0) | |
|
1127 | else: | |
|
1128 | SNRavg = dataOut.data_SNR | |
|
1129 | SNRdB = 10*numpy.log10(SNRavg) | |
|
1130 | 1130 | |
|
1131 | 1131 | thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0]) |
|
1132 | 1132 | title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y")) |
|
1133 | 1133 | xlabel = "" |
|
1134 | 1134 | ylabel = "Range (Km)" |
|
1135 | 1135 | |
|
1136 | if (SNR and not onlySNR): nplots = 2*nplots | |
|
1137 | ||
|
1138 | 1136 | if onlyPositive: |
|
1139 | 1137 | colormap = "jet" |
|
1140 | 1138 | zmin = 0 |
@@ -1152,8 +1150,8 class Parameters1Plot(Figure): | |||
|
1152 | 1150 | |
|
1153 | 1151 | if ymin == None: ymin = numpy.nanmin(y) |
|
1154 | 1152 | if ymax == None: ymax = numpy.nanmax(y) |
|
1155 |
if zmin == None: zmin = numpy.nanmin(z |
|
|
1156 |
if zmax == None: zmax = numpy.nanmax(z |
|
|
1153 | if zmin == None: zmin = numpy.nanmin(z) | |
|
1154 | if zmax == None: zmax = numpy.nanmax(z) | |
|
1157 | 1155 | |
|
1158 | 1156 | if SNR: |
|
1159 | 1157 | if SNRmin == None: SNRmin = numpy.nanmin(SNRdB) |
@@ -1210,8 +1208,7 class Parameters1Plot(Figure): | |||
|
1210 | 1208 |
|
|
1211 | 1209 | |
|
1212 | 1210 |
|
|
1213 | ||
|
1214 | z1 = SNRdB[i,:].reshape((1,-1)) | |
|
1211 | z1 = SNRdB.reshape((1,-1)) | |
|
1215 | 1212 |
|
|
1216 | 1213 |
|
|
1217 | 1214 |
|
@@ -4,7 +4,7 import sys | |||
|
4 | 4 | import matplotlib |
|
5 | 5 | |
|
6 | 6 | if 'linux' in sys.platform: |
|
7 | matplotlib.use("TKAgg") | |
|
7 | matplotlib.use("GTK3Agg") | |
|
8 | 8 | |
|
9 | 9 | if 'darwin' in sys.platform: |
|
10 | 10 | matplotlib.use('TKAgg') |
@@ -18,4 +18,3 from jroIO_madrigal import * | |||
|
18 | 18 | from bltrIO_param import * |
|
19 | 19 | from jroIO_bltr import * |
|
20 | 20 | from jroIO_mira35c import * |
|
21 |
@@ -1269,7 +1269,11 class JRODataReader(JRODataIO): | |||
|
1269 | 1269 | cursor=None, |
|
1270 | 1270 | warnings=True, |
|
1271 | 1271 | verbose=True, |
|
1272 |
server=None |
|
|
1272 | server=None, | |
|
1273 | format=None, | |
|
1274 | oneDDict=None, | |
|
1275 | twoDDict=None, | |
|
1276 | ind2DList=None): | |
|
1273 | 1277 | if server is not None: |
|
1274 | 1278 | if 'tcp://' in server: |
|
1275 | 1279 | address = server |
@@ -1457,7 +1461,11 class JRODataReader(JRODataIO): | |||
|
1457 | 1461 | cursor=None, |
|
1458 | 1462 | warnings=True, |
|
1459 | 1463 | server=None, |
|
1460 |
verbose=True, |
|
|
1464 | verbose=True, | |
|
1465 | format=None, | |
|
1466 | oneDDict=None, | |
|
1467 | twoDDict=None, | |
|
1468 | ind2DList=None, **kwargs): | |
|
1461 | 1469 | |
|
1462 | 1470 | if not(self.isConfig): |
|
1463 | 1471 | self.setup(path=path, |
@@ -1480,7 +1488,11 class JRODataReader(JRODataIO): | |||
|
1480 | 1488 | cursor=cursor, |
|
1481 | 1489 | warnings=warnings, |
|
1482 | 1490 | server=server, |
|
1483 |
verbose=verbose |
|
|
1491 | verbose=verbose, | |
|
1492 | format=format, | |
|
1493 | oneDDict=oneDDict, | |
|
1494 | twoDDict=twoDDict, | |
|
1495 | ind2DList=ind2DList) | |
|
1484 | 1496 | self.isConfig = True |
|
1485 | 1497 | if server is None: |
|
1486 | 1498 | self.getData() |
@@ -8,9 +8,11 import os | |||
|
8 | 8 | import sys |
|
9 | 9 | import time |
|
10 | 10 | import json |
|
11 | import glob | |
|
11 | 12 | import datetime |
|
12 | 13 | |
|
13 | 14 | import numpy |
|
15 | import h5py | |
|
14 | 16 | |
|
15 | 17 | try: |
|
16 | 18 | import madrigal |
@@ -18,10 +20,12 try: | |||
|
18 | 20 | except: |
|
19 | 21 | print 'You should install "madrigal library" module if you want to read/write Madrigal data' |
|
20 | 22 | |
|
21 |
from schainpy.model. |
|
|
23 | from schainpy.model.io.jroIO_base import JRODataReader | |
|
24 | from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | |
|
22 | 25 | from schainpy.model.data.jrodata import Parameters |
|
26 | from schainpy.utils import log | |
|
27 | ||
|
23 | 28 | |
|
24 | MISSING = -32767 | |
|
25 | 29 | DEF_CATALOG = { |
|
26 | 30 | 'principleInvestigator': 'Marco Milla', |
|
27 | 31 | 'expPurpose': None, |
@@ -47,44 +51,377 MNEMONICS = { | |||
|
47 | 51 | 1002: 'obr', |
|
48 | 52 | } |
|
49 | 53 | |
|
54 | UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone) | |
|
55 | ||
|
50 | 56 | def load_json(obj): |
|
51 | 57 | ''' |
|
52 | 58 | Parse json as string instead of unicode |
|
53 | 59 | ''' |
|
54 | 60 | |
|
55 | 61 | if isinstance(obj, str): |
|
56 |
|
|
|
62 | iterable = json.loads(obj) | |
|
57 | 63 | |
|
64 | if isinstance(iterable, dict): | |
|
58 | 65 | return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v |
|
59 |
for k, v in |
|
|
66 | for k, v in iterable.items()} | |
|
67 | elif isinstance(iterable, (list, tuple)): | |
|
68 | return [str(v) if isinstance(v, unicode) else v for v in iterable] | |
|
69 | ||
|
70 | return iterable | |
|
71 | ||
|
72 | ||
|
73 | class MADReader(JRODataReader, ProcessingUnit): | |
|
74 | ||
|
75 | def __init__(self, **kwargs): | |
|
76 | ||
|
77 | ProcessingUnit.__init__(self, **kwargs) | |
|
78 | ||
|
79 | self.dataOut = Parameters() | |
|
80 | self.counter_records = 0 | |
|
81 | self.nrecords = None | |
|
82 | self.flagNoMoreFiles = 0 | |
|
83 | self.isConfig = False | |
|
84 | self.filename = None | |
|
85 | self.intervals = set() | |
|
86 | ||
|
87 | def setup(self, | |
|
88 | path=None, | |
|
89 | startDate=None, | |
|
90 | endDate=None, | |
|
91 | format=None, | |
|
92 | startTime=datetime.time(0, 0, 0), | |
|
93 | endTime=datetime.time(23, 59, 59), | |
|
94 | **kwargs): | |
|
95 | ||
|
96 | self.started = True | |
|
97 | self.path = path | |
|
98 | self.startDate = startDate | |
|
99 | self.endDate = endDate | |
|
100 | self.startTime = startTime | |
|
101 | self.endTime = endTime | |
|
102 | self.datatime = datetime.datetime(1900,1,1) | |
|
103 | self.oneDDict = load_json(kwargs.get('oneDDict', | |
|
104 | "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}")) | |
|
105 | self.twoDDict = load_json(kwargs.get('twoDDict', | |
|
106 | "{\"GDALT\": \"heightList\"}")) | |
|
107 | self.ind2DList = load_json(kwargs.get('ind2DList', | |
|
108 | "[\"GDALT\"]")) | |
|
109 | if self.path is None: | |
|
110 | raise ValueError, 'The path is not valid' | |
|
111 | ||
|
112 | if format is None: | |
|
113 | raise ValueError, 'The format is not valid choose simple or hdf5' | |
|
114 | elif format.lower() in ('simple', 'txt'): | |
|
115 | self.ext = '.txt' | |
|
116 | elif format.lower() in ('cedar',): | |
|
117 | self.ext = '.001' | |
|
118 | else: | |
|
119 | self.ext = '.hdf5' | |
|
120 | ||
|
121 | self.search_files(self.path) | |
|
122 | self.fileId = 0 | |
|
123 | ||
|
124 | if not self.fileList: | |
|
125 | raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path) | |
|
126 | ||
|
127 | self.setNextFile() | |
|
128 | ||
|
129 | def search_files(self, path): | |
|
130 | ''' | |
|
131 | Searching for madrigal files in path | |
|
132 | Creating a list of files to procces included in [startDate,endDate] | |
|
133 | ||
|
134 | Input: | |
|
135 | path - Path to find files | |
|
136 | ''' | |
|
137 | ||
|
138 | print 'Searching files {} in {} '.format(self.ext, path) | |
|
139 | foldercounter = 0 | |
|
140 | fileList0 = glob.glob1(path, '*{}'.format(self.ext)) | |
|
141 | fileList0.sort() | |
|
142 | ||
|
143 | self.fileList = [] | |
|
144 | self.dateFileList = [] | |
|
145 | ||
|
146 | startDate = self.startDate - datetime.timedelta(1) | |
|
147 | endDate = self.endDate + datetime.timedelta(1) | |
|
148 | ||
|
149 | for thisFile in fileList0: | |
|
150 | year = thisFile[3:7] | |
|
151 | if not year.isdigit(): | |
|
152 | continue | |
|
153 | ||
|
154 | month = thisFile[7:9] | |
|
155 | if not month.isdigit(): | |
|
156 | continue | |
|
157 | ||
|
158 | day = thisFile[9:11] | |
|
159 | if not day.isdigit(): | |
|
160 | continue | |
|
161 | ||
|
162 | year, month, day = int(year), int(month), int(day) | |
|
163 | dateFile = datetime.date(year, month, day) | |
|
164 | ||
|
165 | if (startDate > dateFile) or (endDate < dateFile): | |
|
166 | continue | |
|
167 | ||
|
168 | self.fileList.append(thisFile) | |
|
169 | self.dateFileList.append(dateFile) | |
|
170 | ||
|
171 | return | |
|
172 | ||
|
173 | def parseHeader(self): | |
|
174 | ''' | |
|
175 | ''' | |
|
176 | ||
|
177 | self.output = {} | |
|
178 | self.version = '2' | |
|
179 | s_parameters = None | |
|
180 | if self.ext == '.txt': | |
|
181 | self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s] | |
|
182 | elif self.ext == '.hdf5': | |
|
183 | metadata = self.fp['Metadata'] | |
|
184 | data = self.fp['Data']['Array Layout'] | |
|
185 | if 'Independent Spatial Parameters' in metadata: | |
|
186 | s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']] | |
|
187 | self.version = '3' | |
|
188 | one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']] | |
|
189 | one_d = [1 for s in one] | |
|
190 | two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']] | |
|
191 | two_d = [2 for s in two] | |
|
192 | self.parameters = one + two | |
|
193 | self.parameters_d = one_d + two_d | |
|
194 | ||
|
195 | log.success('Parameters found: {}'.format(','.join(self.parameters)), | |
|
196 | 'MADReader') | |
|
197 | if s_parameters: | |
|
198 | log.success('Spatial parameters: {}'.format(','.join(s_parameters)), | |
|
199 | 'MADReader') | |
|
200 | ||
|
201 | for param in self.oneDDict.keys(): | |
|
202 | if param.lower() not in self.parameters: | |
|
203 | print('\x1b[33m[Warning]\x1b[0m Parameter \x1b[1;32m{}\x1b[0m not found will be ignored'.format( | |
|
204 | param | |
|
205 | )) | |
|
206 | self.oneDDict.pop(param, None) | |
|
207 | ||
|
208 | for param, value in self.twoDDict.items(): | |
|
209 | if param.lower() not in self.parameters: | |
|
210 | print('\x1b[33m[Warning]\x1b[0m Parameter \x1b[1;32m{}\x1b[0m not found will be ignored'.format( | |
|
211 | param | |
|
212 | )) | |
|
213 | self.twoDDict.pop(param, None) | |
|
214 | continue | |
|
215 | if isinstance(value, list): | |
|
216 | if value[0] not in self.output: | |
|
217 | self.output[value[0]] = [] | |
|
218 | self.output[value[0]].append(None) | |
|
219 | ||
|
220 | def parseData(self): | |
|
221 | ''' | |
|
222 | ''' | |
|
223 | ||
|
224 | if self.ext == '.txt': | |
|
225 | self.data = numpy.genfromtxt(self.fp, missing_values=('missing')) | |
|
226 | self.nrecords = self.data.shape[0] | |
|
227 | self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())]) | |
|
228 | elif self.ext == '.hdf5': | |
|
229 | self.data = self.fp['Data']['Array Layout'] | |
|
230 | self.nrecords = len(self.data['timestamps'].value) | |
|
231 | self.ranges = self.data['range'].value | |
|
232 | ||
|
233 | def setNextFile(self): | |
|
234 | ''' | |
|
235 | ''' | |
|
236 | ||
|
237 | file_id = self.fileId | |
|
238 | ||
|
239 | if file_id == len(self.fileList): | |
|
240 | print '\nNo more files in the folder' | |
|
241 | print 'Total number of file(s) read : {}'.format(self.fileId) | |
|
242 | self.flagNoMoreFiles = 1 | |
|
243 | return 0 | |
|
244 | ||
|
245 | print('\x1b[32m[Info]\x1b[0m Opening: {}'.format( | |
|
246 | self.fileList[file_id] | |
|
247 | )) | |
|
248 | filename = os.path.join(self.path, self.fileList[file_id]) | |
|
249 | ||
|
250 | if self.filename is not None: | |
|
251 | self.fp.close() | |
|
252 | ||
|
253 | self.filename = filename | |
|
254 | self.filedate = self.dateFileList[file_id] | |
|
255 | ||
|
256 | if self.ext=='.hdf5': | |
|
257 | self.fp = h5py.File(self.filename, 'r') | |
|
258 | else: | |
|
259 | self.fp = open(self.filename, 'rb') | |
|
260 | ||
|
261 | self.parseHeader() | |
|
262 | self.parseData() | |
|
263 | self.sizeOfFile = os.path.getsize(self.filename) | |
|
264 | self.counter_records = 0 | |
|
265 | self.flagIsNewFile = 0 | |
|
266 | self.fileId += 1 | |
|
267 | ||
|
268 | return 1 | |
|
269 | ||
|
270 | def readNextBlock(self): | |
|
271 | ||
|
272 | while True: | |
|
273 | ||
|
274 | if self.flagIsNewFile: | |
|
275 | if not self.setNextFile(): | |
|
276 | return 0 | |
|
277 | ||
|
278 | self.readBlock() | |
|
279 | ||
|
280 | if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \ | |
|
281 | (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)): | |
|
282 | print "\x1b[32m[Reading]\x1b[0m Record No. %d/%d -> %s \x1b[33m[Skipping]\x1b[0m" %( | |
|
283 | self.counter_records, | |
|
284 | self.nrecords, | |
|
285 | self.datatime.ctime()) | |
|
286 | continue | |
|
287 | break | |
|
288 | ||
|
289 | print "\x1b[32m[Reading]\x1b[0m Record No. %d/%d -> %s" %( | |
|
290 | self.counter_records, | |
|
291 | self.nrecords, | |
|
292 | self.datatime.ctime()) | |
|
293 | ||
|
294 | return 1 | |
|
295 | ||
|
296 | def readBlock(self): | |
|
297 | ''' | |
|
298 | ''' | |
|
299 | dum = [] | |
|
300 | if self.ext == '.txt': | |
|
301 | dt = self.data[self.counter_records][:6].astype(int) | |
|
302 | self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]) | |
|
303 | while True: | |
|
304 | dt = self.data[self.counter_records][:6].astype(int) | |
|
305 | datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]) | |
|
306 | if datatime == self.datatime: | |
|
307 | dum.append(self.data[self.counter_records]) | |
|
308 | self.counter_records += 1 | |
|
309 | if self.counter_records == self.nrecords: | |
|
310 | self.flagIsNewFile = True | |
|
311 | break | |
|
312 | continue | |
|
313 | self.intervals.add((datatime-self.datatime).seconds) | |
|
314 | break | |
|
315 | elif self.ext == '.hdf5': | |
|
316 | datatime = datetime.datetime.utcfromtimestamp( | |
|
317 | self.data['timestamps'][self.counter_records]) | |
|
318 | nHeights = len(self.ranges) | |
|
319 | for n, param in enumerate(self.parameters): | |
|
320 | if self.parameters_d[n] == 1: | |
|
321 | dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records]) | |
|
322 | else: | |
|
323 | if self.version == '2': | |
|
324 | dum.append(self.data['2D Parameters'][param][self.counter_records]) | |
|
325 | else: | |
|
326 | tmp = self.data['2D Parameters'][param].value.T | |
|
327 | dum.append(tmp[self.counter_records]) | |
|
328 | self.intervals.add((datatime-self.datatime).seconds) | |
|
329 | self.datatime = datatime | |
|
330 | self.counter_records += 1 | |
|
331 | if self.counter_records == self.nrecords: | |
|
332 | self.flagIsNewFile = True | |
|
333 | ||
|
334 | self.buffer = numpy.array(dum) | |
|
335 | return | |
|
336 | ||
|
337 | def set_output(self): | |
|
338 | ''' | |
|
339 | Storing data from buffer to dataOut object | |
|
340 | ''' | |
|
341 | ||
|
342 | parameters = [None for __ in self.parameters] | |
|
343 | ||
|
344 | for param, attr in self.oneDDict.items(): | |
|
345 | x = self.parameters.index(param.lower()) | |
|
346 | setattr(self.dataOut, attr, self.buffer[0][x]) | |
|
347 | ||
|
348 | for param, value in self.twoDDict.items(): | |
|
349 | x = self.parameters.index(param.lower()) | |
|
350 | if self.ext == '.txt': | |
|
351 | y = self.parameters.index(self.ind2DList[0].lower()) | |
|
352 | ranges = self.buffer[:,y] | |
|
353 | if self.ranges.size == ranges.size: | |
|
354 | continue | |
|
355 | index = numpy.where(numpy.in1d(self.ranges, ranges))[0] | |
|
356 | dummy = numpy.zeros(self.ranges.shape) + numpy.nan | |
|
357 | dummy[index] = self.buffer[:,x] | |
|
358 | else: | |
|
359 | ||
|
360 | dummy = self.buffer[x] | |
|
361 | ||
|
362 | if isinstance(value, str): | |
|
363 | if value not in self.ind2DList: | |
|
364 | setattr(self.dataOut, value, dummy.reshape(1,-1)) | |
|
365 | elif isinstance(value, list): | |
|
366 | self.output[value[0]][value[1]] = dummy | |
|
367 | parameters[value[1]] = param | |
|
368 | ||
|
369 | for key, value in self.output.items(): | |
|
370 | setattr(self.dataOut, key, numpy.array(value)) | |
|
371 | ||
|
372 | self.dataOut.parameters = [s for s in parameters if s] | |
|
373 | self.dataOut.heightList = self.ranges | |
|
374 | self.dataOut.utctime = (self.datatime - UT1970).total_seconds() | |
|
375 | self.dataOut.utctimeInit = self.dataOut.utctime | |
|
376 | self.dataOut.paramInterval = min(self.intervals) | |
|
377 | self.dataOut.useLocalTime = False | |
|
378 | self.dataOut.flagNoData = False | |
|
379 | self.dataOut.started = self.started | |
|
380 | ||
|
381 | def getData(self): | |
|
382 | ''' | |
|
383 | Storing data from databuffer to dataOut object | |
|
384 | ''' | |
|
385 | if self.flagNoMoreFiles: | |
|
386 | self.dataOut.flagNoData = True | |
|
387 | print 'No file left to process' | |
|
388 | return 0 | |
|
389 | ||
|
390 | if not self.readNextBlock(): | |
|
391 | self.dataOut.flagNoData = True | |
|
392 | return 0 | |
|
393 | ||
|
394 | self.set_output() | |
|
395 | ||
|
396 | return 1 | |
|
60 | 397 | |
|
61 | 398 | |
|
62 | 399 | class MAD2Writer(Operation): |
|
63 | 400 | |
|
401 | missing = -32767 | |
|
402 | ext = '.dat' | |
|
403 | ||
|
64 | 404 | def __init__(self, **kwargs): |
|
65 | 405 | |
|
66 | 406 | Operation.__init__(self, **kwargs) |
|
67 | 407 | self.dataOut = Parameters() |
|
68 | 408 | self.path = None |
|
69 | 409 | self.dataOut = None |
|
70 | self.ext = '.dat' | |
|
71 | ||
|
72 | return | |
|
73 | 410 | |
|
74 |
def run(self, dataOut, path, oneD |
|
|
411 | def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}', metadata='{}', **kwargs): | |
|
75 | 412 | ''' |
|
76 | 413 | Inputs: |
|
77 | 414 | path - path where files will be created |
|
78 |
oneD |
|
|
415 | oneDDict - json of one-dimensional parameters in record where keys | |
|
79 | 416 | are Madrigal codes (integers or mnemonics) and values the corresponding |
|
80 | 417 | dataOut attribute e.g: { |
|
81 | 418 | 'gdlatr': 'lat', |
|
82 | 419 | 'gdlonr': 'lon', |
|
83 | 420 | 'gdlat2':'lat', |
|
84 | 421 | 'glon2':'lon'} |
|
85 | twoDParam - independent parameter to get the number of rows e.g: | |
|
86 | heighList | |
|
87 |
twoD |
|
|
422 | ind2DList - list of independent spatial two-dimensional parameters e.g: | |
|
423 | ['heighList'] | |
|
424 | twoDDict - json of two-dimensional parameters in record where keys | |
|
88 | 425 | are Madrigal codes (integers or mnemonics) and values the corresponding |
|
89 | 426 | dataOut attribute if multidimensional array specify as tupple |
|
90 | 427 | ('attr', pos) e.g: { |
@@ -97,13 +434,13 class MAD2Writer(Operation): | |||
|
97 | 434 | metadata - json of madrigal metadata (kinst, kindat, catalog and header) |
|
98 | 435 | ''' |
|
99 | 436 | if not self.isConfig: |
|
100 |
self.setup(dataOut, path, oneD |
|
|
437 | self.setup(dataOut, path, oneDDict, ind2DList, twoDDict, metadata, **kwargs) | |
|
101 | 438 | self.isConfig = True |
|
102 | 439 | |
|
103 | 440 | self.putData() |
|
104 | 441 | return |
|
105 | 442 | |
|
106 |
def setup(self, dataOut, path, oneD |
|
|
443 | def setup(self, dataOut, path, oneDDict, ind2DList, twoDDict, metadata, **kwargs): | |
|
107 | 444 | ''' |
|
108 | 445 | Configure Operation |
|
109 | 446 | ''' |
@@ -113,9 +450,9 class MAD2Writer(Operation): | |||
|
113 | 450 | self.path = path |
|
114 | 451 | self.blocks = kwargs.get('blocks', None) |
|
115 | 452 | self.counter = 0 |
|
116 |
self.oneD |
|
|
117 |
self.twoD |
|
|
118 | self.twoDParam = twoDParam | |
|
453 | self.oneDDict = load_json(oneDDict) | |
|
454 | self.twoDDict = load_json(twoDDict) | |
|
455 | self.ind2DList = load_json(ind2DList) | |
|
119 | 456 | meta = load_json(metadata) |
|
120 | 457 | self.kinst = meta.get('kinst') |
|
121 | 458 | self.kindat = meta.get('kindat') |
@@ -154,14 +491,14 class MAD2Writer(Operation): | |||
|
154 | 491 | |
|
155 | 492 | def writeBlock(self): |
|
156 | 493 | ''' |
|
157 |
Add data records to cedar file taking data from oneD |
|
|
494 | Add data records to cedar file taking data from oneDDict and twoDDict | |
|
158 | 495 | attributes. |
|
159 | 496 | Allowed parameters in: parcodes.tab |
|
160 | 497 | ''' |
|
161 | 498 | |
|
162 | 499 | startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime) |
|
163 | 500 | endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval) |
|
164 |
nrows = len(getattr(self.dataOut, self. |
|
|
501 | nrows = len(getattr(self.dataOut, self.ind2DList)) | |
|
165 | 502 | |
|
166 | 503 | rec = madrigal.cedar.MadrigalDataRecord( |
|
167 | 504 | self.kinst, |
@@ -180,20 +517,20 class MAD2Writer(Operation): | |||
|
180 | 517 | endTime.minute, |
|
181 | 518 | endTime.second, |
|
182 | 519 | endTime.microsecond/10000, |
|
183 |
self.oneD |
|
|
184 |
self.twoD |
|
|
520 | self.oneDDict.keys(), | |
|
521 | self.twoDDict.keys(), | |
|
185 | 522 | nrows |
|
186 | 523 | ) |
|
187 | 524 | |
|
188 | 525 | # Setting 1d values |
|
189 |
for key in self.oneD |
|
|
190 |
rec.set1D(key, getattr(self.dataOut, self.oneD |
|
|
526 | for key in self.oneDDict: | |
|
527 | rec.set1D(key, getattr(self.dataOut, self.oneDDict[key])) | |
|
191 | 528 | |
|
192 | 529 | # Setting 2d values |
|
193 | 530 | invalid = numpy.isnan(self.dataOut.data_output) |
|
194 |
self.dataOut.data_output[invalid] = |
|
|
531 | self.dataOut.data_output[invalid] = self.missing | |
|
195 | 532 | out = {} |
|
196 |
for key, value in self.twoD |
|
|
533 | for key, value in self.twoDDict.items(): | |
|
197 | 534 | if isinstance(value, str): |
|
198 | 535 | out[key] = getattr(self.dataOut, value) |
|
199 | 536 | elif isinstance(value, tuple): |
@@ -256,15 +256,6 class SpectraReader(JRODataReader, ProcessingUnit): | |||
|
256 | 256 | Exceptions: |
|
257 | 257 | Si un bloque leido no es un bloque valido |
|
258 | 258 | """ |
|
259 | print ' ======================================================== ' | |
|
260 | print ' ' | |
|
261 | print ' ' | |
|
262 | print self.processingHeaderObj.totalSpectra, 'TotalSpectra', type(self.processingHeaderObj.totalSpectra) | |
|
263 | print self.processingHeaderObj.spectraComb, 'SpectraComb', type(self.processingHeaderObj.spectraComb) | |
|
264 | print ' ' | |
|
265 | print ' ' | |
|
266 | print ' ======================================================== ' | |
|
267 | ||
|
268 | 259 | |
|
269 | 260 | blockOk_flag = False |
|
270 | 261 | fpointer = self.fp.tell() |
@@ -1401,6 +1401,9 class SpectralMoments(Operation): | |||
|
1401 | 1401 | |
|
1402 | 1402 | dataOut.data_param = data_param[:,1:,:] |
|
1403 | 1403 | dataOut.data_SNR = data_param[:,0] |
|
1404 | dataOut.data_DOP = data_param[:,1] | |
|
1405 | dataOut.data_MEAN = data_param[:,2] | |
|
1406 | dataOut.data_STD = data_param[:,3] | |
|
1404 | 1407 | return |
|
1405 | 1408 | |
|
1406 | 1409 | def __calculateMoments(self, oldspec, oldfreq, n0, |
@@ -113,6 +113,8 class Data(object): | |||
|
113 | 113 | self.__heights = [] |
|
114 | 114 | self.__all_heights = set() |
|
115 | 115 | for plot in self.plottypes: |
|
116 | if 'snr' in plot: | |
|
117 | plot = 'snr' | |
|
116 | 118 | self.data[plot] = {} |
|
117 | 119 | |
|
118 | 120 | def shape(self, key): |
@@ -138,8 +140,9 class Data(object): | |||
|
138 | 140 | self.parameters = getattr(dataOut, 'parameters', []) |
|
139 | 141 | self.pairs = dataOut.pairsList |
|
140 | 142 | self.channels = dataOut.channelList |
|
141 | self.xrange = (dataOut.getFreqRange(1)/1000. , dataOut.getAcfRange(1) , dataOut.getVelRange(1)) | |
|
142 | 143 | self.interval = dataOut.getTimeInterval() |
|
144 | if 'spc' in self.plottypes or 'cspc' in self.plottypes: | |
|
145 | self.xrange = (dataOut.getFreqRange(1)/1000. , dataOut.getAcfRange(1) , dataOut.getVelRange(1)) | |
|
143 | 146 | self.__heights.append(dataOut.heightList) |
|
144 | 147 | self.__all_heights.update(dataOut.heightList) |
|
145 | 148 | self.__times.append(tm) |
|
1 | NO CONTENT: file was removed |
General Comments 0
You need to be logged in to leave comments.
Login now