##// END OF EJS Templates
schain
RSS
Update ppi and rhi plot fix processing ppi+rhi
jespinoza -
r1562:dab94d864b0c
parent child
Show More
Side by Side Unified
Context file:
r1562:dab94d864b0c Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,723 +1,749
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """Base class to create plot operations
5 """Base class to create plot operations
6
6
7 """
7 """
8
8
9 import os
9 import os
10 import sys
10 import sys
11 import zmq
11 import zmq
12 import time
12 import time
13 import numpy
13 import numpy
14 import datetime
14 import datetime
15 from collections import deque
15 from collections import deque
16 from functools import wraps
16 from functools import wraps
17 from threading import Thread
17 from threading import Thread
18 import matplotlib,re
18 import matplotlib,re
19
19
20 if 'BACKEND' in os.environ:
20 if 'BACKEND' in os.environ:
21 matplotlib.use(os.environ['BACKEND'])
21 matplotlib.use(os.environ['BACKEND'])
22 elif 'linux' in sys.platform:
22 elif 'linux' in sys.platform:
23 matplotlib.use("TkAgg")
23 matplotlib.use("Agg")
24 elif 'darwin' in sys.platform:
24 elif 'darwin' in sys.platform:
25 matplotlib.use('MacOSX')
25 matplotlib.use('MacOSX')
26 else:
26 else:
27 from schainpy.utils import log
27 from schainpy.utils import log
28 log.warning('Using default Backend="Agg"', 'INFO')
28 log.warning('Using default Backend="Agg"', 'INFO')
29 matplotlib.use('Agg')
29 matplotlib.use('Agg')
30
30
31 import matplotlib.pyplot as plt
31 import matplotlib.pyplot as plt
32 from matplotlib.patches import Polygon
32 from matplotlib.patches import Polygon
33 from mpl_toolkits.axes_grid1 import make_axes_locatable
33 from mpl_toolkits.axes_grid1 import make_axes_locatable
34 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
34 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
35
35
36 from .plotting_codes import *
36 from .plotting_codes import *
37
37
38 from schainpy.model.data.jrodata import PlotterData
38 from schainpy.model.data.jrodata import PlotterData
39 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
39 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
40 from schainpy.utils import log
40 from schainpy.utils import log
41
41
42 for name, cb_table in sophy_cb_tables:
42 for name, cb_table in sophy_cb_tables:
43 ncmap = matplotlib.colors.ListedColormap(cb_table, name=name)
43 ncmap = matplotlib.colors.ListedColormap(cb_table, name=name)
44 matplotlib.pyplot.register_cmap(cmap=ncmap)
44 matplotlib.pyplot.register_cmap(cmap=ncmap)
45
45
46 EARTH_RADIUS = 6.3710e3
46 EARTH_RADIUS = 6.3710e3
47
47
48 def ll2xy(lat1, lon1, lat2, lon2):
48 def ll2xy(lat1, lon1, lat2, lon2):
49
49
50 p = 0.017453292519943295
50 p = 0.017453292519943295
51 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
51 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
52 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
52 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
53 r = 12742 * numpy.arcsin(numpy.sqrt(a))
53 r = 12742 * numpy.arcsin(numpy.sqrt(a))
54 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
54 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
55 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
55 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
56 theta = -theta + numpy.pi/2
56 theta = -theta + numpy.pi/2
57 return r*numpy.cos(theta), r*numpy.sin(theta)
57 return r*numpy.cos(theta), r*numpy.sin(theta)
58
58
59
59
60 def km2deg(km):
60 def km2deg(km):
61 '''
61 '''
62 Convert distance in km to degrees
62 Convert distance in km to degrees
63 '''
63 '''
64
64
65 return numpy.rad2deg(km/EARTH_RADIUS)
65 return numpy.rad2deg(km/EARTH_RADIUS)
66
66
67
67
68 def figpause(interval):
68 def figpause(interval):
69 backend = plt.rcParams['backend']
69 backend = plt.rcParams['backend']
70 if backend in matplotlib.rcsetup.interactive_bk:
70 if backend in matplotlib.rcsetup.interactive_bk:
71 figManager = matplotlib._pylab_helpers.Gcf.get_active()
71 figManager = matplotlib._pylab_helpers.Gcf.get_active()
72 if figManager is not None:
72 if figManager is not None:
73 canvas = figManager.canvas
73 canvas = figManager.canvas
74 if canvas.figure.stale:
74 if canvas.figure.stale:
75 canvas.draw()
75 canvas.draw()
76 try:
76 try:
77 canvas.start_event_loop(interval)
77 canvas.start_event_loop(interval)
78 except:
78 except:
79 pass
79 pass
80 return
80 return
81
81
82 def popup(message):
82 def popup(message):
83 '''
83 '''
84 '''
84 '''
85
85
86 fig = plt.figure(figsize=(12, 8), facecolor='r')
86 fig = plt.figure(figsize=(12, 8), facecolor='r')
87 text = '\n'.join([s.strip() for s in message.split(':')])
87 text = '\n'.join([s.strip() for s in message.split(':')])
88 fig.text(0.01, 0.5, text, ha='left', va='center',
88 fig.text(0.01, 0.5, text, ha='left', va='center',
89 size='20', weight='heavy', color='w')
89 size='20', weight='heavy', color='w')
90 fig.show()
90 fig.show()
91 figpause(1000)
91 figpause(1000)
92
92
93
93
94 class Throttle(object):
94 class Throttle(object):
95 '''
95 '''
96 Decorator that prevents a function from being called more than once every
96 Decorator that prevents a function from being called more than once every
97 time period.
97 time period.
98 To create a function that cannot be called more than once a minute, but
98 To create a function that cannot be called more than once a minute, but
99 will sleep until it can be called:
99 will sleep until it can be called:
100 @Throttle(minutes=1)
100 @Throttle(minutes=1)
101 def foo():
101 def foo():
102 pass
102 pass
103
103
104 for i in range(10):
104 for i in range(10):
105 foo()
105 foo()
106 print "This function has run %s times." % i
106 print "This function has run %s times." % i
107 '''
107 '''
108
108
109 def __init__(self, seconds=0, minutes=0, hours=0):
109 def __init__(self, seconds=0, minutes=0, hours=0):
110 self.throttle_period = datetime.timedelta(
110 self.throttle_period = datetime.timedelta(
111 seconds=seconds, minutes=minutes, hours=hours
111 seconds=seconds, minutes=minutes, hours=hours
112 )
112 )
113
113
114 self.time_of_last_call = datetime.datetime.min
114 self.time_of_last_call = datetime.datetime.min
115
115
116 def __call__(self, fn):
116 def __call__(self, fn):
117 @wraps(fn)
117 @wraps(fn)
118 def wrapper(*args, **kwargs):
118 def wrapper(*args, **kwargs):
119 coerce = kwargs.pop('coerce', None)
119 coerce = kwargs.pop('coerce', None)
120 if coerce:
120 if coerce:
121 self.time_of_last_call = datetime.datetime.now()
121 self.time_of_last_call = datetime.datetime.now()
122 return fn(*args, **kwargs)
122 return fn(*args, **kwargs)
123 else:
123 else:
124 now = datetime.datetime.now()
124 now = datetime.datetime.now()
125 time_since_last_call = now - self.time_of_last_call
125 time_since_last_call = now - self.time_of_last_call
126 time_left = self.throttle_period - time_since_last_call
126 time_left = self.throttle_period - time_since_last_call
127
127
128 if time_left > datetime.timedelta(seconds=0):
128 if time_left > datetime.timedelta(seconds=0):
129 return
129 return
130
130
131 self.time_of_last_call = datetime.datetime.now()
131 self.time_of_last_call = datetime.datetime.now()
132 return fn(*args, **kwargs)
132 return fn(*args, **kwargs)
133
133
134 return wrapper
134 return wrapper
135
135
136 def apply_throttle(value):
136 def apply_throttle(value):
137
137
138 @Throttle(seconds=value)
138 @Throttle(seconds=value)
139 def fnThrottled(fn):
139 def fnThrottled(fn):
140 fn()
140 fn()
141
141
142 return fnThrottled
142 return fnThrottled
143
143
144
144
145 @MPDecorator
145 @MPDecorator
146 class Plot(Operation):
146 class Plot(Operation):
147 """Base class for Schain plotting operations
147 """Base class for Schain plotting operations
148
148
149 This class should never be use directtly you must subclass a new operation,
149 This class should never be use directtly you must subclass a new operation,
150 children classes must be defined as follow:
150 children classes must be defined as follow:
151
151
152 ExamplePlot(Plot):
152 ExamplePlot(Plot):
153
153
154 CODE = 'code'
154 CODE = 'code'
155 colormap = 'jet'
155 colormap = 'jet'
156 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
156 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
157
157
158 def setup(self):
158 def setup(self):
159 pass
159 pass
160
160
161 def plot(self):
161 def plot(self):
162 pass
162 pass
163
163
164 """
164 """
165
165
166 CODE = 'Figure'
166 CODE = 'Figure'
167 colormap = 'jet'
167 colormap = 'jet'
168 bgcolor = 'white'
168 bgcolor = 'white'
169 buffering = True
169 buffering = True
170 __missing = 1E30
170 __missing = 1E30
171 projection = None
171
172
172 __attrs__ = ['show', 'save', 'ymin', 'ymax', 'zmin', 'zmax', 'title',
173 __attrs__ = ['show', 'save', 'ymin', 'ymax', 'zmin', 'zmax', 'title',
173 'showprofile']
174 'showprofile']
174
175
175 def __init__(self):
176 def __init__(self):
176
177
177 Operation.__init__(self)
178 Operation.__init__(self)
178 self.isConfig = False
179 self.isConfig = False
179 self.isPlotConfig = False
180 self.isPlotConfig = False
180 self.save_time = 0
181 self.save_time = 0
181 self.sender_time = 0
182 self.sender_time = 0
182 self.data = None
183 self.data = None
183 self.firsttime = True
184 self.firsttime = True
184 self.sender_queue = deque(maxlen=10)
185 self.sender_queue = deque(maxlen=10)
185 self.plots_adjust = {'left': 0.125, 'right': 0.9, 'bottom': 0.15, 'top': 0.9, 'wspace': 0.2, 'hspace': 0.2}
186 self.plots_adjust = {'left': 0.125, 'right': 0.9, 'bottom': 0.15, 'top': 0.9, 'wspace': 0.2, 'hspace': 0.2}
186
187
187 def __fmtTime(self, x, pos):
188 def __fmtTime(self, x, pos):
188 '''
189 '''
189 '''
190 '''
190
191
191 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
192 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
192
193
193 def __setup(self, **kwargs):
194 def __setup(self, **kwargs):
194 '''
195 '''
195 Initialize variables
196 Initialize variables
196 '''
197 '''
197
198
198 self.figures = []
199 self.figures = []
199 self.axes = []
200 self.axes = []
200 self.cb_axes = []
201 self.cb_axes = []
201 self.localtime = kwargs.pop('localtime', True)
202 self.localtime = kwargs.pop('localtime', True)
202 self.show = kwargs.get('show', True)
203 self.show = kwargs.get('show', True)
203 self.save = kwargs.get('save', False)
204 self.save = kwargs.get('save', False)
204 self.save_period = kwargs.get('save_period', 0)
205 self.save_period = kwargs.get('save_period', 0)
205 self.colormap = kwargs.get('colormap', self.colormap)
206 self.colormap = kwargs.get('colormap', self.colormap)
206 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
207 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
207 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
208 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
208 self.colormaps = kwargs.get('colormaps', None)
209 self.colormaps = kwargs.get('colormaps', None)
209 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
210 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
210 self.showprofile = kwargs.get('showprofile', False)
211 self.showprofile = kwargs.get('showprofile', False)
211 self.title = kwargs.get('wintitle', self.CODE.upper())
212 self.title = kwargs.get('wintitle', self.CODE.upper())
212 self.cb_label = kwargs.get('cb_label', None)
213 self.cb_label = kwargs.get('cb_label', None)
213 self.cb_labels = kwargs.get('cb_labels', None)
214 self.cb_labels = kwargs.get('cb_labels', None)
214 self.labels = kwargs.get('labels', None)
215 self.labels = kwargs.get('labels', None)
215 self.xaxis = kwargs.get('xaxis', 'frequency')
216 self.xaxis = kwargs.get('xaxis', 'frequency')
216 self.zmin = kwargs.get('zmin', None)
217 self.zmin = kwargs.get('zmin', None)
217 self.zmax = kwargs.get('zmax', None)
218 self.zmax = kwargs.get('zmax', None)
218 self.zlimits = kwargs.get('zlimits', None)
219 self.zlimits = kwargs.get('zlimits', None)
219 self.xmin = kwargs.get('xmin', None)
220 self.xmin = kwargs.get('xmin', None)
220 self.xmax = kwargs.get('xmax', None)
221 self.xmax = kwargs.get('xmax', None)
221 self.xrange = kwargs.get('xrange', 12)
222 self.xrange = kwargs.get('xrange', 12)
222 self.xscale = kwargs.get('xscale', None)
223 self.xscale = kwargs.get('xscale', None)
223 self.ymin = kwargs.get('ymin', None)
224 self.ymin = kwargs.get('ymin', None)
224 self.ymax = kwargs.get('ymax', None)
225 self.ymax = kwargs.get('ymax', None)
225 self.yscale = kwargs.get('yscale', None)
226 self.yscale = kwargs.get('yscale', None)
226 self.xlabel = kwargs.get('xlabel', None)
227 self.xlabel = kwargs.get('xlabel', None)
227 self.attr_time = kwargs.get('attr_time', 'utctime')
228 self.attr_time = kwargs.get('attr_time', 'utctime')
228 self.attr_data = kwargs.get('attr_data', 'data_param')
229 self.attr_data = kwargs.get('attr_data', 'data_param')
229 self.decimation = kwargs.get('decimation', None)
230 self.decimation = kwargs.get('decimation', None)
230 self.oneFigure = kwargs.get('oneFigure', True)
231 self.oneFigure = kwargs.get('oneFigure', True)
231 self.width = kwargs.get('width', None)
232 self.width = kwargs.get('width', None)
232 self.height = kwargs.get('height', None)
233 self.height = kwargs.get('height', None)
233 self.colorbar = kwargs.get('colorbar', True)
234 self.colorbar = kwargs.get('colorbar', True)
234 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
235 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
235 self.channels = kwargs.get('channels', None)
236 self.channels = kwargs.get('channels', None)
236 self.titles = kwargs.get('titles', [])
237 self.titles = kwargs.get('titles', [])
237 self.polar = False
238 self.polar = False
238 self.type = kwargs.get('type', 'iq')
239 self.type = kwargs.get('type', 'iq')
239 self.grid = kwargs.get('grid', False)
240 self.grid = kwargs.get('grid', False)
240 self.pause = kwargs.get('pause', False)
241 self.pause = kwargs.get('pause', False)
241 self.save_code = kwargs.get('save_code', self.CODE)
242 self.save_code = kwargs.get('save_code', self.CODE)
242 self.throttle = kwargs.get('throttle', 0)
243 self.throttle = kwargs.get('throttle', 0)
243 self.exp_code = kwargs.get('exp_code', None)
244 self.exp_code = kwargs.get('exp_code', None)
244 self.server = kwargs.get('server', False)
245 self.server = kwargs.get('server', False)
245 self.sender_period = kwargs.get('sender_period', 60)
246 self.sender_period = kwargs.get('sender_period', 60)
246 self.tag = kwargs.get('tag', '')
247 self.tag = kwargs.get('tag', '')
247 self.height_index = kwargs.get('height_index', None)
248 self.height_index = kwargs.get('height_index', None)
248 self.__throttle_plot = apply_throttle(self.throttle)
249 self.__throttle_plot = apply_throttle(self.throttle)
249 code = self.attr_data if self.attr_data else self.CODE
250 code = self.attr_data if self.attr_data else self.CODE
250 self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
251 self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
251 self.ang_min = kwargs.get('ang_min', None)
252 self.ang_min = kwargs.get('ang_min', None)
252 self.ang_max = kwargs.get('ang_max', None)
253 self.ang_max = kwargs.get('ang_max', None)
253 self.mode = kwargs.get('mode', None)
254 self.mode = kwargs.get('mode', None)
254 self.mask = kwargs.get('mask', False)
255 self.mask = kwargs.get('mask', False)
255
256 self.shapes = kwargs.get('shapes', './')
256
257
257 if self.server:
258 if self.server:
258 if not self.server.startswith('tcp://'):
259 if not self.server.startswith('tcp://'):
259 self.server = 'tcp://{}'.format(self.server)
260 self.server = 'tcp://{}'.format(self.server)
260 log.success(
261 log.success(
261 'Sending to server: {}'.format(self.server),
262 'Sending to server: {}'.format(self.server),
262 self.name
263 self.name
263 )
264 )
264
265
265 if isinstance(self.attr_data, str):
266 if isinstance(self.attr_data, str):
266 self.attr_data = [self.attr_data]
267 self.attr_data = [self.attr_data]
267
268
268 def __setup_plot(self):
269 def __setup_plot(self):
269 '''
270 '''
270 Common setup for all figures, here figures and axes are created
271 Common setup for all figures, here figures and axes are created
271 '''
272 '''
272
273
273 self.setup()
274 self.setup()
274
275
275 self.time_label = 'LT' if self.localtime else 'UTC'
276 self.time_label = 'LT' if self.localtime else 'UTC'
276
277
277 if self.width is None:
278 if self.width is None:
278 self.width = 8
279 self.width = 8
279
280
280 self.figures = []
281 self.figures = {'PPI':[], 'RHI':[]}
281 self.axes = []
282 self.axes = {'PPI':[], 'RHI':[]}
282 self.cb_axes = []
283 self.cb_axes = []
283 self.pf_axes = []
284 self.pf_axes = []
284 self.cmaps = []
285 self.cmaps = []
285
286
286 size = '15%' if self.ncols == 1 else '30%'
287 size = '15%' if self.ncols == 1 else '30%'
287 pad = '4%' if self.ncols == 1 else '8%'
288 pad = '4%' if self.ncols == 1 else '8%'
288
289
289 if self.oneFigure:
290 if self.oneFigure:
290 if self.height is None:
291 if self.height is None:
291 self.height = 1.4 * self.nrows + 1
292 self.height = 1.4 * self.nrows + 1
292 fig = plt.figure(figsize=(self.width, self.height),
293 fig_p = plt.figure(figsize=(self.width, self.height),
293 edgecolor='k',
294 edgecolor='k',
294 facecolor='w')
295 facecolor='w')
295 self.figures.append(fig)
296 fig_r = plt.figure(figsize=(self.width, self.height),
296 for n in range(self.nplots):
297 edgecolor='k',
297 ax = fig.add_subplot(self.nrows, self.ncols,
298 facecolor='w')
298 n + 1, polar=self.polar)
299 self.figures['PPI'].append(fig_p)
299 ax.tick_params(labelsize=8)
300 self.figures['RHI'].append(fig_r)
300 ax.firsttime = True
301 for n in range(self.nplots):
301 ax.index = 0
302 ax_p = fig_p.add_subplot(self.nrows, self.ncols, n+1, polar=self.polar, projection=self.projection)
302 ax.press = None
303 ax_r = fig_r.add_subplot(self.nrows, self.ncols, n+1, polar=self.polar)
303 self.axes.append(ax)
304 ax_p.tick_params(labelsize=8)
305 ax_p.firsttime = True
306 ax_p.index = 0
307 ax_p.press = None
308 ax_r.tick_params(labelsize=8)
309 ax_r.firsttime = True
310 ax_r.index = 0
311 ax_r.press = None
312
313 self.axes['PPI'].append(ax_p)
314 self.axes['RHI'].append(ax_r)
315
304 if self.showprofile:
316 if self.showprofile:
305 cax = self.__add_axes(ax, size=size, pad=pad)
317 cax = self.__add_axes(ax, size=size, pad=pad)
306 cax.tick_params(labelsize=8)
318 cax.tick_params(labelsize=8)
307 self.pf_axes.append(cax)
319 self.pf_axes.append(cax)
308 else:
320 else:
309 if self.height is None:
321 if self.height is None:
310 self.height = 3
322 self.height = 3
311 for n in range(self.nplots):
323 for n in range(self.nplots):
312 fig = plt.figure(figsize=(self.width, self.height),
324 fig = plt.figure(figsize=(self.width, self.height),
313 edgecolor='k',
325 edgecolor='k',
314 facecolor='w')
326 facecolor='w')
315 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
327 ax_p = fig.add_subplot(1, 1, 1, polar=self.polar, projection=self.projection)
316 ax.tick_params(labelsize=8)
328 ax_r = fig.add_subplot(1, 1, 1, polar=self.polar)
317 ax.firsttime = True
329 ax_p.tick_params(labelsize=8)
318 ax.index = 0
330 ax_p.firsttime = True
319 ax.press = None
331 ax_p.index = 0
332 ax_p.press = None
333 ax_r.tick_params(labelsize=8)
334 ax_r.firsttime = True
335 ax_r.index = 0
336 ax_r.press = None
320 self.figures.append(fig)
337 self.figures.append(fig)
321 self.axes.append(ax)
338 self.axes['PPI'].append(ax_p)
339 self.axes['RHI'].append(ax_r)
322 if self.showprofile:
340 if self.showprofile:
323 cax = self.__add_axes(ax, size=size, pad=pad)
341 cax = self.__add_axes(ax, size=size, pad=pad)
324 cax.tick_params(labelsize=8)
342 cax.tick_params(labelsize=8)
325 self.pf_axes.append(cax)
343 self.pf_axes.append(cax)
326
344
327 for n in range(self.nrows):
345 for n in range(self.nrows):
328 if self.colormaps is not None:
346 if self.colormaps is not None:
329 cmap = plt.get_cmap(self.colormaps[n])
347 cmap = plt.get_cmap(self.colormaps[n])
330 else:
348 else:
331 cmap = plt.get_cmap(self.colormap)
349 cmap = plt.get_cmap(self.colormap)
332 cmap.set_bad(self.bgcolor, 1.)
350 cmap.set_bad(self.bgcolor, 1.)
333 self.cmaps.append(cmap)
351 self.cmaps.append(cmap)
334
352
335 def __add_axes(self, ax, size='30%', pad='8%'):
353 def __add_axes(self, ax, size='30%', pad='8%'):
336 '''
354 '''
337 Add new axes to the given figure
355 Add new axes to the given figure
338 '''
356 '''
339 divider = make_axes_locatable(ax)
357 divider = make_axes_locatable(ax)
340 nax = divider.new_horizontal(size=size, pad=pad)
358 nax = divider.new_horizontal(size=size, pad=pad)
341 ax.figure.add_axes(nax)
359 ax.figure.add_axes(nax)
342 return nax
360 return nax
343
361
344 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
362 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
345 '''
363 '''
346 Create a masked array for missing data
364 Create a masked array for missing data
347 '''
365 '''
348 if x_buffer.shape[0] < 2:
366 if x_buffer.shape[0] < 2:
349 return x_buffer, y_buffer, z_buffer
367 return x_buffer, y_buffer, z_buffer
350
368
351 deltas = x_buffer[1:] - x_buffer[0:-1]
369 deltas = x_buffer[1:] - x_buffer[0:-1]
352 x_median = numpy.median(deltas)
370 x_median = numpy.median(deltas)
353
371
354 index = numpy.where(deltas > 5 * x_median)
372 index = numpy.where(deltas > 5 * x_median)
355
373
356 if len(index[0]) != 0:
374 if len(index[0]) != 0:
357 z_buffer[::, index[0], ::] = self.__missing
375 z_buffer[::, index[0], ::] = self.__missing
358 z_buffer = numpy.ma.masked_inside(z_buffer,
376 z_buffer = numpy.ma.masked_inside(z_buffer,
359 0.99 * self.__missing,
377 0.99 * self.__missing,
360 1.01 * self.__missing)
378 1.01 * self.__missing)
361
379
362 return x_buffer, y_buffer, z_buffer
380 return x_buffer, y_buffer, z_buffer
363
381
364 def decimate(self):
382 def decimate(self):
365
383
366 # dx = int(len(self.x)/self.__MAXNUMX) + 1
384 # dx = int(len(self.x)/self.__MAXNUMX) + 1
367 dy = int(len(self.y) / self.decimation) + 1
385 dy = int(len(self.y) / self.decimation) + 1
368
386
369 # x = self.x[::dx]
387 # x = self.x[::dx]
370 x = self.x
388 x = self.x
371 y = self.y[::dy]
389 y = self.y[::dy]
372 z = self.z[::, ::, ::dy]
390 z = self.z[::, ::, ::dy]
373
391
374 return x, y, z
392 return x, y, z
375
393
376 def format(self):
394 def format(self):
377 '''
395 '''
378 Set min and max values, labels, ticks and titles
396 Set min and max values, labels, ticks and titles
379 '''
397 '''
380
398
381 for n, ax in enumerate(self.axes):
399 for n, ax in enumerate(self.axes[self.mode]):
382 if ax.firsttime:
400 if ax.firsttime:
383 if self.xaxis != 'time':
401 if self.xaxis != 'time':
384 xmin = self.xmin
402 xmin = self.xmin
385 xmax = self.xmax
403 xmax = self.xmax
386 else:
404 else:
387 xmin = self.tmin
405 xmin = self.tmin
388 xmax = self.tmin + self.xrange*60*60
406 xmax = self.tmin + self.xrange*60*60
389 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
407 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
390 ax.xaxis.set_major_locator(LinearLocator(9))
408 ax.xaxis.set_major_locator(LinearLocator(9))
391 ymin = self.ymin if self.ymin is not None else numpy.nanmin(self.y[numpy.isfinite(self.y)])
409 ymin = self.ymin if self.ymin is not None else numpy.nanmin(self.y[numpy.isfinite(self.y)])
392 ymax = self.ymax if self.ymax is not None else numpy.nanmax(self.y[numpy.isfinite(self.y)])
410 ymax = self.ymax if self.ymax is not None else numpy.nanmax(self.y[numpy.isfinite(self.y)])
411
393 ax.set_facecolor(self.bgcolor)
412 ax.set_facecolor(self.bgcolor)
413
394 if self.xscale:
414 if self.xscale:
395 ax.xaxis.set_major_formatter(FuncFormatter(
415 ax.xaxis.set_major_formatter(FuncFormatter(
396 lambda x, pos: '{0:g}'.format(x*self.xscale)))
416 lambda x, pos: '{0:g}'.format(x*self.xscale)))
397 if self.yscale:
417 if self.yscale:
398 ax.yaxis.set_major_formatter(FuncFormatter(
418 ax.yaxis.set_major_formatter(FuncFormatter(
399 lambda x, pos: '{0:g}'.format(x*self.yscale)))
419 lambda x, pos: '{0:g}'.format(x*self.yscale)))
400 if self.xlabel is not None:
420 if self.xlabel is not None:
401 ax.set_xlabel(self.xlabel)
421 ax.set_xlabel(self.xlabel)
402 if self.ylabel is not None:
422 if self.ylabel is not None:
403 ax.set_ylabel(self.ylabel)
423 ax.set_ylabel(self.ylabel)
404 if self.showprofile:
424 if self.showprofile:
405 self.pf_axes[n].set_ylim(ymin, ymax)
425 self.pf_axes[n].set_ylim(ymin, ymax)
406 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
426 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
407 self.pf_axes[n].set_xlabel('dB')
427 self.pf_axes[n].set_xlabel('dB')
408 self.pf_axes[n].grid(b=True, axis='x')
428 self.pf_axes[n].grid(b=True, axis='x')
409 [tick.set_visible(False)
429 [tick.set_visible(False)
410 for tick in self.pf_axes[n].get_yticklabels()]
430 for tick in self.pf_axes[n].get_yticklabels()]
411 if self.colorbar:
431 if self.colorbar:
412 ax.cbar = plt.colorbar(
432 ax.cbar = plt.colorbar(
413 ax.plt, ax=ax, fraction=0.05, pad=0.06, aspect=10)
433 ax.plt, ax=ax, fraction=0.05, pad=0.06, aspect=10)
414 ax.cbar.ax.tick_params(labelsize=8)
434 ax.cbar.ax.tick_params(labelsize=8)
415 ax.cbar.ax.press = None
435 ax.cbar.ax.press = None
416 if self.cb_label:
436 if self.cb_label:
417 ax.cbar.set_label(self.cb_label, size=8)
437 ax.cbar.set_label(self.cb_label, size=8)
418 elif self.cb_labels:
438 elif self.cb_labels:
419 ax.cbar.set_label(self.cb_labels[n], size=8)
439 ax.cbar.set_label(self.cb_labels[n], size=8)
420 else:
440 else:
421 ax.cbar = None
441 ax.cbar = None
422 ax.set_xlim(xmin, xmax)
442 if self.mode == 'RHI':
423 ax.set_ylim(ymin, ymax)
443 ax.set_xlim(xmin, xmax)
444 ax.set_ylim(ymin, ymax)
424 ax.firsttime = False
445 ax.firsttime = False
425 if self.grid:
446 if self.grid:
426 ax.grid(True)
447 ax.grid(True)
427 if not self.polar:
448 if not self.polar:
428 ax.set_title('{} {} {}'.format(
449 ax.set_title('{} {} {}'.format(
429 self.titles[n],
450 self.titles[n],
430 self.getDateTime(self.data.max_time).strftime(
451 self.getDateTime(self.data.max_time).strftime(
431 '%Y-%m-%d %H:%M:%S'),
452 '%Y-%m-%d %H:%M:%S'),
432 self.time_label),
453 self.time_label),
433 size=8)
454 size=8)
434 else:
455 else:
435 #ax.set_title('{}'.format(self.titles[n]), size=8)
456 #ax.set_title('{}'.format(self.titles[n]), size=8)
436 ax.set_title('{} {} {}'.format(
457 ax.set_title('{} {} {}'.format(
437 self.titles[n],
458 self.titles[n],
438 self.getDateTime(self.data.max_time).strftime(
459 self.getDateTime(self.data.max_time).strftime(
439 '%Y-%m-%d %H:%M:%S'),
460 '%Y-%m-%d %H:%M:%S'),
440 self.time_label),
461 self.time_label),
441 size=8)
462 size=8)
442 ax.set_ylim(0, self.ymax)
463 ax.set_ylim(0, self.ymax)
443 if self.mode == 'PPI':
464 if self.mode == 'PPI':
444 ax.set_yticks(ax.get_yticks(), labels=ax.get_yticks(), color='white')
465 ax.set_yticks(ax.get_yticks(), labels=ax.get_yticks(), color='white')
445 ax.yaxis.labelpad = 28
466 ax.yaxis.labelpad = 28
446 elif self.mode == 'RHI':
467 elif self.mode == 'RHI':
447 ax.xaxis.labelpad = 16
468 ax.xaxis.labelpad = 16
448
469
449 if self.firsttime:
470 if self.firsttime:
450 for n, fig in enumerate(self.figures):
471 for fig in self.figures['PPI'] + self.figures['RHI']:
451 fig.subplots_adjust(**self.plots_adjust)
472 fig.subplots_adjust(**self.plots_adjust)
452 self.firsttime = False
473 self.firsttime = False
453
474
454 def clear_figures(self):
475 def clear_figures(self):
455 '''
476 '''
456 Reset axes for redraw plots
477 Reset axes for redraw plots
457 '''
478 '''
458
479
459 for ax in self.axes+self.pf_axes+self.cb_axes:
480 axes = self.pf_axes + self.cb_axes + self.axes[self.mode]
481
482 for ax in axes:
460 ax.clear()
483 ax.clear()
461 ax.firsttime = True
484 ax.firsttime = True
462 if hasattr(ax, 'cbar') and ax.cbar:
485 if hasattr(ax, 'cbar') and ax.cbar:
463 ax.cbar.remove()
486 ax.cbar.remove()
464
487
465 def __plot(self):
488 def __plot(self):
466 '''
489 '''
467 Main function to plot, format and save figures
490 Main function to plot, format and save figures
468 '''
491 '''
469
492
470 self.plot()
493 self.plot()
471 self.format()
494 self.format()
472
495 figures = self.figures[self.mode]
473 for n, fig in enumerate(self.figures):
496 for n, fig in enumerate(figures):
474 if self.nrows == 0 or self.nplots == 0:
497 if self.nrows == 0 or self.nplots == 0:
475 log.warning('No data', self.name)
498 log.warning('No data', self.name)
476 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
499 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
477 fig.canvas.manager.set_window_title(self.CODE)
500 fig.canvas.manager.set_window_title(self.CODE)
478 continue
501 continue
479
502
480 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
503 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
481 self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
504 self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
482 fig.canvas.draw()
505 fig.canvas.draw()
483 if self.show:
506 if self.show:
484 fig.show()
507 fig.show()
485 figpause(0.01)
508 figpause(0.01)
486
509
487 if self.save:
510 if self.save:
488 self.save_figure(n)
511 self.save_figure(n)
489
512
490 if self.server:
513 if self.server:
491 if self.mode and self.mode == 'RHI':
514 if self.mode and self.mode == 'RHI':
492 return
515 return
493 self.send_to_server()
516 self.send_to_server()
494
517
495 def __update(self, dataOut, timestamp):
518 def __update(self, dataOut, timestamp):
496 '''
519 '''
497 '''
520 '''
498
521
499 metadata = {
522 metadata = {
500 'yrange': dataOut.heightList,
523 'yrange': dataOut.heightList,
501 'interval': dataOut.timeInterval,
524 'interval': dataOut.timeInterval,
502 'channels': dataOut.channelList
525 'channels': dataOut.channelList
503 }
526 }
504
527
505 data, meta = self.update(dataOut)
528 data, meta = self.update(dataOut)
506 metadata.update(meta)
529 metadata.update(meta)
507 self.data.update(data, timestamp, metadata)
530 self.data.update(data, timestamp, metadata)
508
531
509 def save_figure(self, n):
532 def save_figure(self, n):
510 '''
533 '''
511 '''
534 '''
512 if self.mode is not None:
535 if self.mode is not None:
513 ang = 'AZ' if self.mode == 'RHI' else 'EL'
536 ang = 'AZ' if self.mode == 'RHI' else 'EL'
514 label = '_{}_{}_{}'.format(self.mode, ang, self.mode_value)
537 folder = '_{}_{}_{}'.format(self.mode, ang, self.mode_value)
538 label = '{}{}_{}'.format(ang[0], self.mode_value, self.save_code)
515 else:
539 else:
540 folder = ''
516 label = ''
541 label = ''
517
542
518 if self.oneFigure:
543 if self.oneFigure:
519 if (self.data.max_time - self.save_time) <= self.save_period:
544 if (self.data.max_time - self.save_time) <= self.save_period:
520 return
545 return
521
546
522 self.save_time = self.data.max_time
547 self.save_time = self.data.max_time
523
548
524 fig = self.figures[n]
549 fig = self.figures[self.mode][n]
525
550
526 if self.throttle == 0:
551 if self.throttle == 0:
527 if self.oneFigure:
552 if self.oneFigure:
528 figname = os.path.join(
553 figname = os.path.join(
529 self.save,
554 self.save,
530 self.save_code + label,
555 self.save_code + folder,
531 '{}_{}.png'.format(
556 '{}_{}_{}.png'.format(
532 self.save_code + label,
557 'SOPHY',
533 self.getDateTime(self.data.max_time).strftime(
558 self.getDateTime(self.data.max_time).strftime(
534 '%Y%m%d_%H%M%S'
559 '%Y%m%d_%H%M%S'
535 ),
560 ),
561 label
536 )
562 )
537 )
563 )
538 else:
564 else:
539 figname = os.path.join(
565 figname = os.path.join(
540 self.save,
566 self.save,
541 self.save_code,
567 self.save_code,
542 '{}_ch{}_{}.png'.format(
568 '{}_ch{}_{}.png'.format(
543 self.save_code, n,
569 self.save_code, n,
544 self.getDateTime(self.data.max_time).strftime(
570 self.getDateTime(self.data.max_time).strftime(
545 '%Y%m%d_%H%M%S'
571 '%Y%m%d_%H%M%S'
546 ),
572 ),
547 )
573 )
548 )
574 )
549 log.log('Saving figure: {}'.format(figname), self.name)
575 log.log('Saving figure: {}'.format(figname), self.name)
550 if not os.path.isdir(os.path.dirname(figname)):
576 if not os.path.isdir(os.path.dirname(figname)):
551 os.makedirs(os.path.dirname(figname))
577 os.makedirs(os.path.dirname(figname))
552 fig.savefig(figname)
578 fig.savefig(figname)
553
579
554 figname = os.path.join(
580 figname = os.path.join(
555 self.save,
581 self.save,
556 '{}_{}.png'.format(
582 '{}_{}.png'.format(
557 self.save_code,
583 self.save_code,
558 self.getDateTime(self.data.min_time).strftime(
584 self.getDateTime(self.data.min_time).strftime(
559 '%Y%m%d'
585 '%Y%m%d'
560 ),
586 ),
561 )
587 )
562 )
588 )
563
589
564 log.log('Saving figure: {}'.format(figname), self.name)
590 log.log('Saving figure: {}'.format(figname), self.name)
565 if not os.path.isdir(os.path.dirname(figname)):
591 if not os.path.isdir(os.path.dirname(figname)):
566 os.makedirs(os.path.dirname(figname))
592 os.makedirs(os.path.dirname(figname))
567 fig.savefig(figname)
593 fig.savefig(figname)
568
594
569 def send_to_server(self):
595 def send_to_server(self):
570 '''
596 '''
571 '''
597 '''
572
598
573 if self.exp_code == None:
599 if self.exp_code == None:
574 log.warning('Missing `exp_code` skipping sending to server...')
600 log.warning('Missing `exp_code` skipping sending to server...')
575
601
576 last_time = self.data.max_time
602 last_time = self.data.max_time
577 interval = last_time - self.sender_time
603 interval = last_time - self.sender_time
578 if interval < self.sender_period:
604 if interval < self.sender_period:
579 return
605 return
580
606
581 self.sender_time = last_time
607 self.sender_time = last_time
582
608
583 attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
609 attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
584 for attr in attrs:
610 for attr in attrs:
585 value = getattr(self, attr)
611 value = getattr(self, attr)
586 if value:
612 if value:
587 if isinstance(value, (numpy.float32, numpy.float64)):
613 if isinstance(value, (numpy.float32, numpy.float64)):
588 value = round(float(value), 2)
614 value = round(float(value), 2)
589 self.data.meta[attr] = value
615 self.data.meta[attr] = value
590 if self.colormap == 'jet' or self.colormap == 'sophy_w':
616 if self.colormap == 'jet' or self.colormap == 'sophy_w':
591 self.data.meta['colormap'] = 'Jet'
617 self.data.meta['colormap'] = 'Jet'
592 elif 'sophy_v' in self.colormap:
618 elif 'sophy_v' in self.colormap:
593 self.data.meta['colormap'] = 'RdBu'
619 self.data.meta['colormap'] = 'RdBu'
594 else:
620 else:
595 self.data.meta['colormap'] = 'Viridis'
621 self.data.meta['colormap'] = 'Viridis'
596 self.data.meta['interval'] = int(interval)
622 self.data.meta['interval'] = int(interval)
597
623
598 self.sender_queue.append(last_time)
624 self.sender_queue.append(last_time)
599
625
600 while True:
626 while True:
601 try:
627 try:
602 tm = self.sender_queue.popleft()
628 tm = self.sender_queue.popleft()
603 except IndexError:
629 except IndexError:
604 break
630 break
605 msg = self.data.jsonify(tm, self.save_code, self.plot_type, key='var')
631 msg = self.data.jsonify(tm, self.save_code, self.plot_type, key='var')
606 self.socket.send_string(msg)
632 self.socket.send_string(msg)
607 socks = dict(self.poll.poll(2000))
633 socks = dict(self.poll.poll(2000))
608 if socks.get(self.socket) == zmq.POLLIN:
634 if socks.get(self.socket) == zmq.POLLIN:
609 reply = self.socket.recv_string()
635 reply = self.socket.recv_string()
610 if reply == 'ok':
636 if reply == 'ok':
611 log.log("Response from server ok", self.name)
637 log.log("Response from server ok", self.name)
612 time.sleep(0.1)
638 time.sleep(0.1)
613 continue
639 continue
614 else:
640 else:
615 log.warning(
641 log.warning(
616 "Malformed reply from server: {}".format(reply), self.name)
642 "Malformed reply from server: {}".format(reply), self.name)
617 else:
643 else:
618 log.warning(
644 log.warning(
619 "No response from server, retrying...", self.name)
645 "No response from server, retrying...", self.name)
620 self.sender_queue.appendleft(tm)
646 self.sender_queue.appendleft(tm)
621 self.socket.setsockopt(zmq.LINGER, 0)
647 self.socket.setsockopt(zmq.LINGER, 0)
622 self.socket.close()
648 self.socket.close()
623 self.poll.unregister(self.socket)
649 self.poll.unregister(self.socket)
624 self.socket = self.context.socket(zmq.REQ)
650 self.socket = self.context.socket(zmq.REQ)
625 self.socket.connect(self.server)
651 self.socket.connect(self.server)
626 self.poll.register(self.socket, zmq.POLLIN)
652 self.poll.register(self.socket, zmq.POLLIN)
627 break
653 break
628
654
629 def setup(self):
655 def setup(self):
630 '''
656 '''
631 This method should be implemented in the child class, the following
657 This method should be implemented in the child class, the following
632 attributes should be set:
658 attributes should be set:
633
659
634 self.nrows: number of rows
660 self.nrows: number of rows
635 self.ncols: number of cols
661 self.ncols: number of cols
636 self.nplots: number of plots (channels or pairs)
662 self.nplots: number of plots (channels or pairs)
637 self.ylabel: label for Y axes
663 self.ylabel: label for Y axes
638 self.titles: list of axes title
664 self.titles: list of axes title
639
665
640 '''
666 '''
641 raise NotImplementedError
667 raise NotImplementedError
642
668
643 def plot(self):
669 def plot(self):
644 '''
670 '''
645 Must be defined in the child class, the actual plotting method
671 Must be defined in the child class, the actual plotting method
646 '''
672 '''
647 raise NotImplementedError
673 raise NotImplementedError
648
674
649 def update(self, dataOut):
675 def update(self, dataOut):
650 '''
676 '''
651 Must be defined in the child class, update self.data with new data
677 Must be defined in the child class, update self.data with new data
652 '''
678 '''
653
679
654 data = {
680 data = {
655 self.CODE: getattr(dataOut, 'data_{}'.format(self.CODE))
681 self.CODE: getattr(dataOut, 'data_{}'.format(self.CODE))
656 }
682 }
657 meta = {}
683 meta = {}
658
684
659 return data, meta
685 return data, meta
660
686
661 def run(self, dataOut, **kwargs):
687 def run(self, dataOut, **kwargs):
662 '''
688 '''
663 Main plotting routine
689 Main plotting routine
664 '''
690 '''
665
691
666 if self.isConfig is False:
692 if self.isConfig is False:
667 self.__setup(**kwargs)
693 self.__setup(**kwargs)
668
694
669 if self.localtime:
695 if self.localtime:
670 self.getDateTime = datetime.datetime.fromtimestamp
696 self.getDateTime = datetime.datetime.fromtimestamp
671 else:
697 else:
672 self.getDateTime = datetime.datetime.utcfromtimestamp
698 self.getDateTime = datetime.datetime.utcfromtimestamp
673
699
674 self.data.setup()
700 self.data.setup()
675 self.isConfig = True
701 self.isConfig = True
676 if self.server:
702 if self.server:
677 self.context = zmq.Context()
703 self.context = zmq.Context()
678 self.socket = self.context.socket(zmq.REQ)
704 self.socket = self.context.socket(zmq.REQ)
679 self.socket.connect(self.server)
705 self.socket.connect(self.server)
680 self.poll = zmq.Poller()
706 self.poll = zmq.Poller()
681 self.poll.register(self.socket, zmq.POLLIN)
707 self.poll.register(self.socket, zmq.POLLIN)
682
708
683 tm = getattr(dataOut, self.attr_time)
709 tm = getattr(dataOut, self.attr_time)
684
710
685 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
711 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
686 self.save_time = tm
712 self.save_time = tm
687 self.__plot()
713 self.__plot()
688 self.tmin += self.xrange*60*60
714 self.tmin += self.xrange*60*60
689 self.data.setup()
715 self.data.setup()
690 self.clear_figures()
716 self.clear_figures()
691
717
692 self.__update(dataOut, tm)
718 self.__update(dataOut, tm)
693
719
694 if self.isPlotConfig is False:
720 if self.isPlotConfig is False:
695 self.__setup_plot()
721 self.__setup_plot()
696 self.isPlotConfig = True
722 self.isPlotConfig = True
697 if self.xaxis == 'time':
723 if self.xaxis == 'time':
698 dt = self.getDateTime(tm)
724 dt = self.getDateTime(tm)
699 if self.xmin is None:
725 if self.xmin is None:
700 self.tmin = tm
726 self.tmin = tm
701 self.xmin = dt.hour
727 self.xmin = dt.hour
702 minutes = (self.xmin-int(self.xmin)) * 60
728 minutes = (self.xmin-int(self.xmin)) * 60
703 seconds = (minutes - int(minutes)) * 60
729 seconds = (minutes - int(minutes)) * 60
704 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
730 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
705 datetime.datetime(1970, 1, 1)).total_seconds()
731 datetime.datetime(1970, 1, 1)).total_seconds()
706 if self.localtime:
732 if self.localtime:
707 self.tmin += time.timezone
733 self.tmin += time.timezone
708
734
709 if self.xmin is not None and self.xmax is not None:
735 if self.xmin is not None and self.xmax is not None:
710 self.xrange = self.xmax - self.xmin
736 self.xrange = self.xmax - self.xmin
711
737
712 if self.throttle == 0:
738 if self.throttle == 0:
713 self.__plot()
739 self.__plot()
714 else:
740 else:
715 self.__throttle_plot(self.__plot)#, coerce=coerce)
741 self.__throttle_plot(self.__plot)#, coerce=coerce)
716
742
717 def close(self):
743 def close(self):
718
744
719 if self.data and not self.data.flagNoData:
745 if self.data and not self.data.flagNoData:
720 self.save_time = 0
746 self.save_time = 0
721 self.__plot()
747 self.__plot()
722 if self.data and not self.data.flagNoData and self.pause:
748 if self.data and not self.data.flagNoData and self.pause:
723 figpause(10)
749 figpause(10)
Show file before | Show file after | Hide comments
@@ -1,697 +1,716
1 import os
1 import os
2 import datetime
2 import datetime
3 import warnings
3 import warnings
4 import numpy
4 import numpy
5 from mpl_toolkits.axisartist.grid_finder import FixedLocator, DictFormatter
5 from mpl_toolkits.axisartist.grid_finder import FixedLocator, DictFormatter
6 from matplotlib.patches import Circle
7 import cartopy.crs as ccrs
8 from cartopy.feature import ShapelyFeature
9 import cartopy.io.shapereader as shpreader
6
10
7 from schainpy.model.graphics.jroplot_base import Plot, plt
11 from schainpy.model.graphics.jroplot_base import Plot, plt
8 from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot, SpectraCutPlot
12 from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot, SpectraCutPlot
9 from schainpy.utils import log
13 from schainpy.utils import log
10
14
11
15
12 EARTH_RADIUS = 6.3710e3
16 EARTH_RADIUS = 6.3710e3
13
17
14
18
15 def antenna_to_cartesian(ranges, azimuths, elevations):
19 def antenna_to_cartesian(ranges, azimuths, elevations):
16 """
20 """
17 Return Cartesian coordinates from antenna coordinates.
21 Return Cartesian coordinates from antenna coordinates.
18
22
19 Parameters
23 Parameters
20 ----------
24 ----------
21 ranges : array
25 ranges : array
22 Distances to the center of the radar gates (bins) in kilometers.
26 Distances to the center of the radar gates (bins) in kilometers.
23 azimuths : array
27 azimuths : array
24 Azimuth angle of the radar in degrees.
28 Azimuth angle of the radar in degrees.
25 elevations : array
29 elevations : array
26 Elevation angle of the radar in degrees.
30 Elevation angle of the radar in degrees.
27
31
28 Returns
32 Returns
29 -------
33 -------
30 x, y, z : array
34 x, y, z : array
31 Cartesian coordinates in meters from the radar.
35 Cartesian coordinates in meters from the radar.
32
36
33 Notes
37 Notes
34 -----
38 -----
35 The calculation for Cartesian coordinate is adapted from equations
39 The calculation for Cartesian coordinate is adapted from equations
36 2.28(b) and 2.28(c) of Doviak and Zrnic [1]_ assuming a
40 2.28(b) and 2.28(c) of Doviak and Zrnic [1]_ assuming a
37 standard atmosphere (4/3 Earth's radius model).
41 standard atmosphere (4/3 Earth's radius model).
38
42
39 .. math::
43 .. math::
40
44
41 z = \\sqrt{r^2+R^2+2*r*R*sin(\\theta_e)} - R
45 z = \\sqrt{r^2+R^2+2*r*R*sin(\\theta_e)} - R
42
46
43 s = R * arcsin(\\frac{r*cos(\\theta_e)}{R+z})
47 s = R * arcsin(\\frac{r*cos(\\theta_e)}{R+z})
44
48
45 x = s * sin(\\theta_a)
49 x = s * sin(\\theta_a)
46
50
47 y = s * cos(\\theta_a)
51 y = s * cos(\\theta_a)
48
52
49 Where r is the distance from the radar to the center of the gate,
53 Where r is the distance from the radar to the center of the gate,
50 :math:`\\theta_a` is the azimuth angle, :math:`\\theta_e` is the
54 :math:`\\theta_a` is the azimuth angle, :math:`\\theta_e` is the
51 elevation angle, s is the arc length, and R is the effective radius
55 elevation angle, s is the arc length, and R is the effective radius
52 of the earth, taken to be 4/3 the mean radius of earth (6371 km).
56 of the earth, taken to be 4/3 the mean radius of earth (6371 km).
53
57
54 References
58 References
55 ----------
59 ----------
56 .. [1] Doviak and Zrnic, Doppler Radar and Weather Observations, Second
60 .. [1] Doviak and Zrnic, Doppler Radar and Weather Observations, Second
57 Edition, 1993, p. 21.
61 Edition, 1993, p. 21.
58
62
59 """
63 """
60 theta_e = numpy.deg2rad(elevations) # elevation angle in radians.
64 theta_e = numpy.deg2rad(elevations) # elevation angle in radians.
61 theta_a = numpy.deg2rad(azimuths) # azimuth angle in radians.
65 theta_a = numpy.deg2rad(azimuths) # azimuth angle in radians.
62 R = 6371.0 * 1000.0 * 4.0 / 3.0 # effective radius of earth in meters.
66 R = 6371.0 * 1000.0 * 4.0 / 3.0 # effective radius of earth in meters.
63 r = ranges * 1000.0 # distances to gates in meters.
67 r = ranges * 1000.0 # distances to gates in meters.
64
68
65 z = (r ** 2 + R ** 2 + 2.0 * r * R * numpy.sin(theta_e)) ** 0.5 - R
69 z = (r ** 2 + R ** 2 + 2.0 * r * R * numpy.sin(theta_e)) ** 0.5 - R
66 s = R * numpy.arcsin(r * numpy.cos(theta_e) / (R + z)) # arc length in m.
70 s = R * numpy.arcsin(r * numpy.cos(theta_e) / (R + z)) # arc length in m.
67 x = s * numpy.sin(theta_a)
71 x = s * numpy.sin(theta_a)
68 y = s * numpy.cos(theta_a)
72 y = s * numpy.cos(theta_a)
69 return x, y, z
73 return x, y, z
70
74
71 def cartesian_to_geographic_aeqd(x, y, lon_0, lat_0, R=EARTH_RADIUS):
75 def cartesian_to_geographic_aeqd(x, y, lon_0, lat_0, R=EARTH_RADIUS):
72 """
76 """
73 Azimuthal equidistant Cartesian to geographic coordinate transform.
77 Azimuthal equidistant Cartesian to geographic coordinate transform.
74
78
75 Transform a set of Cartesian/Cartographic coordinates (x, y) to
79 Transform a set of Cartesian/Cartographic coordinates (x, y) to
76 geographic coordinate system (lat, lon) using a azimuthal equidistant
80 geographic coordinate system (lat, lon) using a azimuthal equidistant
77 map projection [1]_.
81 map projection [1]_.
78
82
79 .. math::
83 .. math::
80
84
81 lat = \\arcsin(\\cos(c) * \\sin(lat_0) +
85 lat = \\arcsin(\\cos(c) * \\sin(lat_0) +
82 (y * \\sin(c) * \\cos(lat_0) / \\rho))
86 (y * \\sin(c) * \\cos(lat_0) / \\rho))
83
87
84 lon = lon_0 + \\arctan2(
88 lon = lon_0 + \\arctan2(
85 x * \\sin(c),
89 x * \\sin(c),
86 \\rho * \\cos(lat_0) * \\cos(c) - y * \\sin(lat_0) * \\sin(c))
90 \\rho * \\cos(lat_0) * \\cos(c) - y * \\sin(lat_0) * \\sin(c))
87
91
88 \\rho = \\sqrt(x^2 + y^2)
92 \\rho = \\sqrt(x^2 + y^2)
89
93
90 c = \\rho / R
94 c = \\rho / R
91
95
92 Where x, y are the Cartesian position from the center of projection;
96 Where x, y are the Cartesian position from the center of projection;
93 lat, lon the corresponding latitude and longitude; lat_0, lon_0 are the
97 lat, lon the corresponding latitude and longitude; lat_0, lon_0 are the
94 latitude and longitude of the center of the projection; R is the radius of
98 latitude and longitude of the center of the projection; R is the radius of
95 the earth (defaults to ~6371 km). lon is adjusted to be between -180 and
99 the earth (defaults to ~6371 km). lon is adjusted to be between -180 and
96 180.
100 180.
97
101
98 Parameters
102 Parameters
99 ----------
103 ----------
100 x, y : array-like
104 x, y : array-like
101 Cartesian coordinates in the same units as R, typically meters.
105 Cartesian coordinates in the same units as R, typically meters.
102 lon_0, lat_0 : float
106 lon_0, lat_0 : float
103 Longitude and latitude, in degrees, of the center of the projection.
107 Longitude and latitude, in degrees, of the center of the projection.
104 R : float, optional
108 R : float, optional
105 Earth radius in the same units as x and y. The default value is in
109 Earth radius in the same units as x and y. The default value is in
106 units of meters.
110 units of meters.
107
111
108 Returns
112 Returns
109 -------
113 -------
110 lon, lat : array
114 lon, lat : array
111 Longitude and latitude of Cartesian coordinates in degrees.
115 Longitude and latitude of Cartesian coordinates in degrees.
112
116
113 References
117 References
114 ----------
118 ----------
115 .. [1] Snyder, J. P. Map Projections--A Working Manual. U. S. Geological
119 .. [1] Snyder, J. P. Map Projections--A Working Manual. U. S. Geological
116 Survey Professional Paper 1395, 1987, pp. 191-202.
120 Survey Professional Paper 1395, 1987, pp. 191-202.
117
121
118 """
122 """
119 x = numpy.atleast_1d(numpy.asarray(x))
123 x = numpy.atleast_1d(numpy.asarray(x))
120 y = numpy.atleast_1d(numpy.asarray(y))
124 y = numpy.atleast_1d(numpy.asarray(y))
121
125
122 lat_0_rad = numpy.deg2rad(lat_0)
126 lat_0_rad = numpy.deg2rad(lat_0)
123 lon_0_rad = numpy.deg2rad(lon_0)
127 lon_0_rad = numpy.deg2rad(lon_0)
124
128
125 rho = numpy.sqrt(x*x + y*y)
129 rho = numpy.sqrt(x*x + y*y)
126 c = rho / R
130 c = rho / R
127
131
128 with warnings.catch_warnings():
132 with warnings.catch_warnings():
129 # division by zero may occur here but is properly addressed below so
133 # division by zero may occur here but is properly addressed below so
130 # the warnings can be ignored
134 # the warnings can be ignored
131 warnings.simplefilter("ignore", RuntimeWarning)
135 warnings.simplefilter("ignore", RuntimeWarning)
132 lat_rad = numpy.arcsin(numpy.cos(c) * numpy.sin(lat_0_rad) +
136 lat_rad = numpy.arcsin(numpy.cos(c) * numpy.sin(lat_0_rad) +
133 y * numpy.sin(c) * numpy.cos(lat_0_rad) / rho)
137 y * numpy.sin(c) * numpy.cos(lat_0_rad) / rho)
134 lat_deg = numpy.rad2deg(lat_rad)
138 lat_deg = numpy.rad2deg(lat_rad)
135 # fix cases where the distance from the center of the projection is zero
139 # fix cases where the distance from the center of the projection is zero
136 lat_deg[rho == 0] = lat_0
140 lat_deg[rho == 0] = lat_0
137
141
138 x1 = x * numpy.sin(c)
142 x1 = x * numpy.sin(c)
139 x2 = rho*numpy.cos(lat_0_rad)*numpy.cos(c) - y*numpy.sin(lat_0_rad)*numpy.sin(c)
143 x2 = rho*numpy.cos(lat_0_rad)*numpy.cos(c) - y*numpy.sin(lat_0_rad)*numpy.sin(c)
140 lon_rad = lon_0_rad + numpy.arctan2(x1, x2)
144 lon_rad = lon_0_rad + numpy.arctan2(x1, x2)
141 lon_deg = numpy.rad2deg(lon_rad)
145 lon_deg = numpy.rad2deg(lon_rad)
142 # Longitudes should be from -180 to 180 degrees
146 # Longitudes should be from -180 to 180 degrees
143 lon_deg[lon_deg > 180] -= 360.
147 lon_deg[lon_deg > 180] -= 360.
144 lon_deg[lon_deg < -180] += 360.
148 lon_deg[lon_deg < -180] += 360.
145
149
146 return lon_deg, lat_deg
150 return lon_deg, lat_deg
147
151
148 def antenna_to_geographic(ranges, azimuths, elevations, site):
152 def antenna_to_geographic(ranges, azimuths, elevations, site):
149
153
150 x, y, z = antenna_to_cartesian(numpy.array(ranges), numpy.array(azimuths), numpy.array(elevations))
154 x, y, z = antenna_to_cartesian(numpy.array(ranges), numpy.array(azimuths), numpy.array(elevations))
151 lon, lat = cartesian_to_geographic_aeqd(x, y, site[0], site[1], R=6370997.)
155 lon, lat = cartesian_to_geographic_aeqd(x, y, site[0], site[1], R=6370997.)
152
156
153 return lon, lat
157 return lon, lat
154
158
155 def ll2xy(lat1, lon1, lat2, lon2):
159 def ll2xy(lat1, lon1, lat2, lon2):
156
160
157 p = 0.017453292519943295
161 p = 0.017453292519943295
158 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
162 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
159 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
163 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
160 r = 12742 * numpy.arcsin(numpy.sqrt(a))
164 r = 12742 * numpy.arcsin(numpy.sqrt(a))
161 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
165 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
162 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
166 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
163 theta = -theta + numpy.pi/2
167 theta = -theta + numpy.pi/2
164 return r*numpy.cos(theta), r*numpy.sin(theta)
168 return r*numpy.cos(theta), r*numpy.sin(theta)
165
169
166
170
167 def km2deg(km):
171 def km2deg(km):
168 '''
172 '''
169 Convert distance in km to degrees
173 Convert distance in km to degrees
170 '''
174 '''
171
175
172 return numpy.rad2deg(km/EARTH_RADIUS)
176 return numpy.rad2deg(km/EARTH_RADIUS)
173
177
174
178
175
179
176 class SpectralMomentsPlot(SpectraPlot):
180 class SpectralMomentsPlot(SpectraPlot):
177 '''
181 '''
178 Plot for Spectral Moments
182 Plot for Spectral Moments
179 '''
183 '''
180 CODE = 'spc_moments'
184 CODE = 'spc_moments'
181 # colormap = 'jet'
185 # colormap = 'jet'
182 # plot_type = 'pcolor'
186 # plot_type = 'pcolor'
183
187
184 class DobleGaussianPlot(SpectraPlot):
188 class DobleGaussianPlot(SpectraPlot):
185 '''
189 '''
186 Plot for Double Gaussian Plot
190 Plot for Double Gaussian Plot
187 '''
191 '''
188 CODE = 'gaussian_fit'
192 CODE = 'gaussian_fit'
189 # colormap = 'jet'
193 # colormap = 'jet'
190 # plot_type = 'pcolor'
194 # plot_type = 'pcolor'
191
195
192 class DoubleGaussianSpectraCutPlot(SpectraCutPlot):
196 class DoubleGaussianSpectraCutPlot(SpectraCutPlot):
193 '''
197 '''
194 Plot SpectraCut with Double Gaussian Fit
198 Plot SpectraCut with Double Gaussian Fit
195 '''
199 '''
196 CODE = 'cut_gaussian_fit'
200 CODE = 'cut_gaussian_fit'
197
201
198 class SnrPlot(RTIPlot):
202 class SnrPlot(RTIPlot):
199 '''
203 '''
200 Plot for SNR Data
204 Plot for SNR Data
201 '''
205 '''
202
206
203 CODE = 'snr'
207 CODE = 'snr'
204 colormap = 'jet'
208 colormap = 'jet'
205
209
206 def update(self, dataOut):
210 def update(self, dataOut):
207
211
208 data = {
212 data = {
209 'snr': 10*numpy.log10(dataOut.data_snr)
213 'snr': 10*numpy.log10(dataOut.data_snr)
210 }
214 }
211
215
212 return data, {}
216 return data, {}
213
217
214 class DopplerPlot(RTIPlot):
218 class DopplerPlot(RTIPlot):
215 '''
219 '''
216 Plot for DOPPLER Data (1st moment)
220 Plot for DOPPLER Data (1st moment)
217 '''
221 '''
218
222
219 CODE = 'dop'
223 CODE = 'dop'
220 colormap = 'jet'
224 colormap = 'jet'
221
225
222 def update(self, dataOut):
226 def update(self, dataOut):
223
227
224 data = {
228 data = {
225 'dop': 10*numpy.log10(dataOut.data_dop)
229 'dop': 10*numpy.log10(dataOut.data_dop)
226 }
230 }
227
231
228 return data, {}
232 return data, {}
229
233
230 class PowerPlot(RTIPlot):
234 class PowerPlot(RTIPlot):
231 '''
235 '''
232 Plot for Power Data (0 moment)
236 Plot for Power Data (0 moment)
233 '''
237 '''
234
238
235 CODE = 'pow'
239 CODE = 'pow'
236 colormap = 'jet'
240 colormap = 'jet'
237
241
238 def update(self, dataOut):
242 def update(self, dataOut):
239 data = {
243 data = {
240 'pow': 10*numpy.log10(dataOut.data_pow/dataOut.normFactor)
244 'pow': 10*numpy.log10(dataOut.data_pow/dataOut.normFactor)
241 }
245 }
242 return data, {}
246 return data, {}
243
247
244 class SpectralWidthPlot(RTIPlot):
248 class SpectralWidthPlot(RTIPlot):
245 '''
249 '''
246 Plot for Spectral Width Data (2nd moment)
250 Plot for Spectral Width Data (2nd moment)
247 '''
251 '''
248
252
249 CODE = 'width'
253 CODE = 'width'
250 colormap = 'jet'
254 colormap = 'jet'
251
255
252 def update(self, dataOut):
256 def update(self, dataOut):
253
257
254 data = {
258 data = {
255 'width': dataOut.data_width
259 'width': dataOut.data_width
256 }
260 }
257
261
258 return data, {}
262 return data, {}
259
263
260 class SkyMapPlot(Plot):
264 class SkyMapPlot(Plot):
261 '''
265 '''
262 Plot for meteors detection data
266 Plot for meteors detection data
263 '''
267 '''
264
268
265 CODE = 'param'
269 CODE = 'param'
266
270
267 def setup(self):
271 def setup(self):
268
272
269 self.ncols = 1
273 self.ncols = 1
270 self.nrows = 1
274 self.nrows = 1
271 self.width = 7.2
275 self.width = 7.2
272 self.height = 7.2
276 self.height = 7.2
273 self.nplots = 1
277 self.nplots = 1
274 self.xlabel = 'Zonal Zenith Angle (deg)'
278 self.xlabel = 'Zonal Zenith Angle (deg)'
275 self.ylabel = 'Meridional Zenith Angle (deg)'
279 self.ylabel = 'Meridional Zenith Angle (deg)'
276 self.polar = True
280 self.polar = True
277 self.ymin = -180
281 self.ymin = -180
278 self.ymax = 180
282 self.ymax = 180
279 self.colorbar = False
283 self.colorbar = False
280
284
281 def plot(self):
285 def plot(self):
282
286
283 arrayParameters = numpy.concatenate(self.data['param'])
287 arrayParameters = numpy.concatenate(self.data['param'])
284 error = arrayParameters[:, -1]
288 error = arrayParameters[:, -1]
285 indValid = numpy.where(error == 0)[0]
289 indValid = numpy.where(error == 0)[0]
286 finalMeteor = arrayParameters[indValid, :]
290 finalMeteor = arrayParameters[indValid, :]
287 finalAzimuth = finalMeteor[:, 3]
291 finalAzimuth = finalMeteor[:, 3]
288 finalZenith = finalMeteor[:, 4]
292 finalZenith = finalMeteor[:, 4]
289
293
290 x = finalAzimuth * numpy.pi / 180
294 x = finalAzimuth * numpy.pi / 180
291 y = finalZenith
295 y = finalZenith
292
296
293 ax = self.axes[0]
297 ax = self.axes[0]
294
298
295 if ax.firsttime:
299 if ax.firsttime:
296 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
300 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
297 else:
301 else:
298 ax.plot.set_data(x, y)
302 ax.plot.set_data(x, y)
299
303
300 dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
304 dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
301 dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
305 dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
302 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
306 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
303 dt2,
307 dt2,
304 len(x))
308 len(x))
305 self.titles[0] = title
309 self.titles[0] = title
306
310
307
311
308 class GenericRTIPlot(Plot):
312 class GenericRTIPlot(Plot):
309 '''
313 '''
310 Plot for data_xxxx object
314 Plot for data_xxxx object
311 '''
315 '''
312
316
313 CODE = 'param'
317 CODE = 'param'
314 colormap = 'viridis'
318 colormap = 'viridis'
315 plot_type = 'pcolorbuffer'
319 plot_type = 'pcolorbuffer'
316
320
317 def setup(self):
321 def setup(self):
318 self.xaxis = 'time'
322 self.xaxis = 'time'
319 self.ncols = 1
323 self.ncols = 1
320 self.nrows = self.data.shape('param')[0]
324 self.nrows = self.data.shape('param')[0]
321 self.nplots = self.nrows
325 self.nplots = self.nrows
322 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
326 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
323
327
324 if not self.xlabel:
328 if not self.xlabel:
325 self.xlabel = 'Time'
329 self.xlabel = 'Time'
326
330
327 self.ylabel = 'Range [km]'
331 self.ylabel = 'Range [km]'
328 if not self.titles:
332 if not self.titles:
329 self.titles = ['Param {}'.format(x) for x in range(self.nrows)]
333 self.titles = ['Param {}'.format(x) for x in range(self.nrows)]
330
334
331 def update(self, dataOut):
335 def update(self, dataOut):
332
336
333 data = {
337 data = {
334 'param' : numpy.concatenate([getattr(dataOut, attr) for attr in self.attr_data], axis=0)
338 'param' : numpy.concatenate([getattr(dataOut, attr) for attr in self.attr_data], axis=0)
335 }
339 }
336
340
337 meta = {}
341 meta = {}
338
342
339 return data, meta
343 return data, meta
340
344
341 def plot(self):
345 def plot(self):
342 # self.data.normalize_heights()
346 # self.data.normalize_heights()
343 self.x = self.data.times
347 self.x = self.data.times
344 self.y = self.data.yrange
348 self.y = self.data.yrange
345 self.z = self.data['param']
349 self.z = self.data['param']
346 self.z = 10*numpy.log10(self.z)
350 self.z = 10*numpy.log10(self.z)
347 self.z = numpy.ma.masked_invalid(self.z)
351 self.z = numpy.ma.masked_invalid(self.z)
348
352
349 if self.decimation is None:
353 if self.decimation is None:
350 x, y, z = self.fill_gaps(self.x, self.y, self.z)
354 x, y, z = self.fill_gaps(self.x, self.y, self.z)
351 else:
355 else:
352 x, y, z = self.fill_gaps(*self.decimate())
356 x, y, z = self.fill_gaps(*self.decimate())
353
357
354 for n, ax in enumerate(self.axes):
358 for n, ax in enumerate(self.axes):
355
359
356 self.zmax = self.zmax if self.zmax is not None else numpy.max(
360 self.zmax = self.zmax if self.zmax is not None else numpy.max(
357 self.z[n])
361 self.z[n])
358 self.zmin = self.zmin if self.zmin is not None else numpy.min(
362 self.zmin = self.zmin if self.zmin is not None else numpy.min(
359 self.z[n])
363 self.z[n])
360
364
361 if ax.firsttime:
365 if ax.firsttime:
362 if self.zlimits is not None:
366 if self.zlimits is not None:
363 self.zmin, self.zmax = self.zlimits[n]
367 self.zmin, self.zmax = self.zlimits[n]
364
368
365 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
369 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
366 vmin=self.zmin,
370 vmin=self.zmin,
367 vmax=self.zmax,
371 vmax=self.zmax,
368 cmap=self.cmaps[n]
372 cmap=self.cmaps[n]
369 )
373 )
370 else:
374 else:
371 if self.zlimits is not None:
375 if self.zlimits is not None:
372 self.zmin, self.zmax = self.zlimits[n]
376 self.zmin, self.zmax = self.zlimits[n]
373 ax.collections.remove(ax.collections[0])
377 ax.collections.remove(ax.collections[0])
374 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
378 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
375 vmin=self.zmin,
379 vmin=self.zmin,
376 vmax=self.zmax,
380 vmax=self.zmax,
377 cmap=self.cmaps[n]
381 cmap=self.cmaps[n]
378 )
382 )
379
383
380
384
381 class PolarMapPlot(Plot):
385 class PolarMapPlot(Plot):
382 '''
386 '''
383 Plot for weather radar
387 Plot for weather radar
384 '''
388 '''
385
389
386 CODE = 'param'
390 CODE = 'param'
387 colormap = 'seismic'
391 colormap = 'seismic'
388
392
389 def setup(self):
393 def setup(self):
390 self.ncols = 1
394 self.ncols = 1
391 self.nrows = 1
395 self.nrows = 1
392 self.width = 9
396 self.width = 9
393 self.height = 8
397 self.height = 8
394 self.mode = self.data.meta['mode']
398 self.mode = self.data.meta['mode']
395 if self.channels is not None:
399 if self.channels is not None:
396 self.nplots = len(self.channels)
400 self.nplots = len(self.channels)
397 self.nrows = len(self.channels)
401 self.nrows = len(self.channels)
398 else:
402 else:
399 self.nplots = self.data.shape(self.CODE)[0]
403 self.nplots = self.data.shape(self.CODE)[0]
400 self.nrows = self.nplots
404 self.nrows = self.nplots
401 self.channels = list(range(self.nplots))
405 self.channels = list(range(self.nplots))
402 if self.mode == 'E':
406 if self.mode == 'E':
403 self.xlabel = 'Longitude'
407 self.xlabel = 'Longitude'
404 self.ylabel = 'Latitude'
408 self.ylabel = 'Latitude'
405 else:
409 else:
406 self.xlabel = 'Range (km)'
410 self.xlabel = 'Range (km)'
407 self.ylabel = 'Height (km)'
411 self.ylabel = 'Height (km)'
408 self.bgcolor = 'white'
412 self.bgcolor = 'white'
409 self.cb_labels = self.data.meta['units']
413 self.cb_labels = self.data.meta['units']
410 self.lat = self.data.meta['latitude']
414 self.lat = self.data.meta['latitude']
411 self.lon = self.data.meta['longitude']
415 self.lon = self.data.meta['longitude']
412 self.xmin, self.xmax = float(
416 self.xmin, self.xmax = float(
413 km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
417 km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
414 self.ymin, self.ymax = float(
418 self.ymin, self.ymax = float(
415 km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
419 km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
416 # self.polar = True
420 # self.polar = True
417
421
418 def plot(self):
422 def plot(self):
419
423
420 for n, ax in enumerate(self.axes):
424 for n, ax in enumerate(self.axes):
421 data = self.data['param'][self.channels[n]]
425 data = self.data['param'][self.channels[n]]
422
426
423 zeniths = numpy.linspace(
427 zeniths = numpy.linspace(
424 0, self.data.meta['max_range'], data.shape[1])
428 0, self.data.meta['max_range'], data.shape[1])
425 if self.mode == 'E':
429 if self.mode == 'E':
426 azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
430 azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
427 r, theta = numpy.meshgrid(zeniths, azimuths)
431 r, theta = numpy.meshgrid(zeniths, azimuths)
428 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
432 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
429 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
433 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
430 x = km2deg(x) + self.lon
434 x = km2deg(x) + self.lon
431 y = km2deg(y) + self.lat
435 y = km2deg(y) + self.lat
432 else:
436 else:
433 azimuths = numpy.radians(self.data.yrange)
437 azimuths = numpy.radians(self.data.yrange)
434 r, theta = numpy.meshgrid(zeniths, azimuths)
438 r, theta = numpy.meshgrid(zeniths, azimuths)
435 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
439 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
436 self.y = zeniths
440 self.y = zeniths
437
441
438 if ax.firsttime:
442 if ax.firsttime:
439 if self.zlimits is not None:
443 if self.zlimits is not None:
440 self.zmin, self.zmax = self.zlimits[n]
444 self.zmin, self.zmax = self.zlimits[n]
441 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
445 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
442 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
446 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
443 vmin=self.zmin,
447 vmin=self.zmin,
444 vmax=self.zmax,
448 vmax=self.zmax,
445 cmap=self.cmaps[n])
449 cmap=self.cmaps[n])
446 else:
450 else:
447 if self.zlimits is not None:
451 if self.zlimits is not None:
448 self.zmin, self.zmax = self.zlimits[n]
452 self.zmin, self.zmax = self.zlimits[n]
449 ax.collections.remove(ax.collections[0])
453 ax.collections.remove(ax.collections[0])
450 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
454 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
451 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
455 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
452 vmin=self.zmin,
456 vmin=self.zmin,
453 vmax=self.zmax,
457 vmax=self.zmax,
454 cmap=self.cmaps[n])
458 cmap=self.cmaps[n])
455
459
456 if self.mode == 'A':
460 if self.mode == 'A':
457 continue
461 continue
458
462
459 # plot district names
463 # plot district names
460 f = open('/data/workspace/schain_scripts/distrito.csv')
464 f = open('/data/workspace/schain_scripts/distrito.csv')
461 for line in f:
465 for line in f:
462 label, lon, lat = [s.strip() for s in line.split(',') if s]
466 label, lon, lat = [s.strip() for s in line.split(',') if s]
463 lat = float(lat)
467 lat = float(lat)
464 lon = float(lon)
468 lon = float(lon)
465 # ax.plot(lon, lat, '.b', ms=2)
469 # ax.plot(lon, lat, '.b', ms=2)
466 ax.text(lon, lat, label.decode('utf8'), ha='center',
470 ax.text(lon, lat, label.decode('utf8'), ha='center',
467 va='bottom', size='8', color='black')
471 va='bottom', size='8', color='black')
468
472
469 # plot limites
473 # plot limites
470 limites = []
474 limites = []
471 tmp = []
475 tmp = []
472 for line in open('/data/workspace/schain_scripts/lima.csv'):
476 for line in open('/data/workspace/schain_scripts/lima.csv'):
473 if '#' in line:
477 if '#' in line:
474 if tmp:
478 if tmp:
475 limites.append(tmp)
479 limites.append(tmp)
476 tmp = []
480 tmp = []
477 continue
481 continue
478 values = line.strip().split(',')
482 values = line.strip().split(',')
479 tmp.append((float(values[0]), float(values[1])))
483 tmp.append((float(values[0]), float(values[1])))
480 for points in limites:
484 for points in limites:
481 ax.add_patch(
485 ax.add_patch(
482 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
486 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
483
487
484 # plot Cuencas
488 # plot Cuencas
485 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
489 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
486 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
490 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
487 values = [line.strip().split(',') for line in f]
491 values = [line.strip().split(',') for line in f]
488 points = [(float(s[0]), float(s[1])) for s in values]
492 points = [(float(s[0]), float(s[1])) for s in values]
489 ax.add_patch(Polygon(points, ec='b', fc='none'))
493 ax.add_patch(Polygon(points, ec='b', fc='none'))
490
494
491 # plot grid
495 # plot grid
492 for r in (15, 30, 45, 60):
496 for r in (15, 30, 45, 60):
493 ax.add_artist(plt.Circle((self.lon, self.lat),
497 ax.add_artist(plt.Circle((self.lon, self.lat),
494 km2deg(r), color='0.6', fill=False, lw=0.2))
498 km2deg(r), color='0.6', fill=False, lw=0.2))
495 ax.text(
499 ax.text(
496 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
500 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
497 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
501 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
498 '{}km'.format(r),
502 '{}km'.format(r),
499 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
503 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
500
504
501 if self.mode == 'E':
505 if self.mode == 'E':
502 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
506 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
503 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
507 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
504 else:
508 else:
505 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
509 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
506 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
510 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
507
511
508 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
512 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
509 self.titles = ['{} {}'.format(
513 self.titles = ['{} {}'.format(
510 self.data.parameters[x], title) for x in self.channels]
514 self.data.parameters[x], title) for x in self.channels]
511
515
512 class WeatherParamsPlot(Plot):
516 class WeatherParamsPlot(Plot):
513 #CODE = 'RHI'
517 #CODE = 'RHI'
514 #plot_name = 'RHI'
518 #plot_name = 'RHI'
515 plot_type = 'scattermap'
519 plot_type = 'scattermap'
516 buffering = False
520 buffering = False
521 projection = ccrs.PlateCarree()
517
522
518 def setup(self):
523 def setup(self):
519
524
520 self.ncols = 1
525 self.ncols = 1
521 self.nrows = 1
526 self.nrows = 1
522 self.nplots= 1
527 self.nplots= 1
523 self.ylabel= 'Range [km]'
528 self.ylabel= 'Range [km]'
524 self.xlabel= 'Range [km]'
529 self.xlabel= 'Range [km]'
525 self.polar = True
530
526 self.grid = True
527 if self.channels is not None:
531 if self.channels is not None:
528 self.nplots = len(self.channels)
532 self.nplots = len(self.channels)
529 self.ncols = len(self.channels)
533 self.ncols = len(self.channels)
530 else:
534 else:
531 self.nplots = self.data.shape(self.CODE)[0]
535 self.nplots = self.data.shape(self.CODE)[0]
532 self.ncols = self.nplots
536 self.ncols = self.nplots
533 self.channels = list(range(self.nplots))
537 self.channels = list(range(self.nplots))
534
538
535 self.colorbar=True
539 self.colorbar=True
536 if len(self.channels)>1:
540 if len(self.channels)>1:
537 self.width = 12
541 self.width = 12
538 else:
542 else:
539 self.width =8
543 self.width =8
540 self.height =8
544 self.height =7
541 self.ini =0
545 self.ini =0
542 self.len_azi =0
546 self.len_azi =0
543 self.buffer_ini = None
547 self.buffer_ini = None
544 self.buffer_ele = None
548 self.buffer_ele = None
545 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
549 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
546 self.flag =0
550 self.flag =0
547 self.indicador= 0
551 self.indicador= 0
548 self.last_data_ele = None
552 self.last_data_ele = None
549 self.val_mean = None
553 self.val_mean = None
550
554
551 def update(self, dataOut):
555 def update(self, dataOut):
552
556
553 vars = {
557 vars = {
554 'S' : 0,
558 'S' : 0,
555 'V' : 1,
559 'V' : 1,
556 'W' : 2,
560 'W' : 2,
557 'SNR' : 3,
561 'SNR' : 3,
558 'Z' : 4,
562 'Z' : 4,
559 'D' : 5,
563 'D' : 5,
560 'P' : 6,
564 'P' : 6,
561 'R' : 7,
565 'R' : 7,
562 }
566 }
563
567
564 data = {}
568 data = {}
565 meta = {}
569 meta = {}
566
570
567 if hasattr(dataOut, 'nFFTPoints'):
571 if hasattr(dataOut, 'nFFTPoints'):
568 factor = dataOut.normFactor
572 factor = dataOut.normFactor
569 else:
573 else:
570 factor = 1
574 factor = 1
571
575
572 if hasattr(dataOut, 'dparam'):
576 if hasattr(dataOut, 'dparam'):
573 tmp = getattr(dataOut, 'data_param')
577 tmp = getattr(dataOut, 'data_param')
574 else:
578 else:
575
579
576 if 'S' in self.attr_data[0]:
580 if 'S' in self.attr_data[0]:
577 tmp = 10*numpy.log10(10.0*getattr(dataOut, 'data_param')[:,0,:]/(factor))
581 tmp = 10*numpy.log10(10.0*getattr(dataOut, 'data_param')[:,0,:]/(factor))
578 else:
582 else:
579 tmp = getattr(dataOut, 'data_param')[:,vars[self.attr_data[0]],:]
583 tmp = getattr(dataOut, 'data_param')[:,vars[self.attr_data[0]],:]
580
584
581 if self.mask:
585 if self.mask:
582 mask = dataOut.data_param[:,3,:] < self.mask
586 mask = dataOut.data_param[:,3,:] < self.mask
583 tmp = numpy.ma.masked_array(tmp, mask=mask)
587 tmp = numpy.ma.masked_array(tmp, mask=mask)
584
588
585 r = dataOut.heightList
589 r = dataOut.heightList
586 delta_height = r[1]-r[0]
590 delta_height = r[1]-r[0]
587 valid = numpy.where(r>=0)[0]
591 valid = numpy.where(r>=0)[0]
588 data['r'] = numpy.arange(len(valid))*delta_height
592 data['r'] = numpy.arange(len(valid))*delta_height
589
593
590 data['data'] = [0, 0]
594 data['data'] = [0, 0]
591
595
592 #try:
596 try:
593 data['data'][0] = tmp[0][:,valid]
597 data['data'][0] = tmp[0][:,valid]
594 data['data'][1] = tmp[1][:,valid]
598 data['data'][1] = tmp[1][:,valid]
595 #except:
599 except:
596 # data['data'] = tmp[0][:,valid]
600 data['data'][0] = tmp[0][:,valid]
601 data['data'][1] = tmp[0][:,valid]
597
602
598 if dataOut.mode_op == 'PPI':
603 if dataOut.mode_op == 'PPI':
599 self.CODE = 'PPI'
604 self.CODE = 'PPI'
600 self.title = self.CODE
605 self.title = self.CODE
601 elif dataOut.mode_op == 'RHI':
606 elif dataOut.mode_op == 'RHI':
602 self.CODE = 'RHI'
607 self.CODE = 'RHI'
603 self.title = self.CODE
608 self.title = self.CODE
604
609
605 data['azi'] = dataOut.data_azi
610 data['azi'] = dataOut.data_azi
606 data['ele'] = dataOut.data_ele
611 data['ele'] = dataOut.data_ele
607 data['mode_op'] = dataOut.mode_op
612 data['mode_op'] = dataOut.mode_op
608 self.mode = dataOut.mode_op
613 self.mode = dataOut.mode_op
609 var = data['data'][0].flatten()
610 r = numpy.tile(data['r'], data['data'][0].shape[0])
611 az = numpy.repeat(data['azi'], data['data'][0].shape[1])
612 el = numpy.repeat(data['ele'], data['data'][0].shape[1])
613
614 # lla = georef.spherical_to_proj(r, data['azi'], data['ele'], (-75.295893, -12.040436, 3379.2147))
615
616 latlon = antenna_to_geographic(r, az, el, (-75.295893, -12.040436))
617
618 if self.mask:
619 meta['lat'] = latlon[1][var.mask==False]
620 meta['lon'] = latlon[0][var.mask==False]
621 data['var'] = numpy.array([var[var.mask==False]])
622 else:
623 meta['lat'] = latlon[1]
624 meta['lon'] = latlon[0]
625 data['var'] = numpy.array([var])
626
614
627 return data, meta
615 return data, meta
628
616
629 def plot(self):
617 def plot(self):
630 data = self.data[-1]
618 data = self.data[-1]
631 z = data['data']
619 z = data['data']
632 r = data['r']
620 r = data['r']
633 self.titles = []
621 self.titles = []
634
622
635 self.ymax = self.ymax if self.ymax else numpy.nanmax(r)
623 self.ymax = self.ymax if self.ymax else numpy.nanmax(r)
636 self.ymin = self.ymin if self.ymin else numpy.nanmin(r)
624 self.ymin = self.ymin if self.ymin else numpy.nanmin(r)
637 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
625 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
638 self.zmin = self.zmin if self.zmin is not None else numpy.nanmin(z)
626 self.zmin = self.zmin if self.zmin is not None else numpy.nanmin(z)
639
627
640 if isinstance(data['mode_op'], bytes):
628 if isinstance(data['mode_op'], bytes):
641 data['mode_op'] = data['mode_op'].decode()
629 data['mode_op'] = data['mode_op'].decode()
642
630
643 if data['mode_op'] == 'RHI':
631 if data['mode_op'] == 'RHI':
644 try:
632 r, theta = numpy.meshgrid(r, numpy.radians(data['ele']))
645 if self.data['mode_op'][-2] == 'PPI':
633 len_aux = int(data['azi'].shape[0]/4)
646 self.ang_min = None
634 mean = numpy.mean(data['azi'][len_aux:-len_aux])
647 self.ang_max = None
635 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
648 except:
636 elif data['mode_op'] == 'PPI':
649 pass
637 r, theta = numpy.meshgrid(r, -numpy.radians(data['azi'])+numpy.pi/2)
650 self.ang_min = self.ang_min if self.ang_min else 0
638 len_aux = int(data['ele'].shape[0]/4)
651 self.ang_max = self.ang_max if self.ang_max else 90
639 mean = numpy.mean(data['ele'][len_aux:-len_aux])
652 r, theta = numpy.meshgrid(r, numpy.radians(data['ele']) )
640 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(mean)), r*numpy.sin(
653 elif data['mode_op'] == 'PPI':
641 theta)*numpy.cos(numpy.radians(mean))
654 try:
642 x = km2deg(x) + -75.295893
655 if self.data['mode_op'][-2] == 'RHI':
643 y = km2deg(y) + -12.040436
656 self.ang_min = None
657 self.ang_max = None
658 except:
659 pass
660 self.ang_min = self.ang_min if self.ang_min else 0
661 self.ang_max = self.ang_max if self.ang_max else 360
662 r, theta = numpy.meshgrid(r, numpy.radians(data['azi']) )
663
644
664 self.clear_figures()
645 self.clear_figures()
665
646
666 for i,ax in enumerate(self.axes):
647 if data['mode_op'] == 'PPI':
667
648 axes = self.axes['PPI']
668 if ax.firsttime:
649 else:
669 ax.set_xlim(numpy.radians(self.ang_min),numpy.radians(self.ang_max))
650 axes = self.axes['RHI']
670 ax.plt = ax.pcolormesh(theta, r, z[i], cmap=self.colormap, vmin=self.zmin, vmax=self.zmax)
671 if data['mode_op'] == 'PPI':
672 ax.set_theta_direction(-1)
673 ax.set_theta_offset(numpy.pi/2)
674
651
675 else:
652 for i, ax in enumerate(axes):
676 ax.set_xlim(numpy.radians(self.ang_min),numpy.radians(self.ang_max))
653 if data['mode_op'] == 'PPI':
677 ax.plt = ax.pcolormesh(theta, r, z[i], cmap=self.colormap, vmin=self.zmin, vmax=self.zmax)
654 ax.set_extent([-75.745893, -74.845893, -12.490436, -11.590436])
678 if data['mode_op'] == 'PPI':
655
679 ax.set_theta_direction(-1)
656 ax.plt = ax.pcolormesh(x, y, z[i], cmap=self.colormap, vmin=self.zmin, vmax=self.zmax)
680 ax.set_theta_offset(numpy.pi/2)
681
657
682 ax.grid(True)
683 if data['mode_op'] == 'RHI':
658 if data['mode_op'] == 'RHI':
684 len_aux = int(data['azi'].shape[0]/4)
659 len_aux = int(data['azi'].shape[0]/4)
685 mean = numpy.mean(data['azi'][len_aux:-len_aux])
660 mean = numpy.mean(data['azi'][len_aux:-len_aux])
686 if len(self.channels) !=1:
661 if len(self.channels) !=1:
687 self.titles = ['RHI {} at AZ: {} CH {}'.format(self.labels[x], str(round(mean,1)), x) for x in self.channels]
662 self.titles = ['RHI {} at AZ: {} CH {}'.format(self.labels[x], str(round(mean,1)), x) for x in self.channels]
688 else:
663 else:
689 self.titles = ['RHI {} at AZ: {} CH {}'.format(self.labels[0], str(round(mean,1)), self.channels[0])]
664 self.titles = ['RHI {} at AZ: {} CH {}'.format(self.labels[0], str(round(mean,1)), self.channels[0])]
690 elif data['mode_op'] == 'PPI':
665 elif data['mode_op'] == 'PPI':
691 len_aux = int(data['ele'].shape[0]/4)
666 len_aux = int(data['ele'].shape[0]/4)
692 mean = numpy.mean(data['ele'][len_aux:-len_aux])
667 mean = numpy.mean(data['ele'][len_aux:-len_aux])
693 if len(self.channels) !=1:
668 if len(self.channels) !=1:
694 self.titles = ['PPI {} at EL: {} CH {}'.format(self.labels[x], str(round(mean,1)), x) for x in self.channels]
669 self.titles = ['PPI {} at EL: {} CH {}'.format(self.labels[x], str(round(mean,1)), x) for x in self.channels]
695 else:
670 else:
696 self.titles = ['PPI {} at EL: {} CH {}'.format(self.labels[0], str(round(mean,1)), self.channels[0])]
671 self.titles = ['PPI {} at EL: {} CH {}'.format(self.labels[0], str(round(mean,1)), self.channels[0])]
697 self.mode_value = round(mean,1) No newline at end of file
672 self.mode_value = round(mean,1)
673
674 if data['mode_op'] == 'PPI':
675 gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True,
676 linewidth=1, color='gray', alpha=0.5, linestyle='--')
677 gl.xlabel_style = {'size': 8}
678 gl.ylabel_style = {'size': 8}
679 gl.xlabels_top = False
680 gl.ylabels_right = False
681 shape_p = os.path.join(self.shapes,'PER_ADM2/PER_ADM2.shp')
682 shape_d = os.path.join(self.shapes,'PER_ADM1/PER_ADM1.shp')
683 capitales = os.path.join(self.shapes,'CAPITALES/cap_provincia.shp')
684 vias = os.path.join(self.shapes,'Carreteras/VIAS_NACIONAL_250000.shp')
685 reader_d = shpreader.BasicReader(shape_p, encoding='latin1')
686 reader_p = shpreader.BasicReader(shape_d, encoding='latin1')
687 reader_c = shpreader.BasicReader(capitales, encoding='latin1')
688 reader_v = shpreader.BasicReader(vias, encoding='latin1')
689 caps = [x for x in reader_c.records() if x.attributes["Departa"] in ("JUNIN", "LIMA", "AYACUCHO", "HUANCAVELICA")]
690 districts = [x for x in reader_d.records() if x.attributes["Name"] in ("JUNÍN", "CHANCHAMAYO", "CHUPACA", "CONCEPCIÓN", "HUANCAYO", "JAUJA", "SATIPO", "TARMA", "YAUYOS", "HUAROCHIRÍ", "CANTA", "HUANTA", "TAYACAJA")]
691 provs = [x for x in reader_p.records() if x.attributes["NAME"] in ("Junín", "Lima")]
692 vias = [x for x in reader_v.records() if x.attributes["DEP"] in ("JUNIN", "LIMA")]
693
694 # Display Kenya's shape
695 shape_feature = ShapelyFeature([x.geometry for x in districts], ccrs.PlateCarree(), facecolor="none", edgecolor='grey', lw=0.5)
696 ax.add_feature(shape_feature)
697 shape_feature = ShapelyFeature([x.geometry for x in provs], ccrs.PlateCarree(), facecolor="none", edgecolor='white', lw=1)
698 ax.add_feature(shape_feature)
699 shape_feature = ShapelyFeature([x.geometry for x in vias], ccrs.PlateCarree(), facecolor="none", edgecolor='yellow', lw=1)
700 ax.add_feature(shape_feature)
701
702 for cap in caps:
703 if cap.attributes['Nombre'] in ("LA OROYA", "CONCEPCIÓN", "HUANCAYO", "JAUJA", "CHUPACA", "YAUYOS", "HUANTA", "PAMPAS"):
704 ax.text(cap.attributes['X'], cap.attributes['Y'], cap.attributes['Nombre'].title(), size=7, color='white')
705 ax.text(-75.052003, -11.915552, 'Huaytapallana', size=7, color='cyan')
706 ax.plot(-75.052003, -11.915552, '*')
707
708 for R in (10, 20, 30 , 40, 50):
709 circle = Circle((-75.295893, -12.040436), km2deg(R), facecolor='none',
710 edgecolor='skyblue', linewidth=1, alpha=0.5)
711 ax.add_patch(circle)
712 ax.text(km2deg(R)*numpy.cos(numpy.radians(45))-75.295893,
713 km2deg(R)*numpy.sin(numpy.radians(45))-12.040436,
714 '{}km'.format(R), color='skyblue', size=7)
715 elif data['mode_op'] == 'RHI':
716 ax.grid(color='grey', alpha=0.5, linestyle='--', linewidth=1)
Show file before | Show file after | Hide comments
@@ -1,733 +1,739
1 from email.utils import localtime
1 from email.utils import localtime
2 import os
2 import os
3 import time
3 import time
4 import datetime
4 import datetime
5
5
6 import numpy
6 import numpy
7 import h5py
7 import h5py
8
8
9 import schainpy.admin
9 import schainpy.admin
10 from schainpy.model.data.jrodata import *
10 from schainpy.model.data.jrodata import *
11 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
12 from schainpy.model.io.jroIO_base import *
12 from schainpy.model.io.jroIO_base import *
13 from schainpy.utils import log
13 from schainpy.utils import log
14
14
15
15
16 class HDFReader(Reader, ProcessingUnit):
16 class HDFReader(Reader, ProcessingUnit):
17 """Processing unit to read HDF5 format files
17 """Processing unit to read HDF5 format files
18
18
19 This unit reads HDF5 files created with `HDFWriter` operation contains
19 This unit reads HDF5 files created with `HDFWriter` operation contains
20 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 by default two groups Data and Metadata all variables would be saved as `dataOut`
21 attributes.
21 attributes.
22 It is possible to read any HDF5 file by given the structure in the `description`
22 It is possible to read any HDF5 file by given the structure in the `description`
23 parameter, also you can add extra values to metadata with the parameter `extras`.
23 parameter, also you can add extra values to metadata with the parameter `extras`.
24
24
25 Parameters:
25 Parameters:
26 -----------
26 -----------
27 path : str
27 path : str
28 Path where files are located.
28 Path where files are located.
29 startDate : date
29 startDate : date
30 Start date of the files
30 Start date of the files
31 endDate : list
31 endDate : list
32 End date of the files
32 End date of the files
33 startTime : time
33 startTime : time
34 Start time of the files
34 Start time of the files
35 endTime : time
35 endTime : time
36 End time of the files
36 End time of the files
37 description : dict, optional
37 description : dict, optional
38 Dictionary with the description of the HDF5 file
38 Dictionary with the description of the HDF5 file
39 extras : dict, optional
39 extras : dict, optional
40 Dictionary with extra metadata to be be added to `dataOut`
40 Dictionary with extra metadata to be be added to `dataOut`
41
41
42 Examples
42 Examples
43 --------
43 --------
44
44
45 desc = {
45 desc = {
46 'Data': {
46 'Data': {
47 'data_output': ['u', 'v', 'w'],
47 'data_output': ['u', 'v', 'w'],
48 'utctime': 'timestamps',
48 'utctime': 'timestamps',
49 } ,
49 } ,
50 'Metadata': {
50 'Metadata': {
51 'heightList': 'heights'
51 'heightList': 'heights'
52 }
52 }
53 }
53 }
54
54
55 desc = {
55 desc = {
56 'Data': {
56 'Data': {
57 'data_output': 'winds',
57 'data_output': 'winds',
58 'utctime': 'timestamps'
58 'utctime': 'timestamps'
59 },
59 },
60 'Metadata': {
60 'Metadata': {
61 'heightList': 'heights'
61 'heightList': 'heights'
62 }
62 }
63 }
63 }
64
64
65 extras = {
65 extras = {
66 'timeZone': 300
66 'timeZone': 300
67 }
67 }
68
68
69 reader = project.addReadUnit(
69 reader = project.addReadUnit(
70 name='HDFReader',
70 name='HDFReader',
71 path='/path/to/files',
71 path='/path/to/files',
72 startDate='2019/01/01',
72 startDate='2019/01/01',
73 endDate='2019/01/31',
73 endDate='2019/01/31',
74 startTime='00:00:00',
74 startTime='00:00:00',
75 endTime='23:59:59',
75 endTime='23:59:59',
76 # description=json.dumps(desc),
76 # description=json.dumps(desc),
77 # extras=json.dumps(extras),
77 # extras=json.dumps(extras),
78 )
78 )
79
79
80 """
80 """
81
81
82 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
83
83
84 def __init__(self):
84 def __init__(self):
85 ProcessingUnit.__init__(self)
85 ProcessingUnit.__init__(self)
86 self.dataOut = Parameters()
86 self.dataOut = Parameters()
87 self.ext = ".hdf5"
87 self.ext = ".hdf5"
88 self.optchar = "D"
88 self.optchar = "D"
89 self.meta = {}
89 self.meta = {}
90 self.data = {}
90 self.data = {}
91 self.open_file = h5py.File
91 self.open_file = h5py.File
92 self.open_mode = 'r'
92 self.open_mode = 'r'
93 self.description = {}
93 self.description = {}
94 self.extras = {}
94 self.extras = {}
95 self.filefmt = "*%Y%j***"
95 self.filefmt = "*%Y%j***"
96 self.folderfmt = "*%Y%j"
96 self.folderfmt = "*%Y%j"
97 self.utcoffset = 0
97 self.utcoffset = 0
98 self.filter = None
98 self.filter = None
99 self.dparam = None
99 self.dparam = None
100
100
101 def setup(self, **kwargs):
101 def setup(self, **kwargs):
102
102
103 self.set_kwargs(**kwargs)
103 self.set_kwargs(**kwargs)
104 if not self.ext.startswith('.'):
104 if not self.ext.startswith('.'):
105 self.ext = '.{}'.format(self.ext)
105 self.ext = '.{}'.format(self.ext)
106
106
107 if self.online:
107 if self.online:
108 log.log("Searching files in online mode...", self.name)
108 log.log("Searching files in online mode...", self.name)
109
109
110 for nTries in range(self.nTries):
110 for nTries in range(self.nTries):
111 fullpath = self.searchFilesOnLine(self.path, self.startDate,
111 fullpath = self.searchFilesOnLine(self.path, self.startDate,
112 self.endDate, self.expLabel, self.ext, self.walk,
112 self.endDate, self.expLabel, self.ext, self.walk,
113 self.filefmt, self.folderfmt,self.filter)
113 self.filefmt, self.folderfmt,self.filter)
114 try:
114 try:
115 fullpath = next(fullpath)
115 fullpath = next(fullpath)
116 except:
116 except:
117 fullpath = None
117 fullpath = None
118
118
119 if fullpath:
119 if fullpath:
120 break
120 break
121
121
122 log.warning(
122 log.warning(
123 'Waiting {} sec for a valid file in {}: try {} ...'.format(
123 'Waiting {} sec for a valid file in {}: try {} ...'.format(
124 self.delay, self.path, nTries + 1),
124 self.delay, self.path, nTries + 1),
125 self.name)
125 self.name)
126 time.sleep(self.delay)
126 time.sleep(self.delay)
127
127
128 if not(fullpath):
128 if not(fullpath):
129 raise schainpy.admin.SchainError(
129 raise schainpy.admin.SchainError(
130 'There isn\'t any valid file in {}'.format(self.path))
130 'There isn\'t any valid file in {}'.format(self.path))
131
131
132 pathname, filename = os.path.split(fullpath)
132 pathname, filename = os.path.split(fullpath)
133 self.year = int(filename[1:5])
133 self.year = int(filename[1:5])
134 self.doy = int(filename[5:8])
134 self.doy = int(filename[5:8])
135 self.set = int(filename[8:11]) - 1
135 self.set = int(filename[8:11]) - 1
136 else:
136 else:
137 log.log("Searching files in {}".format(self.path), self.name)
137 log.log("Searching files in {}".format(self.path), self.name)
138 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
138 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
139 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt,self.filter)
139 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt,self.filter)
140
140
141 self.setNextFile()
141 self.setNextFile()
142
142
143 return
143 return
144
144
145 def readFirstHeader(self):
145 def readFirstHeader(self):
146 '''Read metadata and data'''
146 '''Read metadata and data'''
147
147
148 self.__readMetadata()
148 self.__readMetadata()
149 self.__readData()
149 self.__readData()
150 self.__setBlockList()
150 self.__setBlockList()
151
151
152 if 'type' in self.meta:
152 if 'type' in self.meta:
153 self.dataOut = eval(self.meta['type'])()
153 self.dataOut = eval(self.meta['type'])()
154
154
155 if self.dparam:
155 if self.dparam:
156 setattr(self.dataOut, "dparam", 1)
156 setattr(self.dataOut, "dparam", 1)
157
157
158 for attr in self.meta:
158 for attr in self.meta:
159 setattr(self.dataOut, attr, self.meta[attr])
159 setattr(self.dataOut, attr, self.meta[attr])
160
160
161 self.blockIndex = 0
161 self.blockIndex = 0
162
162
163 return
163 return
164
164
165 def __setBlockList(self):
165 def __setBlockList(self):
166 '''
166 '''
167 Selects the data within the times defined
167 Selects the data within the times defined
168
168
169 self.fp
169 self.fp
170 self.startTime
170 self.startTime
171 self.endTime
171 self.endTime
172 self.blockList
172 self.blockList
173 self.blocksPerFile
173 self.blocksPerFile
174
174
175 '''
175 '''
176
176
177 startTime = self.startTime
177 startTime = self.startTime
178 endTime = self.endTime
178 endTime = self.endTime
179 thisUtcTime = self.data['utctime'] + self.utcoffset
179 thisUtcTime = self.data['utctime'] + self.utcoffset
180 try:
180 try:
181 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
181 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
182 except:
182 except:
183 self.interval = 0
183 self.interval = 0
184 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
184 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
185
185
186 thisDate = thisDatetime.date()
186 thisDate = thisDatetime.date()
187 thisTime = thisDatetime.time()
187 thisTime = thisDatetime.time()
188
188
189 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
189 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
190 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
190 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
191
191
192 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
192 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
193
193
194 self.blockList = ind
194 self.blockList = ind
195 self.blocksPerFile = len(ind)
195 self.blocksPerFile = len(ind)
196 return
196 return
197
197
198 def __readMetadata(self):
198 def __readMetadata(self):
199 '''
199 '''
200 Reads Metadata
200 Reads Metadata
201 '''
201 '''
202
202
203 meta = {}
203 meta = {}
204
204
205 if self.description:
205 if self.description:
206 for key, value in self.description['Metadata'].items():
206 for key, value in self.description['Metadata'].items():
207 meta[key] = self.fp[value][()]
207 meta[key] = self.fp[value][()]
208 else:
208 else:
209 grp = self.fp['Metadata']
209 grp = self.fp['Metadata']
210 for name in grp:
210 for name in grp:
211 meta[name] = grp[name][()]
211 meta[name] = grp[name][()]
212
212
213 if self.extras:
213 if self.extras:
214 for key, value in self.extras.items():
214 for key, value in self.extras.items():
215 meta[key] = value
215 meta[key] = value
216 self.meta = meta
216 self.meta = meta
217
217
218 return
218 return
219
219
220 def __readData(self):
220 def __readData(self):
221
221
222 data = {}
222 data = {}
223
223
224 if self.description:
224 if self.description:
225 for key, value in self.description['Data'].items():
225 for key, value in self.description['Data'].items():
226 if isinstance(value, str):
226 if isinstance(value, str):
227 if isinstance(self.fp[value], h5py.Dataset):
227 if isinstance(self.fp[value], h5py.Dataset):
228 data[key] = self.fp[value][()]
228 data[key] = self.fp[value][()]
229 elif isinstance(self.fp[value], h5py.Group):
229 elif isinstance(self.fp[value], h5py.Group):
230 array = []
230 array = []
231 for ch in self.fp[value]:
231 for ch in self.fp[value]:
232 array.append(self.fp[value][ch][()])
232 array.append(self.fp[value][ch][()])
233 data[key] = numpy.array(array)
233 data[key] = numpy.array(array)
234 elif isinstance(value, list):
234 elif isinstance(value, list):
235 array = []
235 array = []
236 for ch in value:
236 for ch in value:
237 array.append(self.fp[ch][()])
237 array.append(self.fp[ch][()])
238 data[key] = numpy.array(array)
238 data[key] = numpy.array(array)
239 else:
239 else:
240 grp = self.fp['Data']
240 grp = self.fp['Data']
241 for name in grp:
241 for name in grp:
242 if isinstance(grp[name], h5py.Dataset):
242 if isinstance(grp[name], h5py.Dataset):
243 array = grp[name][()]
243 array = grp[name][()]
244 elif isinstance(grp[name], h5py.Group):
244 elif isinstance(grp[name], h5py.Group):
245 array = []
245 array = []
246 for ch in grp[name]:
246 for ch in grp[name]:
247 array.append(grp[name][ch][()])
247 array.append(grp[name][ch][()])
248 array = numpy.array(array)
248 array = numpy.array(array)
249 else:
249 else:
250 log.warning('Unknown type: {}'.format(name))
250 log.warning('Unknown type: {}'.format(name))
251
251
252 if name in self.description:
252 if name in self.description:
253 key = self.description[name]
253 key = self.description[name]
254 else:
254 else:
255 key = name
255 key = name
256 data[key] = array
256 data[key] = array
257
257
258 self.data = data
258 self.data = data
259 return
259 return
260
260
261 def getData(self):
261 def getData(self):
262
262
263 for attr in self.data:
263 for attr in self.data:
264 if self.data[attr].ndim == 1:
264 if self.data[attr].ndim == 1:
265 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
265 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
266 else:
266 else:
267 if self.dparam:
267 if self.dparam:
268 setattr(self.dataOut, attr, self.data[attr])
268 setattr(self.dataOut, attr, self.data[attr])
269 else:
269 else:
270 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
270 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
271
271
272 self.dataOut.flagNoData = False
272 self.dataOut.flagNoData = False
273 self.blockIndex += 1
273 self.blockIndex += 1
274
274
275 log.log("Block No. {}/{} -> {}".format(
275 log.log("Block No. {}/{} -> {}".format(
276 self.blockIndex,
276 self.blockIndex,
277 self.blocksPerFile,
277 self.blocksPerFile,
278 self.dataOut.datatime.ctime()), self.name)
278 self.dataOut.datatime.ctime()), self.name)
279
279
280 return
280 return
281
281
282 def run(self, **kwargs):
282 def run(self, **kwargs):
283
283
284 if not(self.isConfig):
284 if not(self.isConfig):
285 self.setup(**kwargs)
285 self.setup(**kwargs)
286 self.isConfig = True
286 self.isConfig = True
287
287
288 if self.blockIndex == self.blocksPerFile:
288 if self.blockIndex == self.blocksPerFile:
289 self.setNextFile()
289 self.setNextFile()
290
290
291 self.getData()
291 self.getData()
292
292
293 return
293 return
294
294
295 @MPDecorator
295 @MPDecorator
296 class HDFWriter(Operation):
296 class HDFWriter(Operation):
297 """Operation to write HDF5 files.
297 """Operation to write HDF5 files.
298
298
299 The HDF5 file contains by default two groups Data and Metadata where
299 The HDF5 file contains by default two groups Data and Metadata where
300 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
300 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
301 parameters, data attributes are normaly time dependent where the metadata
301 parameters, data attributes are normaly time dependent where the metadata
302 are not.
302 are not.
303 It is possible to customize the structure of the HDF5 file with the
303 It is possible to customize the structure of the HDF5 file with the
304 optional description parameter see the examples.
304 optional description parameter see the examples.
305
305
306 Parameters:
306 Parameters:
307 -----------
307 -----------
308 path : str
308 path : str
309 Path where files will be saved.
309 Path where files will be saved.
310 blocksPerFile : int
310 blocksPerFile : int
311 Number of blocks per file
311 Number of blocks per file
312 metadataList : list
312 metadataList : list
313 List of the dataOut attributes that will be saved as metadata
313 List of the dataOut attributes that will be saved as metadata
314 dataList : int
314 dataList : int
315 List of the dataOut attributes that will be saved as data
315 List of the dataOut attributes that will be saved as data
316 setType : bool
316 setType : bool
317 If True the name of the files corresponds to the timestamp of the data
317 If True the name of the files corresponds to the timestamp of the data
318 description : dict, optional
318 description : dict, optional
319 Dictionary with the desired description of the HDF5 file
319 Dictionary with the desired description of the HDF5 file
320
320
321 Examples
321 Examples
322 --------
322 --------
323
323
324 desc = {
324 desc = {
325 'data_output': {'winds': ['z', 'w', 'v']},
325 'data_output': {'winds': ['z', 'w', 'v']},
326 'utctime': 'timestamps',
326 'utctime': 'timestamps',
327 'heightList': 'heights'
327 'heightList': 'heights'
328 }
328 }
329 desc = {
329 desc = {
330 'data_output': ['z', 'w', 'v'],
330 'data_output': ['z', 'w', 'v'],
331 'utctime': 'timestamps',
331 'utctime': 'timestamps',
332 'heightList': 'heights'
332 'heightList': 'heights'
333 }
333 }
334 desc = {
334 desc = {
335 'Data': {
335 'Data': {
336 'data_output': 'winds',
336 'data_output': 'winds',
337 'utctime': 'timestamps'
337 'utctime': 'timestamps'
338 },
338 },
339 'Metadata': {
339 'Metadata': {
340 'heightList': 'heights'
340 'heightList': 'heights'
341 }
341 }
342 }
342 }
343
343
344 writer = proc_unit.addOperation(name='HDFWriter')
344 writer = proc_unit.addOperation(name='HDFWriter')
345 writer.addParameter(name='path', value='/path/to/file')
345 writer.addParameter(name='path', value='/path/to/file')
346 writer.addParameter(name='blocksPerFile', value='32')
346 writer.addParameter(name='blocksPerFile', value='32')
347 writer.addParameter(name='metadataList', value='heightList,timeZone')
347 writer.addParameter(name='metadataList', value='heightList,timeZone')
348 writer.addParameter(name='dataList',value='data_output,utctime')
348 writer.addParameter(name='dataList',value='data_output,utctime')
349 # writer.addParameter(name='description',value=json.dumps(desc))
349 # writer.addParameter(name='description',value=json.dumps(desc))
350
350
351 """
351 """
352
352
353 ext = ".hdf5"
353 ext = ".hdf5"
354 optchar = "D"
354 optchar = "D"
355 filename = None
355 filename = None
356 path = None
356 path = None
357 setFile = None
357 setFile = None
358 fp = None
358 fp = None
359 firsttime = True
359 firsttime = True
360 #Configurations
360 #Configurations
361 blocksPerFile = None
361 blocksPerFile = None
362 blockIndex = None
362 blockIndex = None
363 dataOut = None
363 dataOut = None
364 #Data Arrays
364 #Data Arrays
365 dataList = None
365 dataList = None
366 metadataList = None
366 metadataList = None
367 currentDay = None
367 currentDay = None
368 lastTime = None
368 lastTime = None
369 last_Azipos = None
369 last_Azipos = None
370 last_Elepos = None
370 last_Elepos = None
371 mode = None
371 mode = None
372 #-----------------------
372 #-----------------------
373 Typename = None
373 Typename = None
374 mask = False
374 mask = False
375
375
376 def __init__(self):
376 def __init__(self):
377
377
378 Operation.__init__(self)
378 Operation.__init__(self)
379 return
379 return
380
380
381 def set_kwargs(self, **kwargs):
381 def set_kwargs(self, **kwargs):
382
382
383 for key, value in kwargs.items():
383 for key, value in kwargs.items():
384 setattr(self, key, value)
384 setattr(self, key, value)
385
385
386 def set_kwargs_obj(self,obj, **kwargs):
386 def set_kwargs_obj(self,obj, **kwargs):
387
387
388 for key, value in kwargs.items():
388 for key, value in kwargs.items():
389 setattr(obj, key, value)
389 setattr(obj, key, value)
390
390
391 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None,type_data=None, localtime=True, **kwargs):
391 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None,type_data=None, localtime=True, **kwargs):
392 self.path = path
392 self.path = path
393 self.blocksPerFile = blocksPerFile
393 self.blocksPerFile = blocksPerFile
394 self.metadataList = metadataList
394 self.metadataList = metadataList
395 self.dataList = [s.strip() for s in dataList]
395 self.dataList = [s.strip() for s in dataList]
396 self.setType = setType
396 self.setType = setType
397 if self.setType == "weather":
397 if self.setType == "weather":
398 self.set_kwargs(**kwargs)
398 self.set_kwargs(**kwargs)
399 self.set_kwargs_obj(self.dataOut,**kwargs)
399 self.set_kwargs_obj(self.dataOut,**kwargs)
400 self.weather_vars = {
400 self.weather_vars = {
401 'S' : 0,
401 'S' : 0,
402 'V' : 1,
402 'V' : 1,
403 'W' : 2,
403 'W' : 2,
404 'SNR' : 3,
404 'SNR' : 3,
405 'Z' : 4,
405 'Z' : 4,
406 'D' : 5,
406 'D' : 5,
407 'P' : 6,
407 'P' : 6,
408 'R' : 7,
408 'R' : 7,
409 }
409 }
410
410
411 if localtime:
411 if localtime:
412 self.getDateTime = datetime.datetime.fromtimestamp
412 self.getDateTime = datetime.datetime.fromtimestamp
413 else:
413 else:
414 self.getDateTime = datetime.datetime.utcfromtimestamp
414 self.getDateTime = datetime.datetime.utcfromtimestamp
415
415
416 self.description = description
416 self.description = description
417 self.type_data=type_data
417 self.type_data=type_data
418
418
419 if self.metadataList is None:
419 if self.metadataList is None:
420 self.metadataList = self.dataOut.metadata_list
420 self.metadataList = self.dataOut.metadata_list
421
421
422 dsList = []
422 dsList = []
423
423
424 for i in range(len(self.dataList)):
424 for i in range(len(self.dataList)):
425 dsDict = {}
425 dsDict = {}
426 if hasattr(self.dataOut, self.dataList[i]):
426 if hasattr(self.dataOut, self.dataList[i]):
427 dataAux = getattr(self.dataOut, self.dataList[i])
427 dataAux = getattr(self.dataOut, self.dataList[i])
428 if self.setType == 'weather' and self.dataList[i] == 'data_param':
428 if self.setType == 'weather' and self.dataList[i] == 'data_param':
429 dataAux = dataAux[:,self.weather_vars[self.weather_var],:]
429 dataAux = dataAux[:,self.weather_vars[self.weather_var],:]
430 dsDict['variable'] = self.dataList[i]
430 dsDict['variable'] = self.dataList[i]
431 else:
431 else:
432 log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]), self.name)
432 log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]), self.name)
433 continue
433 continue
434
434
435 if dataAux is None:
435 if dataAux is None:
436 continue
436 continue
437 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
437 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
438 dsDict['nDim'] = 0
438 dsDict['nDim'] = 0
439 else:
439 else:
440 dsDict['nDim'] = len(dataAux.shape)
440 dsDict['nDim'] = len(dataAux.shape)
441 dsDict['shape'] = dataAux.shape
441 dsDict['shape'] = dataAux.shape
442 dsDict['dsNumber'] = dataAux.shape[0]
442 dsDict['dsNumber'] = dataAux.shape[0]
443 dsDict['dtype'] = dataAux.dtype
443 dsDict['dtype'] = dataAux.dtype
444 dsList.append(dsDict)
444 dsList.append(dsDict)
445
445
446 self.dsList = dsList
446 self.dsList = dsList
447 self.currentDay = self.dataOut.datatime.date()
447 self.currentDay = self.dataOut.datatime.date()
448
448
449 def timeFlag(self):
449 def timeFlag(self):
450 currentTime = self.dataOut.utctime
450 currentTime = self.dataOut.utctime
451 dt = self.getDateTime(currentTime)
451 dt = self.getDateTime(currentTime)
452
452
453 dataDay = int(dt.strftime('%j'))
453 dataDay = int(dt.strftime('%j'))
454
454
455 if self.lastTime is None:
455 if self.lastTime is None:
456 self.lastTime = currentTime
456 self.lastTime = currentTime
457 self.currentDay = dataDay
457 self.currentDay = dataDay
458 return False
458 return False
459
459
460 timeDiff = currentTime - self.lastTime
460 timeDiff = currentTime - self.lastTime
461
461
462 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
462 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
463 if dataDay != self.currentDay:
463 if dataDay != self.currentDay:
464 self.currentDay = dataDay
464 self.currentDay = dataDay
465 return True
465 return True
466 elif timeDiff > 3*60*60:
466 elif timeDiff > 3*60*60:
467 self.lastTime = currentTime
467 self.lastTime = currentTime
468 return True
468 return True
469 else:
469 else:
470 self.lastTime = currentTime
470 self.lastTime = currentTime
471 return False
471 return False
472
472
473 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
473 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
474 dataList=[], setType=None, description={}, mode= None,
474 dataList=[], setType=None, description={}, mode= None,
475 type_data=None, Reset = False, localtime=True, **kwargs):
475 type_data=None, Reset = False, localtime=True, **kwargs):
476
476
477 if Reset:
477 if Reset:
478 self.isConfig = False
478 self.isConfig = False
479 self.closeFile()
479 self.closeFile()
480 self.lastTime = None
480 self.lastTime = None
481 self.blockIndex = 0
481 self.blockIndex = 0
482
482
483 self.dataOut = dataOut
483 self.dataOut = dataOut
484 self.mode = mode
484 self.mode = mode
485
485
486 if not(self.isConfig):
486 if not(self.isConfig):
487 self.setup(path=path, blocksPerFile=blocksPerFile,
487 self.setup(path=path, blocksPerFile=blocksPerFile,
488 metadataList=metadataList, dataList=dataList,
488 metadataList=metadataList, dataList=dataList,
489 setType=setType, description=description,type_data=type_data,
489 setType=setType, description=description,type_data=type_data,
490 localtime=localtime, **kwargs)
490 localtime=localtime, **kwargs)
491
491
492 self.isConfig = True
492 self.isConfig = True
493 self.setNextFile()
493 self.setNextFile()
494
494
495 self.putData()
495 self.putData()
496 return
496 return
497
497
498 def setNextFile(self):
498 def setNextFile(self):
499
499
500 ext = self.ext
500 ext = self.ext
501 path = self.path
501 path = self.path
502 setFile = self.setFile
502 setFile = self.setFile
503
503
504 dt = self.getDateTime(self.dataOut.utctime)
504 dt = self.getDateTime(self.dataOut.utctime)
505
505
506 if self.setType == 'weather':
506 if self.setType == 'weather':
507 subfolder = dt.strftime('%Y-%m-%dT%H-00-00')
507 subfolder = dt.strftime('%Y-%m-%dT%H-00-00')
508 subfolder = ''
508 else:
509 else:
509 subfolder = dt.strftime('d%Y%j')
510 subfolder = dt.strftime('d%Y%j')
510
511
511 fullpath = os.path.join(path, subfolder)
512 fullpath = os.path.join(path, subfolder)
512
513
513 if os.path.exists(fullpath):
514 if os.path.exists(fullpath):
514 filesList = os.listdir(fullpath)
515 filesList = os.listdir(fullpath)
515 filesList = [k for k in filesList if k.startswith(self.optchar)]
516 filesList = [k for k in filesList if k.startswith(self.optchar)]
516 if len( filesList ) > 0:
517 if len( filesList ) > 0:
517 filesList = sorted(filesList, key=str.lower)
518 filesList = sorted(filesList, key=str.lower)
518 filen = filesList[-1]
519 filen = filesList[-1]
519 # el filename debera tener el siguiente formato
520 # el filename debera tener el siguiente formato
520 # 0 1234 567 89A BCDE (hex)
521 # 0 1234 567 89A BCDE (hex)
521 # x YYYY DDD SSS .ext
522 # x YYYY DDD SSS .ext
522 if isNumber(filen[8:11]):
523 if isNumber(filen[8:11]):
523 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
524 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
524 else:
525 else:
525 setFile = -1
526 setFile = -1
526 else:
527 else:
527 setFile = -1 #inicializo mi contador de seteo
528 setFile = -1 #inicializo mi contador de seteo
528 else:
529 else:
529 os.makedirs(fullpath)
530 os.makedirs(fullpath)
530 setFile = -1 #inicializo mi contador de seteo
531 setFile = -1 #inicializo mi contador de seteo
531
532
532 if self.setType is None:
533 if self.setType is None:
533 setFile += 1
534 setFile += 1
534 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
535 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
535 dt.year,
536 dt.year,
536 int(dt.strftime('%j')),
537 int(dt.strftime('%j')),
537 setFile,
538 setFile,
538 ext )
539 ext )
539 elif self.setType == "weather":
540 elif self.setType == "weather":
540
541
541 #SOPHY_20200505_140215_E10.0_Z.h5
542 #SOPHY_20200505_140215_E10.0_Z.h5
542 #SOPHY_20200505_140215_A40.0_Z.h5
543 #SOPHY_20200505_140215_A40.0_Z.h5
543 if self.dataOut.flagMode == 1: #'AZI' #PPI
544 if self.dataOut.flagMode == 1: #'AZI' #PPI
544 ang_type = 'E'
545 ang_type = 'EL'
546 mode_type = 'PPI'
545 len_aux = int(self.dataOut.data_ele.shape[0]/4)
547 len_aux = int(self.dataOut.data_ele.shape[0]/4)
546 mean = numpy.mean(self.dataOut.data_ele[len_aux:-len_aux])
548 mean = numpy.mean(self.dataOut.data_ele[len_aux:-len_aux])
547 ang_ = round(mean,1)
549 ang_ = round(mean,1)
548 elif self.dataOut.flagMode == 0: #'ELE' #RHI
550 elif self.dataOut.flagMode == 0: #'ELE' #RHI
549 ang_type = 'A'
551 ang_type = 'AZ'
552 mode_type = 'RHI'
550 len_aux = int(self.dataOut.data_azi.shape[0]/4)
553 len_aux = int(self.dataOut.data_azi.shape[0]/4)
551 mean = numpy.mean(self.dataOut.data_azi[len_aux:-len_aux])
554 mean = numpy.mean(self.dataOut.data_azi[len_aux:-len_aux])
552 ang_ = round(mean,1)
555 ang_ = round(mean,1)
553
556
554 file = '%s_%2.2d%2.2d%2.2d_%2.2d%2.2d%2.2d_%s%2.1f_%s%s' % (
557 file = '%s_%2.2d%2.2d%2.2d_%2.2d%2.2d%2.2d_%s%2.1f_%s%s' % (
555 'SOPHY',
558 'SOPHY',
556 dt.year,
559 dt.year,
557 dt.month,
560 dt.month,
558 dt.day,
561 dt.day,
559 dt.hour,
562 dt.hour,
560 dt.minute,
563 dt.minute,
561 dt.second,
564 dt.second,
562 ang_type,
565 ang_type[0],
563 ang_,
566 ang_,
564 self.weather_var,
567 self.weather_var,
565 ext )
568 ext )
566
569 subfolder = '{}_{}_{}_{:2.1f}'.format(self.weather_var, mode_type, ang_type, ang_)
570 fullpath = os.path.join(path, subfolder)
571 if not os.path.exists(fullpath):
572 os.makedirs(fullpath)
567 else:
573 else:
568 setFile = dt.hour*60+dt.minute
574 setFile = dt.hour*60+dt.minute
569 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
575 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
570 dt.year,
576 dt.year,
571 int(dt.strftime('%j')),
577 int(dt.strftime('%j')),
572 setFile,
578 setFile,
573 ext )
579 ext )
574
580
575 self.filename = os.path.join( path, subfolder, file )
581 self.filename = os.path.join( path, subfolder, file )
576
582
577 self.fp = h5py.File(self.filename, 'w')
583 self.fp = h5py.File(self.filename, 'w')
578 #write metadata
584 #write metadata
579 self.writeMetadata(self.fp)
585 self.writeMetadata(self.fp)
580 #Write data
586 #Write data
581 self.writeData(self.fp)
587 self.writeData(self.fp)
582
588
583 def getLabel(self, name, x=None):
589 def getLabel(self, name, x=None):
584
590
585 if x is None:
591 if x is None:
586 if 'Data' in self.description:
592 if 'Data' in self.description:
587 data = self.description['Data']
593 data = self.description['Data']
588 if 'Metadata' in self.description:
594 if 'Metadata' in self.description:
589 data.update(self.description['Metadata'])
595 data.update(self.description['Metadata'])
590 else:
596 else:
591 data = self.description
597 data = self.description
592 if name in data:
598 if name in data:
593 if isinstance(data[name], str):
599 if isinstance(data[name], str):
594 return data[name]
600 return data[name]
595 elif isinstance(data[name], list):
601 elif isinstance(data[name], list):
596 return None
602 return None
597 elif isinstance(data[name], dict):
603 elif isinstance(data[name], dict):
598 for key, value in data[name].items():
604 for key, value in data[name].items():
599 return key
605 return key
600 return name
606 return name
601 else:
607 else:
602 if 'Data' in self.description:
608 if 'Data' in self.description:
603 data = self.description['Data']
609 data = self.description['Data']
604 if 'Metadata' in self.description:
610 if 'Metadata' in self.description:
605 data.update(self.description['Metadata'])
611 data.update(self.description['Metadata'])
606 else:
612 else:
607 data = self.description
613 data = self.description
608 if name in data:
614 if name in data:
609 if isinstance(data[name], list):
615 if isinstance(data[name], list):
610 return data[name][x]
616 return data[name][x]
611 elif isinstance(data[name], dict):
617 elif isinstance(data[name], dict):
612 for key, value in data[name].items():
618 for key, value in data[name].items():
613 return value[x]
619 return value[x]
614 if 'cspc' in name:
620 if 'cspc' in name:
615 return 'pair{:02d}'.format(x)
621 return 'pair{:02d}'.format(x)
616 else:
622 else:
617 return 'channel{:02d}'.format(x)
623 return 'channel{:02d}'.format(x)
618
624
619 def writeMetadata(self, fp):
625 def writeMetadata(self, fp):
620
626
621 if self.description:
627 if self.description:
622 if 'Metadata' in self.description:
628 if 'Metadata' in self.description:
623 grp = fp.create_group('Metadata')
629 grp = fp.create_group('Metadata')
624 else:
630 else:
625 grp = fp
631 grp = fp
626 else:
632 else:
627 grp = fp.create_group('Metadata')
633 grp = fp.create_group('Metadata')
628
634
629 for i in range(len(self.metadataList)):
635 for i in range(len(self.metadataList)):
630 if not hasattr(self.dataOut, self.metadataList[i]):
636 if not hasattr(self.dataOut, self.metadataList[i]):
631 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
637 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
632 continue
638 continue
633 value = getattr(self.dataOut, self.metadataList[i])
639 value = getattr(self.dataOut, self.metadataList[i])
634 if isinstance(value, bool):
640 if isinstance(value, bool):
635 if value is True:
641 if value is True:
636 value = 1
642 value = 1
637 else:
643 else:
638 value = 0
644 value = 0
639 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
645 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
640 return
646 return
641
647
642 def writeData(self, fp):
648 def writeData(self, fp):
643
649
644 if self.description:
650 if self.description:
645 if 'Data' in self.description:
651 if 'Data' in self.description:
646 grp = fp.create_group('Data')
652 grp = fp.create_group('Data')
647 else:
653 else:
648 grp = fp
654 grp = fp
649 else:
655 else:
650 grp = fp.create_group('Data')
656 grp = fp.create_group('Data')
651
657
652 dtsets = []
658 dtsets = []
653 data = []
659 data = []
654
660
655 for dsInfo in self.dsList:
661 for dsInfo in self.dsList:
656
662
657 if dsInfo['nDim'] == 0:
663 if dsInfo['nDim'] == 0:
658 ds = grp.create_dataset(
664 ds = grp.create_dataset(
659 self.getLabel(dsInfo['variable']),
665 self.getLabel(dsInfo['variable']),
660 (self.blocksPerFile, ),
666 (self.blocksPerFile, ),
661 chunks=True,
667 chunks=True,
662 dtype=numpy.float64)
668 dtype=numpy.float64)
663 dtsets.append(ds)
669 dtsets.append(ds)
664 data.append((dsInfo['variable'], -1))
670 data.append((dsInfo['variable'], -1))
665 else:
671 else:
666 label = self.getLabel(dsInfo['variable'])
672 label = self.getLabel(dsInfo['variable'])
667 if label is not None:
673 if label is not None:
668 sgrp = grp.create_group(label)
674 sgrp = grp.create_group(label)
669 else:
675 else:
670 sgrp = grp
676 sgrp = grp
671 if self.blocksPerFile == 1:
677 if self.blocksPerFile == 1:
672 shape = dsInfo['shape'][1:]
678 shape = dsInfo['shape'][1:]
673 else:
679 else:
674 shape = (self.blocksPerFile, ) + dsInfo['shape'][1:]
680 shape = (self.blocksPerFile, ) + dsInfo['shape'][1:]
675 for i in range(dsInfo['dsNumber']):
681 for i in range(dsInfo['dsNumber']):
676 ds = sgrp.create_dataset(
682 ds = sgrp.create_dataset(
677 self.getLabel(dsInfo['variable'], i),
683 self.getLabel(dsInfo['variable'], i),
678 shape,
684 shape,
679 chunks=True,
685 chunks=True,
680 dtype=dsInfo['dtype'],
686 dtype=dsInfo['dtype'],
681 compression='gzip',
687 compression='gzip',
682 )
688 )
683 dtsets.append(ds)
689 dtsets.append(ds)
684 data.append((dsInfo['variable'], i))
690 data.append((dsInfo['variable'], i))
685 fp.flush()
691 fp.flush()
686
692
687 log.log('Creating file: {}'.format(fp.filename), self.name)
693 log.log('Creating file: {}'.format(fp.filename), self.name)
688
694
689 self.ds = dtsets
695 self.ds = dtsets
690 self.data = data
696 self.data = data
691 self.firsttime = True
697 self.firsttime = True
692 self.blockIndex = 0
698 self.blockIndex = 0
693 return
699 return
694
700
695 def putData(self):
701 def putData(self):
696
702
697 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
703 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
698 self.closeFile()
704 self.closeFile()
699 self.setNextFile()
705 self.setNextFile()
700
706
701 for i, ds in enumerate(self.ds):
707 for i, ds in enumerate(self.ds):
702 attr, ch = self.data[i]
708 attr, ch = self.data[i]
703 if ch == -1:
709 if ch == -1:
704 ds[self.blockIndex] = getattr(self.dataOut, attr)
710 ds[self.blockIndex] = getattr(self.dataOut, attr)
705 else:
711 else:
706 if self.blocksPerFile == 1:
712 if self.blocksPerFile == 1:
707 mask = self.dataOut.data_param[:,3,:][ch] < self.mask
713 mask = self.dataOut.data_param[:,3,:][ch] < self.mask
708 tmp = getattr(self.dataOut, attr)[:,self.weather_vars[self.weather_var],:][ch]
714 tmp = getattr(self.dataOut, attr)[:,self.weather_vars[self.weather_var],:][ch]
709 if self.mask:
715 if self.mask:
710 tmp[mask] = numpy.nan
716 tmp[mask] = numpy.nan
711 ds[:] = tmp
717 ds[:] = tmp
712 else:
718 else:
713 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
719 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
714
720
715 self.fp.flush()
721 self.fp.flush()
716 self.blockIndex += 1
722 self.blockIndex += 1
717 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
723 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
718
724
719 return
725 return
720
726
721 def closeFile(self):
727 def closeFile(self):
722
728
723 if self.blockIndex != self.blocksPerFile:
729 if self.blockIndex != self.blocksPerFile:
724 for ds in self.ds:
730 for ds in self.ds:
725 ds.resize(self.blockIndex, axis=0)
731 ds.resize(self.blockIndex, axis=0)
726
732
727 if self.fp:
733 if self.fp:
728 self.fp.flush()
734 self.fp.flush()
729 self.fp.close()
735 self.fp.close()
730
736
731 def close(self):
737 def close(self):
732
738
733 self.closeFile()
739 self.closeFile()
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now