##// END OF EJS Templates
schain-jc
RSS
New plotting architecture with buffering/throttle capabilities
Juan C. Espinoza -
r1187:66a3db7e736d
parent child
Show More
Side by Side Unified
Context file:
r1187:66a3db7e736d Loading diff...:
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (803 lines changed) Show them Hide them
@@ -0,0 +1,803
1
2 import os
3 import sys
4 import zmq
5 import time
6 import datetime
7 from functools import wraps
8 import numpy
9 import matplotlib
10
11 if 'BACKEND' in os.environ:
12 matplotlib.use(os.environ['BACKEND'])
13 elif 'linux' in sys.platform:
14 matplotlib.use("TkAgg")
15 elif 'darwin' in sys.platform:
16 matplotlib.use('TkAgg')
17 else:
18 from schainpy.utils import log
19 log.warning('Using default Backend="Agg"', 'INFO')
20 matplotlib.use('Agg')
21
22 import matplotlib.pyplot as plt
23 from matplotlib.patches import Polygon
24 from mpl_toolkits.axes_grid1 import make_axes_locatable
25 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
26
27 from schainpy.model.data.jrodata import PlotterData
28 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
29 from schainpy.utils import log
30
31 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
32 blu_values = matplotlib.pyplot.get_cmap(
33 'seismic_r', 20)(numpy.arange(20))[10:15]
34 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
35 'jro', numpy.vstack((blu_values, jet_values)))
36 matplotlib.pyplot.register_cmap(cmap=ncmap)
37
38 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis',
39 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
40
41 EARTH_RADIUS = 6.3710e3
42
43
44 def ll2xy(lat1, lon1, lat2, lon2):
45
46 p = 0.017453292519943295
47 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
48 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
49 r = 12742 * numpy.arcsin(numpy.sqrt(a))
50 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
51 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
52 theta = -theta + numpy.pi/2
53 return r*numpy.cos(theta), r*numpy.sin(theta)
54
55
56 def km2deg(km):
57 '''
58 Convert distance in km to degrees
59 '''
60
61 return numpy.rad2deg(km/EARTH_RADIUS)
62
63
64 def figpause(interval):
65 backend = plt.rcParams['backend']
66 if backend in matplotlib.rcsetup.interactive_bk:
67 figManager = matplotlib._pylab_helpers.Gcf.get_active()
68 if figManager is not None:
69 canvas = figManager.canvas
70 if canvas.figure.stale:
71 canvas.draw()
72 try:
73 canvas.start_event_loop(interval)
74 except:
75 pass
76 return
77
78
79 def popup(message):
80 '''
81 '''
82
83 fig = plt.figure(figsize=(12, 8), facecolor='r')
84 text = '\n'.join([s.strip() for s in message.split(':')])
85 fig.text(0.01, 0.5, text, ha='left', va='center',
86 size='20', weight='heavy', color='w')
87 fig.show()
88 figpause(1000)
89
90
91 class Throttle(object):
92 '''
93 Decorator that prevents a function from being called more than once every
94 time period.
95 To create a function that cannot be called more than once a minute, but
96 will sleep until it can be called:
97 @Throttle(minutes=1)
98 def foo():
99 pass
100
101 for i in range(10):
102 foo()
103 print "This function has run %s times." % i
104 '''
105
106 def __init__(self, seconds=0, minutes=0, hours=0):
107 self.throttle_period = datetime.timedelta(
108 seconds=seconds, minutes=minutes, hours=hours
109 )
110
111 self.time_of_last_call = datetime.datetime.min
112
113 def __call__(self, fn):
114 @wraps(fn)
115 def wrapper(*args, **kwargs):
116 coerce = kwargs.pop('coerce', None)
117 if coerce:
118 self.time_of_last_call = datetime.datetime.now()
119 return fn(*args, **kwargs)
120 else:
121 now = datetime.datetime.now()
122 time_since_last_call = now - self.time_of_last_call
123 time_left = self.throttle_period - time_since_last_call
124
125 if time_left > datetime.timedelta(seconds=0):
126 return
127
128 self.time_of_last_call = datetime.datetime.now()
129 return fn(*args, **kwargs)
130
131 return wrapper
132
133 def apply_throttle(value):
134
135 @Throttle(seconds=value)
136 def fnThrottled(fn):
137 fn()
138
139 return fnThrottled
140
141 @MPDecorator
142 class Plotter(ProcessingUnit):
143 '''
144 Proccessing unit to handle plot operations
145 '''
146
147 def __init__(self):
148
149 ProcessingUnit.__init__(self)
150
151 def setup(self, **kwargs):
152
153 self.connections = 0
154 self.web_address = kwargs.get('web_server', False)
155 self.realtime = kwargs.get('realtime', False)
156 self.localtime = kwargs.get('localtime', True)
157 self.buffering = kwargs.get('buffering', True)
158 self.throttle = kwargs.get('throttle', 2)
159 self.exp_code = kwargs.get('exp_code', None)
160 self.set_ready = apply_throttle(self.throttle)
161 self.dates = []
162 self.data = PlotterData(
163 self.plots, self.throttle, self.exp_code, self.buffering)
164 self.isConfig = True
165
166 def ready(self):
167 '''
168 Set dataOut ready
169 '''
170
171 self.data.ready = True
172 self.dataOut.data_plt = self.data
173
174 def run(self, realtime=True, localtime=True, buffering=True,
175 throttle=2, exp_code=None, web_server=None):
176
177 if not self.isConfig:
178 self.setup(realtime=realtime, localtime=localtime,
179 buffering=buffering, throttle=throttle, exp_code=exp_code,
180 web_server=web_server)
181
182 if self.web_address:
183 log.success(
184 'Sending to web: {}'.format(self.web_address),
185 self.name
186 )
187 self.context = zmq.Context()
188 self.sender_web = self.context.socket(zmq.REQ)
189 self.sender_web.connect(self.web_address)
190 self.poll = zmq.Poller()
191 self.poll.register(self.sender_web, zmq.POLLIN)
192 time.sleep(1)
193
194 # t = Thread(target=self.event_monitor, args=(monitor,))
195 # t.start()
196
197 self.dataOut = self.dataIn
198 self.data.ready = False
199
200 if self.dataOut.flagNoData:
201 coerce = True
202 else:
203 coerce = False
204
205 if self.dataOut.type == 'Parameters':
206 tm = self.dataOut.utctimeInit
207 else:
208 tm = self.dataOut.utctime
209 if self.dataOut.useLocalTime:
210 if not self.localtime:
211 tm += time.timezone
212 dt = datetime.datetime.fromtimestamp(tm).date()
213 else:
214 if self.localtime:
215 tm -= time.timezone
216 dt = datetime.datetime.utcfromtimestamp(tm).date()
217 if dt not in self.dates:
218 if self.data:
219 self.ready()
220 self.data.setup()
221 self.dates.append(dt)
222
223 self.data.update(self.dataOut, tm)
224
225 if False: # TODO check when publishers ends
226 self.connections -= 1
227 if self.connections == 0 and dt in self.dates:
228 self.data.ended = True
229 self.ready()
230 time.sleep(1)
231 else:
232 if self.realtime:
233 self.ready()
234 if self.web_address:
235 retries = 5
236 while True:
237 self.sender_web.send(self.data.jsonify())
238 socks = dict(self.poll.poll(5000))
239 if socks.get(self.sender_web) == zmq.POLLIN:
240 reply = self.sender_web.recv_string()
241 if reply == 'ok':
242 log.log("Response from server ok", self.name)
243 break
244 else:
245 log.warning(
246 "Malformed reply from server: {}".format(reply), self.name)
247
248 else:
249 log.warning(
250 "No response from server, retrying...", self.name)
251 self.sender_web.setsockopt(zmq.LINGER, 0)
252 self.sender_web.close()
253 self.poll.unregister(self.sender_web)
254 retries -= 1
255 if retries == 0:
256 log.error(
257 "Server seems to be offline, abandoning", self.name)
258 self.sender_web = self.context.socket(zmq.REQ)
259 self.sender_web.connect(self.web_address)
260 self.poll.register(self.sender_web, zmq.POLLIN)
261 time.sleep(1)
262 break
263 self.sender_web = self.context.socket(zmq.REQ)
264 self.sender_web.connect(self.web_address)
265 self.poll.register(self.sender_web, zmq.POLLIN)
266 time.sleep(1)
267 else:
268 self.set_ready(self.ready, coerce=coerce)
269
270 return
271
272 def close(self):
273 pass
274
275
276 @MPDecorator
277 class Plot(Operation):
278 '''
279 Base class for Schain plotting operations
280 '''
281
282 CODE = 'Figure'
283 colormap = 'jro'
284 bgcolor = 'white'
285 __missing = 1E30
286
287 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
288 'zlimits', 'xlabel', 'ylabel', 'xaxis', 'cb_label', 'title',
289 'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
290 'showprofile', 'decimation', 'pause']
291
292 def __init__(self):
293
294 Operation.__init__(self)
295 self.isConfig = False
296 self.isPlotConfig = False
297
298 def __fmtTime(self, x, pos):
299 '''
300 '''
301
302 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
303
304 def __setup(self, **kwargs):
305 '''
306 Initialize variables
307 '''
308
309 self.figures = []
310 self.axes = []
311 self.cb_axes = []
312 self.localtime = kwargs.pop('localtime', True)
313 self.show = kwargs.get('show', True)
314 self.save = kwargs.get('save', False)
315 self.ftp = kwargs.get('ftp', False)
316 self.colormap = kwargs.get('colormap', self.colormap)
317 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
318 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
319 self.colormaps = kwargs.get('colormaps', None)
320 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
321 self.showprofile = kwargs.get('showprofile', False)
322 self.title = kwargs.get('wintitle', self.CODE.upper())
323 self.cb_label = kwargs.get('cb_label', None)
324 self.cb_labels = kwargs.get('cb_labels', None)
325 self.labels = kwargs.get('labels', None)
326 self.xaxis = kwargs.get('xaxis', 'frequency')
327 self.zmin = kwargs.get('zmin', None)
328 self.zmax = kwargs.get('zmax', None)
329 self.zlimits = kwargs.get('zlimits', None)
330 self.xmin = kwargs.get('xmin', None)
331 self.xmax = kwargs.get('xmax', None)
332 self.xrange = kwargs.get('xrange', 12)
333 self.xscale = kwargs.get('xscale', None)
334 self.ymin = kwargs.get('ymin', None)
335 self.ymax = kwargs.get('ymax', None)
336 self.yscale = kwargs.get('yscale', None)
337 self.xlabel = kwargs.get('xlabel', None)
338 self.decimation = kwargs.get('decimation', None)
339 self.showSNR = kwargs.get('showSNR', False)
340 self.oneFigure = kwargs.get('oneFigure', True)
341 self.width = kwargs.get('width', None)
342 self.height = kwargs.get('height', None)
343 self.colorbar = kwargs.get('colorbar', True)
344 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
345 self.channels = kwargs.get('channels', None)
346 self.titles = kwargs.get('titles', [])
347 self.polar = False
348 self.grid = kwargs.get('grid', False)
349 self.pause = kwargs.get('pause', False)
350 self.save_labels = kwargs.get('save_labels', None)
351 self.realtime = kwargs.get('realtime', True)
352 self.buffering = kwargs.get('buffering', True)
353 self.throttle = kwargs.get('throttle', 2)
354 self.exp_code = kwargs.get('exp_code', None)
355 self.__throttle_plot = apply_throttle(self.throttle)
356 self.data = PlotterData(
357 self.CODE, self.throttle, self.exp_code, self.buffering)
358
359 def __setup_plot(self):
360 '''
361 Common setup for all figures, here figures and axes are created
362 '''
363
364 self.setup()
365
366 self.time_label = 'LT' if self.localtime else 'UTC'
367 if self.data.localtime:
368 self.getDateTime = datetime.datetime.fromtimestamp
369 else:
370 self.getDateTime = datetime.datetime.utcfromtimestamp
371
372 if self.width is None:
373 self.width = 8
374
375 self.figures = []
376 self.axes = []
377 self.cb_axes = []
378 self.pf_axes = []
379 self.cmaps = []
380
381 size = '15%' if self.ncols == 1 else '30%'
382 pad = '4%' if self.ncols == 1 else '8%'
383
384 if self.oneFigure:
385 if self.height is None:
386 self.height = 1.4 * self.nrows + 1
387 fig = plt.figure(figsize=(self.width, self.height),
388 edgecolor='k',
389 facecolor='w')
390 self.figures.append(fig)
391 for n in range(self.nplots):
392 ax = fig.add_subplot(self.nrows, self.ncols,
393 n + 1, polar=self.polar)
394 ax.tick_params(labelsize=8)
395 ax.firsttime = True
396 ax.index = 0
397 ax.press = None
398 self.axes.append(ax)
399 if self.showprofile:
400 cax = self.__add_axes(ax, size=size, pad=pad)
401 cax.tick_params(labelsize=8)
402 self.pf_axes.append(cax)
403 else:
404 if self.height is None:
405 self.height = 3
406 for n in range(self.nplots):
407 fig = plt.figure(figsize=(self.width, self.height),
408 edgecolor='k',
409 facecolor='w')
410 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
411 ax.tick_params(labelsize=8)
412 ax.firsttime = True
413 ax.index = 0
414 ax.press = None
415 self.figures.append(fig)
416 self.axes.append(ax)
417 if self.showprofile:
418 cax = self.__add_axes(ax, size=size, pad=pad)
419 cax.tick_params(labelsize=8)
420 self.pf_axes.append(cax)
421
422 for n in range(self.nrows):
423 if self.colormaps is not None:
424 cmap = plt.get_cmap(self.colormaps[n])
425 else:
426 cmap = plt.get_cmap(self.colormap)
427 cmap.set_bad(self.bgcolor, 1.)
428 self.cmaps.append(cmap)
429
430 for fig in self.figures:
431 fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
432 fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
433 fig.canvas.mpl_connect('button_press_event', self.onBtnPress)
434 fig.canvas.mpl_connect('motion_notify_event', self.onMotion)
435 fig.canvas.mpl_connect('button_release_event', self.onBtnRelease)
436 if self.show:
437 fig.show()
438
439 def OnKeyPress(self, event):
440 '''
441 Event for pressing keys (up, down) change colormap
442 '''
443 ax = event.inaxes
444 if ax in self.axes:
445 if event.key == 'down':
446 ax.index += 1
447 elif event.key == 'up':
448 ax.index -= 1
449 if ax.index < 0:
450 ax.index = len(CMAPS) - 1
451 elif ax.index == len(CMAPS):
452 ax.index = 0
453 cmap = CMAPS[ax.index]
454 ax.cbar.set_cmap(cmap)
455 ax.cbar.draw_all()
456 ax.plt.set_cmap(cmap)
457 ax.cbar.patch.figure.canvas.draw()
458 self.colormap = cmap.name
459
460 def OnBtnScroll(self, event):
461 '''
462 Event for scrolling, scale figure
463 '''
464 cb_ax = event.inaxes
465 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
466 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
467 pt = ax.cbar.ax.bbox.get_points()[:, 1]
468 nrm = ax.cbar.norm
469 vmin, vmax, p0, p1, pS = (
470 nrm.vmin, nrm.vmax, pt[0], pt[1], event.y)
471 scale = 2 if event.step == 1 else 0.5
472 point = vmin + (vmax - vmin) / (p1 - p0) * (pS - p0)
473 ax.cbar.norm.vmin = point - scale * (point - vmin)
474 ax.cbar.norm.vmax = point - scale * (point - vmax)
475 ax.plt.set_norm(ax.cbar.norm)
476 ax.cbar.draw_all()
477 ax.cbar.patch.figure.canvas.draw()
478
479 def onBtnPress(self, event):
480 '''
481 Event for mouse button press
482 '''
483 cb_ax = event.inaxes
484 if cb_ax is None:
485 return
486
487 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
488 cb_ax.press = event.x, event.y
489 else:
490 cb_ax.press = None
491
492 def onMotion(self, event):
493 '''
494 Event for move inside colorbar
495 '''
496 cb_ax = event.inaxes
497 if cb_ax is None:
498 return
499 if cb_ax not in [ax.cbar.ax for ax in self.axes if ax.cbar]:
500 return
501 if cb_ax.press is None:
502 return
503
504 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
505 xprev, yprev = cb_ax.press
506 dx = event.x - xprev
507 dy = event.y - yprev
508 cb_ax.press = event.x, event.y
509 scale = ax.cbar.norm.vmax - ax.cbar.norm.vmin
510 perc = 0.03
511
512 if event.button == 1:
513 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
514 ax.cbar.norm.vmax -= (perc * scale) * numpy.sign(dy)
515 elif event.button == 3:
516 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
517 ax.cbar.norm.vmax += (perc * scale) * numpy.sign(dy)
518
519 ax.cbar.draw_all()
520 ax.plt.set_norm(ax.cbar.norm)
521 ax.cbar.patch.figure.canvas.draw()
522
523 def onBtnRelease(self, event):
524 '''
525 Event for mouse button release
526 '''
527 cb_ax = event.inaxes
528 if cb_ax is not None:
529 cb_ax.press = None
530
531 def __add_axes(self, ax, size='30%', pad='8%'):
532 '''
533 Add new axes to the given figure
534 '''
535 divider = make_axes_locatable(ax)
536 nax = divider.new_horizontal(size=size, pad=pad)
537 ax.figure.add_axes(nax)
538 return nax
539
540 def setup(self):
541 '''
542 This method should be implemented in the child class, the following
543 attributes should be set:
544
545 self.nrows: number of rows
546 self.ncols: number of cols
547 self.nplots: number of plots (channels or pairs)
548 self.ylabel: label for Y axes
549 self.titles: list of axes title
550
551 '''
552 raise NotImplementedError
553
554 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
555 '''
556 Create a masked array for missing data
557 '''
558 if x_buffer.shape[0] < 2:
559 return x_buffer, y_buffer, z_buffer
560
561 deltas = x_buffer[1:] - x_buffer[0:-1]
562 x_median = numpy.median(deltas)
563
564 index = numpy.where(deltas > 5 * x_median)
565
566 if len(index[0]) != 0:
567 z_buffer[::, index[0], ::] = self.__missing
568 z_buffer = numpy.ma.masked_inside(z_buffer,
569 0.99 * self.__missing,
570 1.01 * self.__missing)
571
572 return x_buffer, y_buffer, z_buffer
573
574 def decimate(self):
575
576 # dx = int(len(self.x)/self.__MAXNUMX) + 1
577 dy = int(len(self.y) / self.decimation) + 1
578
579 # x = self.x[::dx]
580 x = self.x
581 y = self.y[::dy]
582 z = self.z[::, ::, ::dy]
583
584 return x, y, z
585
586 def format(self):
587 '''
588 Set min and max values, labels, ticks and titles
589 '''
590
591 if self.xmin is None:
592 xmin = self.data.min_time
593 else:
594 if self.xaxis is 'time':
595 dt = self.getDateTime(self.data.min_time)
596 xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
597 datetime.datetime(1970, 1, 1)).total_seconds()
598 if self.data.localtime:
599 xmin += time.timezone
600 else:
601 xmin = self.xmin
602
603 if self.xmax is None:
604 xmax = xmin + self.xrange * 60 * 60
605 else:
606 if self.xaxis is 'time':
607 dt = self.getDateTime(self.data.max_time)
608 xmax = (dt.replace(hour=int(self.xmax), minute=59, second=59) -
609 datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds()
610 if self.data.localtime:
611 xmax += time.timezone
612 else:
613 xmax = self.xmax
614
615 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
616 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
617
618 Y = numpy.array([1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000])
619 i = 1 if numpy.where(
620 abs(ymax-ymin) <= Y)[0][0] < 0 else numpy.where(abs(ymax-ymin) <= Y)[0][0]
621 ystep = Y[i] / 10.
622
623 if self.xaxis is not 'time':
624 X = numpy.array([1, 2, 5, 10, 20, 50, 100,
625 200, 500, 1000, 2000, 5000])/2.
626 i = 1 if numpy.where(
627 abs(xmax-xmin) <= X)[0][0] < 0 else numpy.where(abs(xmax-xmin) <= X)[0][0]
628 xstep = X[i] / 10.
629
630 for n, ax in enumerate(self.axes):
631 if ax.firsttime:
632 ax.set_facecolor(self.bgcolor)
633 ax.yaxis.set_major_locator(MultipleLocator(ystep))
634 if self.xscale:
635 ax.xaxis.set_major_formatter(FuncFormatter(
636 lambda x, pos: '{0:g}'.format(x*self.xscale)))
637 if self.xscale:
638 ax.yaxis.set_major_formatter(FuncFormatter(
639 lambda x, pos: '{0:g}'.format(x*self.yscale)))
640 if self.xaxis is 'time':
641 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
642 ax.xaxis.set_major_locator(LinearLocator(9))
643 else:
644 ax.xaxis.set_major_locator(MultipleLocator(xstep))
645 if self.xlabel is not None:
646 ax.set_xlabel(self.xlabel)
647 ax.set_ylabel(self.ylabel)
648 ax.firsttime = False
649 if self.showprofile:
650 self.pf_axes[n].set_ylim(ymin, ymax)
651 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
652 self.pf_axes[n].set_xlabel('dB')
653 self.pf_axes[n].grid(b=True, axis='x')
654 [tick.set_visible(False)
655 for tick in self.pf_axes[n].get_yticklabels()]
656 if self.colorbar:
657 ax.cbar = plt.colorbar(
658 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
659 ax.cbar.ax.tick_params(labelsize=8)
660 ax.cbar.ax.press = None
661 if self.cb_label:
662 ax.cbar.set_label(self.cb_label, size=8)
663 elif self.cb_labels:
664 ax.cbar.set_label(self.cb_labels[n], size=8)
665 else:
666 ax.cbar = None
667 if self.grid:
668 ax.grid(True)
669
670 if not self.polar:
671 ax.set_xlim(xmin, xmax)
672 ax.set_ylim(ymin, ymax)
673 ax.set_title('{} {} {}'.format(
674 self.titles[n],
675 self.getDateTime(self.data.max_time).strftime(
676 '%Y-%m-%dT%H:%M:%S'),
677 self.time_label),
678 size=8)
679 else:
680 ax.set_title('{}'.format(self.titles[n]), size=8)
681 ax.set_ylim(0, 90)
682 ax.set_yticks(numpy.arange(0, 90, 20))
683 ax.yaxis.labelpad = 40
684
685 def clear_figures(self):
686 '''
687 Reset axes for redraw plots
688 '''
689
690 for ax in self.axes:
691 ax.clear()
692 ax.firsttime = True
693 if ax.cbar:
694 ax.cbar.remove()
695
696 def __plot(self):
697 '''
698 Main function to plot, format and save figures
699 '''
700
701 #try:
702 self.plot()
703 self.format()
704 #except Exception as e:
705 # log.warning('{} Plot could not be updated... check data'.format(
706 # self.CODE), self.name)
707 # log.error(str(e), '')
708 # return
709
710 for n, fig in enumerate(self.figures):
711 if self.nrows == 0 or self.nplots == 0:
712 log.warning('No data', self.name)
713 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
714 fig.canvas.manager.set_window_title(self.CODE)
715 continue
716
717 fig.tight_layout()
718 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
719 self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
720 fig.canvas.draw()
721
722 if self.save:
723
724 if self.save_labels:
725 labels = self.save_labels
726 else:
727 labels = list(range(self.nrows))
728
729 if self.oneFigure:
730 label = ''
731 else:
732 label = '-{}'.format(labels[n])
733 figname = os.path.join(
734 self.save,
735 self.CODE,
736 '{}{}_{}.png'.format(
737 self.CODE,
738 label,
739 self.getDateTime(self.data.max_time).strftime(
740 '%Y%m%d_%H%M%S'),
741 )
742 )
743 log.log('Saving figure: {}'.format(figname), self.name)
744 if not os.path.isdir(os.path.dirname(figname)):
745 os.makedirs(os.path.dirname(figname))
746 fig.savefig(figname)
747
748 def plot(self):
749 '''
750 Must be defined in the child class
751 '''
752 raise NotImplementedError
753
754 def run(self, dataOut, **kwargs):
755
756 if dataOut.flagNoData and not dataOut.error:
757 return dataOut
758
759 if dataOut.error:
760 coerce = True
761 else:
762 coerce = False
763
764 if self.isConfig is False:
765 self.__setup(**kwargs)
766 self.data.setup()
767 self.isConfig = True
768
769 if dataOut.type == 'Parameters':
770 tm = dataOut.utctimeInit
771 else:
772 tm = dataOut.utctime
773
774 if dataOut.useLocalTime:
775 if not self.localtime:
776 tm += time.timezone
777 else:
778 if self.localtime:
779 tm -= time.timezone
780
781 if self.data and (tm - self.data.min_time) >= self.xrange*60*60:
782 self.__plot()
783 self.data.setup()
784 self.clear_figures()
785
786 self.data.update(dataOut, tm)
787
788 if self.isPlotConfig is False:
789 self.__setup_plot()
790 self.isPlotConfig = True
791
792 if self.realtime:
793 self.__plot()
794 else:
795 self.__throttle_plot(self.__plot, coerce=coerce)
796
797 figpause(0.001)
798
799 def close(self):
800
801 if self.data and self.pause:
802 figpause(10)
803
Show file before | Show file after | Hide comments
@@ -1,5 +1,12
1 1 ## CHANGELOG:
2 2
3 ### 3.0
4 * Python 3.x compatible
5 * New architecture with multiprocessing and IPC communication
6 * Add @MPDecorator for multiprocessing Units and Operations
7 * Added new type of operation `external` for non-locking operations
8 * New plotting architecture with buffering/throttle capabilities to speed up plots
9
3 10 ### 2.3
4 11 * Added support for Madrigal formats (reading/writing).
5 12 * Added support for reading BLTR parameters (*.sswma).
Show file before | Show file after | Hide comments
@@ -24,7 +24,7 from email.mime.multipart import MIMEMultipart
24 24
25 25 import schainpy
26 26 from schainpy.utils import log
27 from schainpy.model.graphics.jroplot_data import popup
27 from schainpy.model.graphics.jroplot_base import popup
28 28
29 29 def get_path():
30 30 '''
Show file before | Show file after | Hide comments
@@ -97,9 +97,7 def wait(context):
97 97 receiver = c.socket(zmq.SUB)
98 98 receiver.connect('ipc:///tmp/schain_{}_pub'.format(self.id))
99 99 receiver.setsockopt(zmq.SUBSCRIBE, self.id.encode())
100 log.error('startinggg')
101 100 msg = receiver.recv_multipart()[1]
102 #log.error(msg)
103 101 context.terminate()
104 102
105 103 class ParameterConf():
@@ -1245,7 +1243,7 class Project(Process):
1245 1243
1246 1244 try:
1247 1245 zmq.proxy(xpub, xsub)
1248 except zmq.ContextTerminated:
1246 except: # zmq.ContextTerminated:
1249 1247 xpub.close()
1250 1248 xsub.close()
1251 1249
@@ -1260,6 +1258,6 class Project(Process):
1260 1258
1261 1259 # Iniciar todos los procesos .start(), monitoreo de procesos. ELiminar lo de abajo
1262 1260
1263 log.success('{} finished (time: {}s)'.format(
1261 log.success('{} Done (time: {}s)'.format(
1264 1262 self.name,
1265 1263 time.time()-self.start_time))
Show file before | Show file after | Hide comments
@@ -7,7 +7,9 $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
7 7 import copy
8 8 import numpy
9 9 import datetime
10 import json
10 11
12 from schainpy.utils import log
11 13 from .jroheaderIO import SystemHeader, RadarControllerHeader
12 14
13 15
@@ -147,85 +149,49 class JROData(GenericData):
147 149 # m_ProcessingHeader = ProcessingHeader()
148 150
149 151 systemHeaderObj = SystemHeader()
150
151 152 radarControllerHeaderObj = RadarControllerHeader()
152
153 153 # data = None
154
155 154 type = None
156
157 155 datatype = None # dtype but in string
158
159 156 # dtype = None
160
161 157 # nChannels = None
162
163 158 # nHeights = None
164
165 159 nProfiles = None
166
167 160 heightList = None
168
169 161 channelList = None
170
171 162 flagDiscontinuousBlock = False
172
173 163 useLocalTime = False
174
175 164 utctime = None
176
177 165 timeZone = None
178
179 166 dstFlag = None
180
181 167 errorCount = None
182
183 168 blocksize = None
184
185 169 # nCode = None
186 #
187 170 # nBaud = None
188 #
189 171 # code = None
190
191 172 flagDecodeData = False # asumo q la data no esta decodificada
192
193 173 flagDeflipData = False # asumo q la data no esta sin flip
194
195 174 flagShiftFFT = False
196
197 175 # ippSeconds = None
198
199 176 # timeInterval = None
200
201 177 nCohInt = None
202
203 178 # noise = None
204
205 179 windowOfFilter = 1
206
207 180 # Speed of ligth
208 181 C = 3e8
209
210 182 frequency = 49.92e6
211
212 183 realtime = False
213
214 184 beacon_heiIndexList = None
215
216 185 last_block = None
217
218 186 blocknow = None
219
220 187 azimuth = None
221
222 188 zenith = None
223
224 189 beam = Beam()
225
226 190 profileIndex = None
191 error = None
192 data = None
193 data_plt = None
227 194
228 error = (0, '')
229 195
230 196 def __str__(self):
231 197
@@ -395,53 +361,29 class Voltage(JROData):
395 361 '''
396 362
397 363 self.useLocalTime = True
398
399 364 self.radarControllerHeaderObj = RadarControllerHeader()
400
401 365 self.systemHeaderObj = SystemHeader()
402
403 366 self.type = "Voltage"
404
405 367 self.data = None
406
407 368 # self.dtype = None
408
409 369 # self.nChannels = 0
410
411 370 # self.nHeights = 0
412
413 371 self.nProfiles = None
414
415 self.heightList = None
416
372 self.heightList = Non
417 373 self.channelList = None
418
419 374 # self.channelIndexList = None
420
421 375 self.flagNoData = True
422
423 376 self.flagDiscontinuousBlock = False
424
425 377 self.utctime = None
426
427 378 self.timeZone = None
428
429 379 self.dstFlag = None
430
431 380 self.errorCount = None
432
433 381 self.nCohInt = None
434
435 382 self.blocksize = None
436
437 383 self.flagDecodeData = False # asumo q la data no esta decodificada
438
439 384 self.flagDeflipData = False # asumo q la data no esta sin flip
440
441 385 self.flagShiftFFT = False
442
443 386 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
444
445 387 self.profileIndex = 0
446 388
447 389 def getNoisebyHildebrand(self, channel=None):
@@ -505,93 +447,53 class Spectra(JROData):
505 447
506 448 # data spc es un numpy array de 2 dmensiones (canales, perfiles, alturas)
507 449 data_spc = None
508
509 450 # data cspc es un numpy array de 2 dmensiones (canales, pares, alturas)
510 451 data_cspc = None
511
512 452 # data dc es un numpy array de 2 dmensiones (canales, alturas)
513 453 data_dc = None
514
515 454 # data power
516 455 data_pwr = None
517
518 456 nFFTPoints = None
519
520 457 # nPairs = None
521
522 458 pairsList = None
523
524 459 nIncohInt = None
525
526 460 wavelength = None # Necesario para cacular el rango de velocidad desde la frecuencia
527
528 461 nCohInt = None # se requiere para determinar el valor de timeInterval
529
530 462 ippFactor = None
531
532 463 profileIndex = 0
533
534 464 plotting = "spectra"
535
536 465 def __init__(self):
537 466 '''
538 467 Constructor
539 468 '''
540 469
541 470 self.useLocalTime = True
542
543 471 self.radarControllerHeaderObj = RadarControllerHeader()
544
545 472 self.systemHeaderObj = SystemHeader()
546
547 473 self.type = "Spectra"
548
549 474 # self.data = None
550
551 475 # self.dtype = None
552
553 476 # self.nChannels = 0
554
555 477 # self.nHeights = 0
556
557 478 self.nProfiles = None
558
559 479 self.heightList = None
560
561 480 self.channelList = None
562
563 481 # self.channelIndexList = None
564
565 482 self.pairsList = None
566
567 483 self.flagNoData = True
568
569 484 self.flagDiscontinuousBlock = False
570
571 485 self.utctime = None
572
573 486 self.nCohInt = None
574
575 487 self.nIncohInt = None
576
577 488 self.blocksize = None
578
579 489 self.nFFTPoints = None
580
581 490 self.wavelength = None
582
583 491 self.flagDecodeData = False # asumo q la data no esta decodificada
584
585 492 self.flagDeflipData = False # asumo q la data no esta sin flip
586
587 493 self.flagShiftFFT = False
588
589 494 self.ippFactor = 1
590
591 495 #self.noise = None
592
593 496 self.beacon_heiIndexList = []
594
595 497 self.noise_estimation = None
596 498
597 499 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
@@ -692,7 +594,8 class Spectra(JROData):
692 594
693 595 def getTimeInterval(self):
694 596
695 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
597 timeInterval = self.ippSeconds * self.nCohInt * \
598 self.nIncohInt * self.nProfiles * self.ippFactor
696 599
697 600 return timeInterval
698 601
@@ -755,19 +658,12 class Spectra(JROData):
755 658 class SpectraHeis(Spectra):
756 659
757 660 data_spc = None
758
759 661 data_cspc = None
760
761 662 data_dc = None
762
763 663 nFFTPoints = None
764
765 664 # nPairs = None
766
767 665 pairsList = None
768
769 666 nCohInt = None
770
771 667 nIncohInt = None
772 668
773 669 def __init__(self):
@@ -830,36 +726,21 class SpectraHeis(Spectra):
830 726 class Fits(JROData):
831 727
832 728 heightList = None
833
834 729 channelList = None
835
836 730 flagNoData = True
837
838 731 flagDiscontinuousBlock = False
839
840 732 useLocalTime = False
841
842 733 utctime = None
843
844 734 timeZone = None
845
846 735 # ippSeconds = None
847
848 736 # timeInterval = None
849
850 737 nCohInt = None
851
852 738 nIncohInt = None
853
854 739 noise = None
855
856 740 windowOfFilter = 1
857
858 741 # Speed of ligth
859 742 C = 3e8
860
861 743 frequency = 49.92e6
862
863 744 realtime = False
864 745
865 746 def __init__(self):
@@ -978,33 +859,19 class Fits(JROData):
978 859 class Correlation(JROData):
979 860
980 861 noise = None
981
982 862 SNR = None
983
984 863 #--------------------------------------------------
985
986 864 mode = None
987
988 865 split = False
989
990 866 data_cf = None
991
992 867 lags = None
993
994 868 lagRange = None
995
996 869 pairsList = None
997
998 870 normFactor = None
999
1000 871 #--------------------------------------------------
1001
1002 872 # calculateVelocity = None
1003
1004 873 nLags = None
1005
1006 874 nPairs = None
1007
1008 875 nAvg = None
1009 876
1010 877 def __init__(self):
@@ -1068,7 +935,8 class Correlation(JROData):
1068 935 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
1069 936
1070 937 if ind_vel[0] < 0:
1071 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
938 ind_vel[list(range(0, 1))] = ind_vel[list(
939 range(0, 1))] + self.num_prof
1072 940
1073 941 if mode == 1:
1074 942 jspectra[:, freq_dc, :] = (
@@ -1154,55 +1022,30 class Correlation(JROData):
1154 1022 class Parameters(Spectra):
1155 1023
1156 1024 experimentInfo = None # Information about the experiment
1157
1158 1025 # Information from previous data
1159
1160 1026 inputUnit = None # Type of data to be processed
1161
1162 1027 operation = None # Type of operation to parametrize
1163
1164 1028 # normFactor = None #Normalization Factor
1165
1166 1029 groupList = None # List of Pairs, Groups, etc
1167
1168 1030 # Parameters
1169
1170 1031 data_param = None # Parameters obtained
1171
1172 1032 data_pre = None # Data Pre Parametrization
1173
1174 1033 data_SNR = None # Signal to Noise Ratio
1175
1176 1034 # heightRange = None #Heights
1177
1178 1035 abscissaList = None # Abscissa, can be velocities, lags or time
1179
1180 1036 # noise = None #Noise Potency
1181
1182 1037 utctimeInit = None # Initial UTC time
1183
1184 1038 paramInterval = None # Time interval to calculate Parameters in seconds
1185
1186 1039 useLocalTime = True
1187
1188 1040 # Fitting
1189
1190 1041 data_error = None # Error of the estimation
1191
1192 1042 constants = None
1193
1194 1043 library = None
1195
1196 1044 # Output signal
1197
1198 1045 outputInterval = None # Time interval to calculate output signal in seconds
1199
1200 1046 data_output = None # Out signal
1201
1202 1047 nAvg = None
1203
1204 1048 noise_estimation = None
1205
1206 1049 GauSPC = None # Fit gaussian SPC
1207 1050
1208 1051 def __init__(self):
@@ -1249,3 +1092,251 class Parameters(Spectra):
1249 1092
1250 1093 timeInterval = property(getTimeInterval)
1251 1094 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
1095
1096
1097 class PlotterData(object):
1098 '''
1099 Object to hold data to be plotted
1100 '''
1101
1102 MAXNUMX = 100
1103 MAXNUMY = 100
1104
1105 def __init__(self, code, throttle_value, exp_code, buffering=True):
1106
1107 self.throttle = throttle_value
1108 self.exp_code = exp_code
1109 self.buffering = buffering
1110 self.ready = False
1111 self.localtime = False
1112 self.data = {}
1113 self.meta = {}
1114 self.__times = []
1115 self.__heights = []
1116
1117 if 'snr' in code:
1118 self.plottypes = ['snr']
1119 elif code == 'spc':
1120 self.plottypes = ['spc', 'noise', 'rti']
1121 elif code == 'rti':
1122 self.plottypes = ['noise', 'rti']
1123 else:
1124 self.plottypes = [code]
1125
1126 for plot in self.plottypes:
1127 self.data[plot] = {}
1128
1129 def __str__(self):
1130 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
1131 return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times))
1132
1133 def __len__(self):
1134 return len(self.__times)
1135
1136 def __getitem__(self, key):
1137
1138 if key not in self.data:
1139 raise KeyError(log.error('Missing key: {}'.format(key)))
1140 if 'spc' in key or not self.buffering:
1141 ret = self.data[key]
1142 else:
1143 ret = numpy.array([self.data[key][x] for x in self.times])
1144 if ret.ndim > 1:
1145 ret = numpy.swapaxes(ret, 0, 1)
1146 return ret
1147
1148 def __contains__(self, key):
1149 return key in self.data
1150
1151 def setup(self):
1152 '''
1153 Configure object
1154 '''
1155
1156 self.type = ''
1157 self.ready = False
1158 self.data = {}
1159 self.__times = []
1160 self.__heights = []
1161 self.__all_heights = set()
1162 for plot in self.plottypes:
1163 if 'snr' in plot:
1164 plot = 'snr'
1165 self.data[plot] = {}
1166
1167 if 'spc' in self.data or 'rti' in self.data:
1168 self.data['noise'] = {}
1169 if 'noise' not in self.plottypes:
1170 self.plottypes.append('noise')
1171
1172 def shape(self, key):
1173 '''
1174 Get the shape of the one-element data for the given key
1175 '''
1176
1177 if len(self.data[key]):
1178 if 'spc' in key or not self.buffering:
1179 return self.data[key].shape
1180 return self.data[key][self.__times[0]].shape
1181 return (0,)
1182
1183 def update(self, dataOut, tm):
1184 '''
1185 Update data object with new dataOut
1186 '''
1187
1188 if tm in self.__times:
1189 return
1190
1191 self.type = dataOut.type
1192 self.parameters = getattr(dataOut, 'parameters', [])
1193 if hasattr(dataOut, 'pairsList'):
1194 self.pairs = dataOut.pairsList
1195 if hasattr(dataOut, 'meta'):
1196 self.meta = dataOut.meta
1197 self.channels = dataOut.channelList
1198 self.interval = dataOut.getTimeInterval()
1199 self.localtime = dataOut.useLocalTime
1200 if 'spc' in self.plottypes or 'cspc' in self.plottypes:
1201 self.xrange = (dataOut.getFreqRange(1)/1000.,
1202 dataOut.getAcfRange(1), dataOut.getVelRange(1))
1203 self.__heights.append(dataOut.heightList)
1204 self.__all_heights.update(dataOut.heightList)
1205 self.__times.append(tm)
1206
1207 for plot in self.plottypes:
1208 if plot == 'spc':
1209 z = dataOut.data_spc/dataOut.normFactor
1210 buffer = 10*numpy.log10(z)
1211 if plot == 'cspc':
1212 buffer = dataOut.data_cspc
1213 if plot == 'noise':
1214 buffer = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
1215 if plot == 'rti':
1216 buffer = dataOut.getPower()
1217 if plot == 'snr_db':
1218 buffer = dataOut.data_SNR
1219 if plot == 'snr':
1220 buffer = 10*numpy.log10(dataOut.data_SNR)
1221 if plot == 'dop':
1222 buffer = 10*numpy.log10(dataOut.data_DOP)
1223 if plot == 'mean':
1224 buffer = dataOut.data_MEAN
1225 if plot == 'std':
1226 buffer = dataOut.data_STD
1227 if plot == 'coh':
1228 buffer = dataOut.getCoherence()
1229 if plot == 'phase':
1230 buffer = dataOut.getCoherence(phase=True)
1231 if plot == 'output':
1232 buffer = dataOut.data_output
1233 if plot == 'param':
1234 buffer = dataOut.data_param
1235
1236 if 'spc' in plot:
1237 self.data[plot] = buffer
1238 else:
1239 if self.buffering:
1240 self.data[plot][tm] = buffer
1241 else:
1242 self.data[plot] = buffer
1243
1244 def normalize_heights(self):
1245 '''
1246 Ensure same-dimension of the data for different heighList
1247 '''
1248
1249 H = numpy.array(list(self.__all_heights))
1250 H.sort()
1251 for key in self.data:
1252 shape = self.shape(key)[:-1] + H.shape
1253 for tm, obj in list(self.data[key].items()):
1254 h = self.__heights[self.__times.index(tm)]
1255 if H.size == h.size:
1256 continue
1257 index = numpy.where(numpy.in1d(H, h))[0]
1258 dummy = numpy.zeros(shape) + numpy.nan
1259 if len(shape) == 2:
1260 dummy[:, index] = obj
1261 else:
1262 dummy[index] = obj
1263 self.data[key][tm] = dummy
1264
1265 self.__heights = [H for tm in self.__times]
1266
1267 def jsonify(self, decimate=False):
1268 '''
1269 Convert data to json
1270 '''
1271
1272 data = {}
1273 tm = self.times[-1]
1274 dy = int(self.heights.size/self.MAXNUMY) + 1
1275 for key in self.data:
1276 if key in ('spc', 'cspc') or not self.buffering:
1277 dx = int(self.data[key].shape[1]/self.MAXNUMX) + 1
1278 data[key] = self.roundFloats(
1279 self.data[key][::, ::dx, ::dy].tolist())
1280 else:
1281 data[key] = self.roundFloats(self.data[key][tm].tolist())
1282
1283 ret = {'data': data}
1284 ret['exp_code'] = self.exp_code
1285 ret['time'] = float(tm)
1286 ret['interval'] = float(self.interval)
1287 ret['localtime'] = self.localtime
1288 ret['yrange'] = self.roundFloats(self.heights[::dy].tolist())
1289 if 'spc' in self.data or 'cspc' in self.data:
1290 ret['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1291 else:
1292 ret['xrange'] = []
1293 if hasattr(self, 'pairs'):
1294 ret['pairs'] = [(int(p[0]), int(p[1])) for p in self.pairs]
1295 else:
1296 ret['pairs'] = []
1297
1298 for key, value in list(self.meta.items()):
1299 ret[key] = value
1300
1301 return json.dumps(ret)
1302
1303 @property
1304 def times(self):
1305 '''
1306 Return the list of times of the current data
1307 '''
1308
1309 ret = numpy.array(self.__times)
1310 ret.sort()
1311 return ret
1312
1313 @property
1314 def min_time(self):
1315 '''
1316 Return the minimun time value
1317 '''
1318
1319 return self.times[0]
1320
1321 @property
1322 def max_time(self):
1323 '''
1324 Return the maximun time value
1325 '''
1326
1327 return self.times[-1]
1328
1329 @property
1330 def heights(self):
1331 '''
1332 Return the list of heights of the current data
1333 '''
1334
1335 return numpy.array(self.__heights[-1])
1336
1337 @staticmethod
1338 def roundFloats(obj):
1339 if isinstance(obj, list):
1340 return list(map(PlotterData.roundFloats, obj))
1341 elif isinstance(obj, float):
1342 return round(obj, 2)
Show file before | Show file after | Hide comments
@@ -4,4 +4,3 from .jroplot_heispectra import *
4 4 from .jroplot_correlation import *
5 5 from .jroplot_parameters import *
6 6 from .jroplot_data import *
7 from .jroplotter import *
Show file before | Show file after | Hide comments
@@ -5,7 +5,7 import copy
5 5 from schainpy.model import *
6 6 from .figure import Figure, isRealtime
7 7
8 class CorrelationPlot(Figure):
8 class CorrelationPlot_(Figure):
9 9 isConfig = None
10 10 __nsubplots = None
11 11
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (639 lines changed) Show them Hide them
@@ -1,41 +1,32
1 '''
2 New Plots Operations
3
4 @author: juan.espinoza@jro.igp.gob.pe
5 '''
6
1 7
2 import os
3 8 import time
4 import glob
5 9 import datetime
6 from multiprocessing import Process
7
8 import zmq
9 10 import numpy
10 import matplotlib
11 import matplotlib.pyplot as plt
12 from matplotlib.patches import Polygon
13 from mpl_toolkits.axes_grid1 import make_axes_locatable
14 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
15 11
16 from schainpy.model.proc.jroproc_base import Operation
12 from schainpy.model.graphics.jroplot_base import Plot, plt
17 13 from schainpy.utils import log
18 14
19 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
20 blu_values = matplotlib.pyplot.get_cmap(
21 'seismic_r', 20)(numpy.arange(20))[10:15]
22 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
23 'jro', numpy.vstack((blu_values, jet_values)))
24 matplotlib.pyplot.register_cmap(cmap=ncmap)
25
26 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
27
28 15 EARTH_RADIUS = 6.3710e3
29 16
17
30 18 def ll2xy(lat1, lon1, lat2, lon2):
31 19
32 20 p = 0.017453292519943295
33 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
21 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
22 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
34 23 r = 12742 * numpy.arcsin(numpy.sqrt(a))
35 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)*numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
24 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
25 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
36 26 theta = -theta + numpy.pi/2
37 27 return r*numpy.cos(theta), r*numpy.sin(theta)
38 28
29
39 30 def km2deg(km):
40 31 '''
41 32 Convert distance in km to degrees
@@ -43,536 +34,8 def km2deg(km):
43 34
44 35 return numpy.rad2deg(km/EARTH_RADIUS)
45 36
46 def figpause(interval):
47 backend = plt.rcParams['backend']
48 if backend in matplotlib.rcsetup.interactive_bk:
49 figManager = matplotlib._pylab_helpers.Gcf.get_active()
50 if figManager is not None:
51 canvas = figManager.canvas
52 if canvas.figure.stale:
53 canvas.draw()
54 try:
55 canvas.start_event_loop(interval)
56 except:
57 pass
58 return
59
60 def popup(message):
61 '''
62 '''
63
64 fig = plt.figure(figsize=(12, 8), facecolor='r')
65 text = '\n'.join([s.strip() for s in message.split(':')])
66 fig.text(0.01, 0.5, text, ha='left', va='center', size='20', weight='heavy', color='w')
67 fig.show()
68 figpause(1000)
69
70
71 class PlotData(Operation, Process):
72 '''
73 Base class for Schain plotting operations
74 '''
75
76 CODE = 'Figure'
77 colormap = 'jro'
78 bgcolor = 'white'
79 CONFLATE = False
80 __missing = 1E30
81
82 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
83 'zlimits', 'xlabel', 'ylabel', 'xaxis','cb_label', 'title',
84 'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
85 'showprofile', 'decimation', 'ftp']
86
87 def __init__(self, **kwargs):
88
89 Operation.__init__(self, plot=True, **kwargs)
90 Process.__init__(self)
91
92 self.kwargs['code'] = self.CODE
93 self.mp = False
94 self.data = None
95 self.isConfig = False
96 self.figures = []
97 self.axes = []
98 self.cb_axes = []
99 self.localtime = kwargs.pop('localtime', True)
100 self.show = kwargs.get('show', True)
101 self.save = kwargs.get('save', False)
102 self.ftp = kwargs.get('ftp', False)
103 self.colormap = kwargs.get('colormap', self.colormap)
104 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
105 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
106 self.colormaps = kwargs.get('colormaps', None)
107 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
108 self.showprofile = kwargs.get('showprofile', False)
109 self.title = kwargs.get('wintitle', self.CODE.upper())
110 self.cb_label = kwargs.get('cb_label', None)
111 self.cb_labels = kwargs.get('cb_labels', None)
112 self.labels = kwargs.get('labels', None)
113 self.xaxis = kwargs.get('xaxis', 'frequency')
114 self.zmin = kwargs.get('zmin', None)
115 self.zmax = kwargs.get('zmax', None)
116 self.zlimits = kwargs.get('zlimits', None)
117 self.xmin = kwargs.get('xmin', None)
118 self.xmax = kwargs.get('xmax', None)
119 self.xrange = kwargs.get('xrange', 24)
120 self.xscale = kwargs.get('xscale', None)
121 self.ymin = kwargs.get('ymin', None)
122 self.ymax = kwargs.get('ymax', None)
123 self.yscale = kwargs.get('yscale', None)
124 self.xlabel = kwargs.get('xlabel', None)
125 self.decimation = kwargs.get('decimation', None)
126 self.showSNR = kwargs.get('showSNR', False)
127 self.oneFigure = kwargs.get('oneFigure', True)
128 self.width = kwargs.get('width', None)
129 self.height = kwargs.get('height', None)
130 self.colorbar = kwargs.get('colorbar', True)
131 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
132 self.channels = kwargs.get('channels', None)
133 self.titles = kwargs.get('titles', [])
134 self.polar = False
135 self.grid = kwargs.get('grid', False)
136
137 def __fmtTime(self, x, pos):
138 '''
139 '''
140
141 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
142
143 def __setup(self):
144 '''
145 Common setup for all figures, here figures and axes are created
146 '''
147
148 if self.CODE not in self.data:
149 raise ValueError(log.error('Missing data for {}'.format(self.CODE),
150 self.name))
151
152 self.setup()
153
154 self.time_label = 'LT' if self.localtime else 'UTC'
155 if self.data.localtime:
156 self.getDateTime = datetime.datetime.fromtimestamp
157 else:
158 self.getDateTime = datetime.datetime.utcfromtimestamp
159
160 if self.width is None:
161 self.width = 8
162
163 self.figures = []
164 self.axes = []
165 self.cb_axes = []
166 self.pf_axes = []
167 self.cmaps = []
168
169 size = '15%' if self.ncols == 1 else '30%'
170 pad = '4%' if self.ncols == 1 else '8%'
171
172 if self.oneFigure:
173 if self.height is None:
174 self.height = 1.4 * self.nrows + 1
175 fig = plt.figure(figsize=(self.width, self.height),
176 edgecolor='k',
177 facecolor='w')
178 self.figures.append(fig)
179 for n in range(self.nplots):
180 ax = fig.add_subplot(self.nrows, self.ncols,
181 n + 1, polar=self.polar)
182 ax.tick_params(labelsize=8)
183 ax.firsttime = True
184 ax.index = 0
185 ax.press = None
186 self.axes.append(ax)
187 if self.showprofile:
188 cax = self.__add_axes(ax, size=size, pad=pad)
189 cax.tick_params(labelsize=8)
190 self.pf_axes.append(cax)
191 else:
192 if self.height is None:
193 self.height = 3
194 for n in range(self.nplots):
195 fig = plt.figure(figsize=(self.width, self.height),
196 edgecolor='k',
197 facecolor='w')
198 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
199 ax.tick_params(labelsize=8)
200 ax.firsttime = True
201 ax.index = 0
202 ax.press = None
203 self.figures.append(fig)
204 self.axes.append(ax)
205 if self.showprofile:
206 cax = self.__add_axes(ax, size=size, pad=pad)
207 cax.tick_params(labelsize=8)
208 self.pf_axes.append(cax)
209
210 for n in range(self.nrows):
211 if self.colormaps is not None:
212 cmap = plt.get_cmap(self.colormaps[n])
213 else:
214 cmap = plt.get_cmap(self.colormap)
215 cmap.set_bad(self.bgcolor, 1.)
216 self.cmaps.append(cmap)
217
218 for fig in self.figures:
219 fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
220 fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
221 fig.canvas.mpl_connect('button_press_event', self.onBtnPress)
222 fig.canvas.mpl_connect('motion_notify_event', self.onMotion)
223 fig.canvas.mpl_connect('button_release_event', self.onBtnRelease)
224 if self.show:
225 fig.show()
226
227 def OnKeyPress(self, event):
228 '''
229 Event for pressing keys (up, down) change colormap
230 '''
231 ax = event.inaxes
232 if ax in self.axes:
233 if event.key == 'down':
234 ax.index += 1
235 elif event.key == 'up':
236 ax.index -= 1
237 if ax.index < 0:
238 ax.index = len(CMAPS) - 1
239 elif ax.index == len(CMAPS):
240 ax.index = 0
241 cmap = CMAPS[ax.index]
242 ax.cbar.set_cmap(cmap)
243 ax.cbar.draw_all()
244 ax.plt.set_cmap(cmap)
245 ax.cbar.patch.figure.canvas.draw()
246 self.colormap = cmap.name
247
248 def OnBtnScroll(self, event):
249 '''
250 Event for scrolling, scale figure
251 '''
252 cb_ax = event.inaxes
253 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
254 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
255 pt = ax.cbar.ax.bbox.get_points()[:, 1]
256 nrm = ax.cbar.norm
257 vmin, vmax, p0, p1, pS = (
258 nrm.vmin, nrm.vmax, pt[0], pt[1], event.y)
259 scale = 2 if event.step == 1 else 0.5
260 point = vmin + (vmax - vmin) / (p1 - p0) * (pS - p0)
261 ax.cbar.norm.vmin = point - scale * (point - vmin)
262 ax.cbar.norm.vmax = point - scale * (point - vmax)
263 ax.plt.set_norm(ax.cbar.norm)
264 ax.cbar.draw_all()
265 ax.cbar.patch.figure.canvas.draw()
266
267 def onBtnPress(self, event):
268 '''
269 Event for mouse button press
270 '''
271 cb_ax = event.inaxes
272 if cb_ax is None:
273 return
274
275 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
276 cb_ax.press = event.x, event.y
277 else:
278 cb_ax.press = None
279
280 def onMotion(self, event):
281 '''
282 Event for move inside colorbar
283 '''
284 cb_ax = event.inaxes
285 if cb_ax is None:
286 return
287 if cb_ax not in [ax.cbar.ax for ax in self.axes if ax.cbar]:
288 return
289 if cb_ax.press is None:
290 return
291
292 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
293 xprev, yprev = cb_ax.press
294 dx = event.x - xprev
295 dy = event.y - yprev
296 cb_ax.press = event.x, event.y
297 scale = ax.cbar.norm.vmax - ax.cbar.norm.vmin
298 perc = 0.03
299
300 if event.button == 1:
301 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
302 ax.cbar.norm.vmax -= (perc * scale) * numpy.sign(dy)
303 elif event.button == 3:
304 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
305 ax.cbar.norm.vmax += (perc * scale) * numpy.sign(dy)
306
307 ax.cbar.draw_all()
308 ax.plt.set_norm(ax.cbar.norm)
309 ax.cbar.patch.figure.canvas.draw()
310
311 def onBtnRelease(self, event):
312 '''
313 Event for mouse button release
314 '''
315 cb_ax = event.inaxes
316 if cb_ax is not None:
317 cb_ax.press = None
318
319 def __add_axes(self, ax, size='30%', pad='8%'):
320 '''
321 Add new axes to the given figure
322 '''
323 divider = make_axes_locatable(ax)
324 nax = divider.new_horizontal(size=size, pad=pad)
325 ax.figure.add_axes(nax)
326 return nax
327
328 self.setup()
329
330 def setup(self):
331 '''
332 This method should be implemented in the child class, the following
333 attributes should be set:
334
335 self.nrows: number of rows
336 self.ncols: number of cols
337 self.nplots: number of plots (channels or pairs)
338 self.ylabel: label for Y axes
339 self.titles: list of axes title
340
341 '''
342 raise NotImplementedError
343
344 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
345 '''
346 Create a masked array for missing data
347 '''
348 if x_buffer.shape[0] < 2:
349 return x_buffer, y_buffer, z_buffer
350
351 deltas = x_buffer[1:] - x_buffer[0:-1]
352 x_median = numpy.median(deltas)
353 37
354 index = numpy.where(deltas > 5 * x_median)
355
356 if len(index[0]) != 0:
357 z_buffer[::, index[0], ::] = self.__missing
358 z_buffer = numpy.ma.masked_inside(z_buffer,
359 0.99 * self.__missing,
360 1.01 * self.__missing)
361
362 return x_buffer, y_buffer, z_buffer
363
364 def decimate(self):
365
366 # dx = int(len(self.x)/self.__MAXNUMX) + 1
367 dy = int(len(self.y) / self.decimation) + 1
368
369 # x = self.x[::dx]
370 x = self.x
371 y = self.y[::dy]
372 z = self.z[::, ::, ::dy]
373
374 return x, y, z
375
376 def format(self):
377 '''
378 Set min and max values, labels, ticks and titles
379 '''
380
381 if self.xmin is None:
382 xmin = self.min_time
383 else:
384 if self.xaxis is 'time':
385 dt = self.getDateTime(self.min_time)
386 xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
387 datetime.datetime(1970, 1, 1)).total_seconds()
388 if self.data.localtime:
389 xmin += time.timezone
390 else:
391 xmin = self.xmin
392
393 if self.xmax is None:
394 xmax = xmin + self.xrange * 60 * 60
395 else:
396 if self.xaxis is 'time':
397 dt = self.getDateTime(self.max_time)
398 xmax = (dt.replace(hour=int(self.xmax), minute=59, second=59) -
399 datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds()
400 if self.data.localtime:
401 xmax += time.timezone
402 else:
403 xmax = self.xmax
404
405 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
406 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
407
408 Y = numpy.array([1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000])
409 i = 1 if numpy.where(abs(ymax-ymin) <= Y)[0][0] < 0 else numpy.where(abs(ymax-ymin) <= Y)[0][0]
410 ystep = Y[i] / 10.
411
412 if self.xaxis is not 'time':
413 X = numpy.array([1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000])/2.
414 i = 1 if numpy.where(abs(xmax-xmin) <= X)[0][0] < 0 else numpy.where(abs(xmax-xmin) <= X)[0][0]
415 xstep = X[i] / 10.
416
417 for n, ax in enumerate(self.axes):
418 if ax.firsttime:
419 ax.set_facecolor(self.bgcolor)
420 ax.yaxis.set_major_locator(MultipleLocator(ystep))
421 if self.xscale:
422 ax.xaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.xscale)))
423 if self.xscale:
424 ax.yaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.yscale)))
425 if self.xaxis is 'time':
426 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
427 ax.xaxis.set_major_locator(LinearLocator(9))
428 else:
429 ax.xaxis.set_major_locator(MultipleLocator(xstep))
430 if self.xlabel is not None:
431 ax.set_xlabel(self.xlabel)
432 ax.set_ylabel(self.ylabel)
433 ax.firsttime = False
434 if self.showprofile:
435 self.pf_axes[n].set_ylim(ymin, ymax)
436 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
437 self.pf_axes[n].set_xlabel('dB')
438 self.pf_axes[n].grid(b=True, axis='x')
439 [tick.set_visible(False)
440 for tick in self.pf_axes[n].get_yticklabels()]
441 if self.colorbar:
442 ax.cbar = plt.colorbar(
443 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
444 ax.cbar.ax.tick_params(labelsize=8)
445 ax.cbar.ax.press = None
446 if self.cb_label:
447 ax.cbar.set_label(self.cb_label, size=8)
448 elif self.cb_labels:
449 ax.cbar.set_label(self.cb_labels[n], size=8)
450 else:
451 ax.cbar = None
452 if self.grid:
453 ax.grid(True)
454
455 if not self.polar:
456 ax.set_xlim(xmin, xmax)
457 ax.set_ylim(ymin, ymax)
458 ax.set_title('{} {} {}'.format(
459 self.titles[n],
460 self.getDateTime(self.max_time).strftime('%Y-%m-%dT%H:%M:%S'),
461 self.time_label),
462 size=8)
463 else:
464 ax.set_title('{}'.format(self.titles[n]), size=8)
465 ax.set_ylim(0, 90)
466 ax.set_yticks(numpy.arange(0, 90, 20))
467 ax.yaxis.labelpad = 40
468
469 def __plot(self):
470 '''
471 '''
472 log.log('Plotting', self.name)
473
474 try:
475 self.plot()
476 self.format()
477 except Exception as e:
478 log.warning('{} Plot could not be updated... check data'.format(self.CODE), self.name)
479 log.error(str(e), '')
480 return
481
482 for n, fig in enumerate(self.figures):
483 if self.nrows == 0 or self.nplots == 0:
484 log.warning('No data', self.name)
485 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
486 fig.canvas.manager.set_window_title(self.CODE)
487 continue
488
489 fig.tight_layout()
490 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
491 self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
492 fig.canvas.draw()
493
494 if self.save and (self.data.ended or not self.data.buffering):
495
496 if self.save_labels:
497 labels = self.save_labels
498 else:
499 labels = list(range(self.nrows))
500
501 if self.oneFigure:
502 label = ''
503 else:
504 label = '-{}'.format(labels[n])
505 figname = os.path.join(
506 self.save,
507 self.CODE,
508 '{}{}_{}.png'.format(
509 self.CODE,
510 label,
511 self.getDateTime(self.saveTime).strftime(
512 '%Y%m%d_%H%M%S'),
513 )
514 )
515 log.log('Saving figure: {}'.format(figname), self.name)
516 if not os.path.isdir(os.path.dirname(figname)):
517 os.makedirs(os.path.dirname(figname))
518 fig.savefig(figname)
519
520 def plot(self):
521 '''
522 '''
523 raise NotImplementedError
524
525 def run(self):
526
527 log.log('Starting', self.name)
528
529 context = zmq.Context()
530 receiver = context.socket(zmq.SUB)
531 receiver.setsockopt(zmq.SUBSCRIBE, '')
532 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
533
534 if 'server' in self.kwargs['parent']:
535 receiver.connect(
536 'ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
537 else:
538 receiver.connect("ipc:///tmp/zmq.plots")
539
540 while True:
541 try:
542 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
543 if self.data.localtime and self.localtime:
544 self.times = self.data.times
545 elif self.data.localtime and not self.localtime:
546 self.times = self.data.times + time.timezone
547 elif not self.data.localtime and self.localtime:
548 self.times = self.data.times - time.timezone
549 else:
550 self.times = self.data.times
551
552 self.min_time = self.times[0]
553 self.max_time = self.times[-1]
554
555 if self.isConfig is False:
556 self.__setup()
557 self.isConfig = True
558
559 self.__plot()
560
561 except zmq.Again as e:
562 if self.data and self.data.ended:
563 break
564 log.log('Waiting for data...')
565 if self.data:
566 figpause(self.data.throttle)
567 else:
568 time.sleep(2)
569
570 def close(self):
571 if self.data:
572 self.__plot()
573
574
575 class PlotSpectraData(PlotData):
38 class SpectraPlot(Plot):
576 39 '''
577 40 Plot for Spectra data
578 41 '''
@@ -644,10 +107,9 class PlotSpectraData(PlotData):
644 107 ax.plt_mean.set_data(mean, y)
645 108
646 109 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
647 self.saveTime = self.max_time
648 110
649 111
650 class PlotCrossSpectraData(PlotData):
112 class CrossSpectraPlot(Plot):
651 113
652 114 CODE = 'cspc'
653 115 zmin_coh = None
@@ -741,10 +203,8 class PlotCrossSpectraData(PlotData):
741 203 ax.plt.set_array(phase.T.ravel())
742 204 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
743 205
744 self.saveTime = self.max_time
745
746 206
747 class PlotSpectraMeanData(PlotSpectraData):
207 class SpectraMeanPlot(SpectraPlot):
748 208 '''
749 209 Plot for Spectra and Mean
750 210 '''
@@ -752,7 +212,7 class PlotSpectraMeanData(PlotSpectraData):
752 212 colormap = 'jro'
753 213
754 214
755 class PlotRTIData(PlotData):
215 class RTIPlot(Plot):
756 216 '''
757 217 Plot for RTI data
758 218 '''
@@ -771,7 +231,7 class PlotRTIData(PlotData):
771 231 self.CODE.upper(), x) for x in range(self.nrows)]
772 232
773 233 def plot(self):
774 self.x = self.times
234 self.x = self.data.times
775 235 self.y = self.data.heights
776 236 self.z = self.data[self.CODE]
777 237 self.z = numpy.ma.masked_invalid(self.z)
@@ -807,10 +267,8 class PlotRTIData(PlotData):
807 267 ax.plot_noise.set_data(numpy.repeat(
808 268 self.data['noise'][n][-1], len(self.y)), self.y)
809 269
810 self.saveTime = self.min_time
811 270
812
813 class PlotCOHData(PlotRTIData):
271 class CoherencePlot(RTIPlot):
814 272 '''
815 273 Plot for Coherence data
816 274 '''
@@ -833,7 +291,7 class PlotCOHData(PlotRTIData):
833 291 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
834 292
835 293
836 class PlotPHASEData(PlotCOHData):
294 class PhasePlot(CoherencePlot):
837 295 '''
838 296 Plot for Phase map data
839 297 '''
@@ -842,7 +300,7 class PlotPHASEData(PlotCOHData):
842 300 colormap = 'seismic'
843 301
844 302
845 class PlotNoiseData(PlotData):
303 class NoisePlot(Plot):
846 304 '''
847 305 Plot for noise
848 306 '''
@@ -860,8 +318,8 class PlotNoiseData(PlotData):
860 318
861 319 def plot(self):
862 320
863 x = self.times
864 xmin = self.min_time
321 x = self.data.times
322 xmin = self.data.min_time
865 323 xmax = xmin + self.xrange * 60 * 60
866 324 Y = self.data[self.CODE]
867 325
@@ -877,10 +335,9 class PlotNoiseData(PlotData):
877 335
878 336 self.ymin = numpy.nanmin(Y) - 5
879 337 self.ymax = numpy.nanmax(Y) + 5
880 self.saveTime = self.min_time
881 338
882 339
883 class PlotSNRData(PlotRTIData):
340 class SnrPlot(RTIPlot):
884 341 '''
885 342 Plot for SNR Data
886 343 '''
@@ -889,7 +346,7 class PlotSNRData(PlotRTIData):
889 346 colormap = 'jet'
890 347
891 348
892 class PlotDOPData(PlotRTIData):
349 class DopplerPlot(RTIPlot):
893 350 '''
894 351 Plot for DOPPLER Data
895 352 '''
@@ -898,7 +355,7 class PlotDOPData(PlotRTIData):
898 355 colormap = 'jet'
899 356
900 357
901 class PlotSkyMapData(PlotData):
358 class SkyMapPlot(Plot):
902 359 '''
903 360 Plot for meteors detection data
904 361 '''
@@ -938,16 +395,15 class PlotSkyMapData(PlotData):
938 395 else:
939 396 ax.plot.set_data(x, y)
940 397
941 dt1 = self.getDateTime(self.min_time).strftime('%y/%m/%d %H:%M:%S')
942 dt2 = self.getDateTime(self.max_time).strftime('%y/%m/%d %H:%M:%S')
398 dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
399 dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
943 400 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
944 401 dt2,
945 402 len(x))
946 403 self.titles[0] = title
947 self.saveTime = self.max_time
948 404
949 405
950 class PlotParamData(PlotRTIData):
406 class ParametersPlot(RTIPlot):
951 407 '''
952 408 Plot for data_param object
953 409 '''
@@ -973,7 +429,7 class PlotParamData(PlotRTIData):
973 429
974 430 def plot(self):
975 431 self.data.normalize_heights()
976 self.x = self.times
432 self.x = self.data.times
977 433 self.y = self.data.heights
978 434 if self.showSNR:
979 435 self.z = numpy.concatenate(
@@ -1015,10 +471,8 class PlotParamData(PlotRTIData):
1015 471 cmap=self.cmaps[n]
1016 472 )
1017 473
1018 self.saveTime = self.min_time
1019
1020 474
1021 class PlotOutputData(PlotParamData):
475 class OutputPlot(ParametersPlot):
1022 476 '''
1023 477 Plot data_output object
1024 478 '''
@@ -1027,9 +481,9 class PlotOutputData(PlotParamData):
1027 481 colormap = 'seismic'
1028 482
1029 483
1030 class PlotPolarMapData(PlotData):
484 class PolarMapPlot(Plot):
1031 485 '''
1032 Plot for meteors detection data
486 Plot for weather radar
1033 487 '''
1034 488
1035 489 CODE = 'param'
@@ -1058,8 +512,10 class PlotPolarMapData(PlotData):
1058 512 self.cb_labels = self.data.meta['units']
1059 513 self.lat = self.data.meta['latitude']
1060 514 self.lon = self.data.meta['longitude']
1061 self.xmin, self.xmax = float(km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
1062 self.ymin, self.ymax = float(km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
515 self.xmin, self.xmax = float(
516 km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
517 self.ymin, self.ymax = float(
518 km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
1063 519 # self.polar = True
1064 520
1065 521 def plot(self):
@@ -1067,11 +523,13 class PlotPolarMapData(PlotData):
1067 523 for n, ax in enumerate(self.axes):
1068 524 data = self.data['param'][self.channels[n]]
1069 525
1070 zeniths = numpy.linspace(0, self.data.meta['max_range'], data.shape[1])
526 zeniths = numpy.linspace(
527 0, self.data.meta['max_range'], data.shape[1])
1071 528 if self.mode == 'E':
1072 529 azimuths = -numpy.radians(self.data.heights)+numpy.pi/2
1073 530 r, theta = numpy.meshgrid(zeniths, azimuths)
1074 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
531 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
532 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
1075 533 x = km2deg(x) + self.lon
1076 534 y = km2deg(y) + self.lat
1077 535 else:
@@ -1108,7 +566,8 class PlotPolarMapData(PlotData):
1108 566 lat = float(lat)
1109 567 lon = float(lon)
1110 568 # ax.plot(lon, lat, '.b', ms=2)
1111 ax.text(lon, lat, label.decode('utf8'), ha='center', va='bottom', size='8', color='black')
569 ax.text(lon, lat, label.decode('utf8'), ha='center',
570 va='bottom', size='8', color='black')
1112 571
1113 572 # plot limites
1114 573 limites =[]
@@ -1122,7 +581,8 class PlotPolarMapData(PlotData):
1122 581 values = line.strip().split(',')
1123 582 tmp.append((float(values[0]), float(values[1])))
1124 583 for points in limites:
1125 ax.add_patch(Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
584 ax.add_patch(
585 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
1126 586
1127 587 # plot Cuencas
1128 588 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
@@ -1133,7 +593,8 class PlotPolarMapData(PlotData):
1133 593
1134 594 # plot grid
1135 595 for r in (15, 30, 45, 60):
1136 ax.add_artist(plt.Circle((self.lon, self.lat), km2deg(r), color='0.6', fill=False, lw=0.2))
596 ax.add_artist(plt.Circle((self.lon, self.lat),
597 km2deg(r), color='0.6', fill=False, lw=0.2))
1137 598 ax.text(
1138 599 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
1139 600 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
@@ -1148,7 +609,5 class PlotPolarMapData(PlotData):
1148 609 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
1149 610
1150 611 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
1151 self.titles = ['{} {}'.format(self.data.parameters[x], title) for x in self.channels]
1152 self.saveTime = self.max_time
1153
1154 No newline at end of file
612 self.titles = ['{} {}'.format(
613 self.data.parameters[x], title) for x in self.channels]
Show file before | Show file after | Hide comments
@@ -10,7 +10,7 import numpy
10 10 from .figure import Figure, isRealtime
11 11 from .plotting_codes import *
12 12
13 class SpectraHeisScope(Figure):
13 class SpectraHeisScope_(Figure):
14 14
15 15
16 16 isConfig = None
@@ -173,7 +173,7 class SpectraHeisScope(Figure):
173 173 wr_period=wr_period,
174 174 thisDatetime=thisDatetime)
175 175
176 class RTIfromSpectraHeis(Figure):
176 class RTIfromSpectraHeis_(Figure):
177 177
178 178 isConfig = None
179 179 __nsubplots = None
Show file before | Show file after | Hide comments
@@ -7,7 +7,7 from .plotting_codes import *
7 7 from schainpy.model.proc.jroproc_base import MPDecorator
8 8 from schainpy.utils import log
9 9
10 class FitGauPlot(Figure):
10 class FitGauPlot_(Figure):
11 11
12 12 isConfig = None
13 13 __nsubplots = None
@@ -218,7 +218,7 class FitGauPlot(Figure):
218 218
219 219
220 220
221 class MomentsPlot(Figure):
221 class MomentsPlot_(Figure):
222 222
223 223 isConfig = None
224 224 __nsubplots = None
@@ -405,7 +405,7 class MomentsPlot(Figure):
405 405 thisDatetime=thisDatetime)
406 406
407 407
408 class SkyMapPlot(Figure):
408 class SkyMapPlot_(Figure):
409 409
410 410 __isConfig = None
411 411 __nsubplots = None
@@ -561,7 +561,7 class SkyMapPlot(Figure):
561 561
562 562
563 563
564 class WindProfilerPlot(Figure):
564 class WindProfilerPlot_(Figure):
565 565
566 566 __isConfig = None
567 567 __nsubplots = None
@@ -774,7 +774,7 class WindProfilerPlot(Figure):
774 774 update_figfile = True
775 775
776 776 @MPDecorator
777 class ParametersPlot(Figure):
777 class ParametersPlot_(Figure):
778 778
779 779 __isConfig = None
780 780 __nsubplots = None
@@ -986,7 +986,7 class ParametersPlot(Figure):
986 986
987 987 return dataOut
988 988 @MPDecorator
989 class Parameters1Plot(Figure):
989 class Parameters1Plot_(Figure):
990 990
991 991 __isConfig = None
992 992 __nsubplots = None
@@ -1237,7 +1237,7 class Parameters1Plot(Figure):
1237 1237 update_figfile=False)
1238 1238 return dataOut
1239 1239
1240 class SpectralFittingPlot(Figure):
1240 class SpectralFittingPlot_(Figure):
1241 1241
1242 1242 __isConfig = None
1243 1243 __nsubplots = None
@@ -1415,7 +1415,7 class SpectralFittingPlot(Figure):
1415 1415 thisDatetime=thisDatetime)
1416 1416
1417 1417
1418 class EWDriftsPlot(Figure):
1418 class EWDriftsPlot_(Figure):
1419 1419
1420 1420 __isConfig = None
1421 1421 __nsubplots = None
@@ -1621,7 +1621,7 class EWDriftsPlot(Figure):
1621 1621
1622 1622
1623 1623
1624 class PhasePlot(Figure):
1624 class PhasePlot_(Figure):
1625 1625
1626 1626 __isConfig = None
1627 1627 __nsubplots = None
@@ -1785,7 +1785,7 class PhasePlot(Figure):
1785 1785
1786 1786
1787 1787
1788 class NSMeteorDetection1Plot(Figure):
1788 class NSMeteorDetection1Plot_(Figure):
1789 1789
1790 1790 isConfig = None
1791 1791 __nsubplots = None
@@ -1969,7 +1969,7 class NSMeteorDetection1Plot(Figure):
1969 1969 thisDatetime=thisDatetime)
1970 1970
1971 1971
1972 class NSMeteorDetection2Plot(Figure):
1972 class NSMeteorDetection2Plot_(Figure):
1973 1973
1974 1974 isConfig = None
1975 1975 __nsubplots = None
Show file before | Show file after | Hide comments
@@ -14,7 +14,7 from schainpy.model.proc.jroproc_base import MPDecorator
14 14 from schainpy.utils import log
15 15
16 16 @MPDecorator
17 class SpectraPlot(Figure):
17 class SpectraPlot_(Figure):
18 18
19 19 isConfig = None
20 20 __nsubplots = None
@@ -226,7 +226,7 class SpectraPlot(Figure):
226 226
227 227 return dataOut
228 228 @MPDecorator
229 class CrossSpectraPlot(Figure):
229 class CrossSpectraPlot_(Figure):
230 230
231 231 isConfig = None
232 232 __nsubplots = None
@@ -453,7 +453,7 class CrossSpectraPlot(Figure):
453 453 return dataOut
454 454
455 455 @MPDecorator
456 class RTIPlot(Figure):
456 class RTIPlot_(Figure):
457 457
458 458 __isConfig = None
459 459 __nsubplots = None
@@ -667,7 +667,7 class RTIPlot(Figure):
667 667 return dataOut
668 668
669 669 @MPDecorator
670 class CoherenceMap(Figure):
670 class CoherenceMap_(Figure):
671 671 isConfig = None
672 672 __nsubplots = None
673 673
@@ -878,7 +878,7 class CoherenceMap(Figure):
878 878 return dataOut
879 879
880 880 @MPDecorator
881 class PowerProfilePlot(Figure):
881 class PowerProfilePlot_(Figure):
882 882
883 883 isConfig = None
884 884 __nsubplots = None
@@ -1008,7 +1008,7 class PowerProfilePlot(Figure):
1008 1008 return dataOut
1009 1009
1010 1010 @MPDecorator
1011 class SpectraCutPlot(Figure):
1011 class SpectraCutPlot_(Figure):
1012 1012
1013 1013 isConfig = None
1014 1014 __nsubplots = None
@@ -1145,7 +1145,7 class SpectraCutPlot(Figure):
1145 1145 return dataOut
1146 1146
1147 1147 @MPDecorator
1148 class Noise(Figure):
1148 class Noise_(Figure):
1149 1149
1150 1150 isConfig = None
1151 1151 __nsubplots = None
@@ -1352,7 +1352,7 class Noise(Figure):
1352 1352 return dataOut
1353 1353
1354 1354 @MPDecorator
1355 class BeaconPhase(Figure):
1355 class BeaconPhase_(Figure):
1356 1356
1357 1357 __isConfig = None
1358 1358 __nsubplots = None
Show file before | Show file after | Hide comments
@@ -12,7 +12,7 from .figure import Figure
12 12
13 13
14 14 @MPDecorator
15 class Scope(Figure):
15 class Scope_(Figure):
16 16
17 17 isConfig = None
18 18
Show file before | Show file after | Hide comments
@@ -11,13 +11,15 Based on:
11 11 $Author: murco $
12 12 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
13 13 '''
14 from platform import python_version
14
15 15 import inspect
16 16 import zmq
17 17 import time
18 18 import pickle
19 19 import os
20 20 from multiprocessing import Process
21 from zmq.utils.monitor import recv_monitor_message
22
21 23 from schainpy.utils import log
22 24
23 25
@@ -36,21 +38,13 class ProcessingUnit(object):
36 38
37 39 """
38 40
39 METHODS = {}
40 dataIn = None
41 dataInList = []
42 id = None
43 inputId = None
44 dataOut = None
45 dictProcs = None
46 isConfig = False
47
48 41 def __init__(self):
49 42
50 43 self.dataIn = None
51 44 self.dataOut = None
52 45 self.isConfig = False
53 46 self.operations = []
47 self.plots = []
54 48
55 49 def getAllowedArgs(self):
56 50 if hasattr(self, '__attrs__'):
@@ -59,7 +53,6 class ProcessingUnit(object):
59 53 return inspect.getargspec(self.run).args
60 54
61 55 def addOperation(self, conf, operation):
62
63 56 """
64 57 This method is used in the controller, and update the dictionary containing the operations to execute. The dict
65 58 posses the id of the operation process (IPC purposes)
@@ -76,7 +69,11 class ProcessingUnit(object):
76 69 objId : identificador del objeto, necesario para comunicar con master(procUnit)
77 70 """
78 71
79 self.operations.append((operation, conf.type, conf.id, conf.getKwargs()))
72 self.operations.append(
73 (operation, conf.type, conf.id, conf.getKwargs()))
74
75 if 'plot' in self.name.lower():
76 self.plots.append(operation.CODE)
80 77
81 78 def getOperationObj(self, objId):
82 79
@@ -86,7 +83,6 class ProcessingUnit(object):
86 83 return self.operations[objId]
87 84
88 85 def operation(self, **kwargs):
89
90 86 """
91 87 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
92 88 atributos del objeto dataOut
@@ -107,9 +103,10 class ProcessingUnit(object):
107 103 raise NotImplementedError
108 104
109 105 def close(self):
110 #Close every thread, queue or any other object here is it is neccesary.
106
111 107 return
112 108
109
113 110 class Operation(object):
114 111
115 112 """
@@ -126,18 +123,11 class Operation(object):
126 123 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
127 124
128 125 """
129 id = None
130 __buffer = None
131 dest = None
132 isConfig = False
133 readyFlag = None
134 126
135 127 def __init__(self):
136 128
137 self.buffer = None
138 self.dest = None
129 self.id = None
139 130 self.isConfig = False
140 self.readyFlag = False
141 131
142 132 if not hasattr(self, 'name'):
143 133 self.name = self.__class__.__name__
@@ -154,9 +144,7 class Operation(object):
154 144
155 145 raise NotImplementedError
156 146
157
158 147 def run(self, dataIn, **kwargs):
159
160 148 """
161 149 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
162 150 atributos del objeto dataIn.
@@ -180,11 +168,10 class Operation(object):
180 168
181 169 def close(self):
182 170
183 pass
171 return
184 172
185 173
186 174 def MPDecorator(BaseClass):
187
188 175 """
189 176 Multiprocessing class decorator
190 177
@@ -203,25 +190,19 def MPDecorator(BaseClass):
203 190 self.sender = None
204 191 self.receiver = None
205 192 self.name = BaseClass.__name__
193 self.start_time = time.time()
206 194
207 195 if len(self.args) is 3:
208 196 self.typeProc = "ProcUnit"
209 197 self.id = args[0]
210 198 self.inputId = args[1]
211 199 self.project_id = args[2]
212 else:
200 elif len(self.args) is 2:
213 201 self.id = args[0]
214 202 self.inputId = args[0]
215 203 self.project_id = args[1]
216 204 self.typeProc = "Operation"
217 205
218 def getAllowedArgs(self):
219
220 if hasattr(self, '__attrs__'):
221 return self.__attrs__
222 else:
223 return inspect.getargspec(BaseClass.run).args
224
225 206 def subscribe(self):
226 207 '''
227 208 This function create a socket to receive objects from the
@@ -230,7 +211,8 def MPDecorator(BaseClass):
230 211
231 212 c = zmq.Context()
232 213 self.receiver = c.socket(zmq.SUB)
233 self.receiver.connect('ipc:///tmp/schain/{}_pub'.format(self.project_id))
214 self.receiver.connect(
215 'ipc:///tmp/schain/{}_pub'.format(self.project_id))
234 216 self.receiver.setsockopt(zmq.SUBSCRIBE, self.inputId.encode())
235 217
236 218 def listen(self):
@@ -239,6 +221,7 def MPDecorator(BaseClass):
239 221 '''
240 222
241 223 data = pickle.loads(self.receiver.recv_multipart()[1])
224
242 225 return data
243 226
244 227 def set_publisher(self):
@@ -249,13 +232,13 def MPDecorator(BaseClass):
249 232 time.sleep(1)
250 233 c = zmq.Context()
251 234 self.sender = c.socket(zmq.PUB)
252 self.sender.connect('ipc:///tmp/schain/{}_sub'.format(self.project_id))
235 self.sender.connect(
236 'ipc:///tmp/schain/{}_sub'.format(self.project_id))
253 237
254 238 def publish(self, data, id):
255 239 '''
256 240 This function publish an object, to a specific topic.
257 241 '''
258
259 242 self.sender.send_multipart([str(id).encode(), pickle.dumps(data)])
260 243
261 244 def runReader(self):
@@ -266,13 +249,7 def MPDecorator(BaseClass):
266 249
267 250 BaseClass.run(self, **self.kwargs)
268 251
269 if self.dataOut.error[0] == -1:
270 log.error(self.dataOut.error[1])
271 self.publish('end', self.id)
272 #self.sender.send_multipart([str(self.project_id).encode(), 'end'.encode()])
273 break
274
275 for op, optype, id, kwargs in self.operations:
252 for op, optype, opId, kwargs in self.operations:
276 253 if optype=='self':
277 254 op(**kwargs)
278 255 elif optype=='other':
@@ -280,11 +257,21 def MPDecorator(BaseClass):
280 257 elif optype=='external':
281 258 self.publish(self.dataOut, opId)
282 259
283 if self.dataOut.flagNoData:
260 if self.dataOut.flagNoData and self.dataOut.error is None:
284 261 continue
285 262
286 263 self.publish(self.dataOut, self.id)
287 264
265 if self.dataOut.error:
266 if self.dataOut.error[0] == -1:
267 log.error(self.dataOut.error[1], self.name)
268 if self.dataOut.error[0] == 1:
269 log.success(self.dataOut.error[1], self.name)
270 # self.sender.send_multipart([str(self.project_id).encode(), 'end'.encode()])
271 break
272
273 time.sleep(1)
274
288 275 def runProc(self):
289 276 '''
290 277 Run function for proccessing units
@@ -293,14 +280,7 def MPDecorator(BaseClass):
293 280 while True:
294 281 self.dataIn = self.listen()
295 282
296 if self.dataIn == 'end':
297 self.publish('end', self.id)
298 for op, optype, opId, kwargs in self.operations:
299 if optype == 'external':
300 self.publish('end', opId)
301 break
302
303 if self.dataIn.flagNoData:
283 if self.dataIn.flagNoData and self.dataIn.error is None:
304 284 continue
305 285
306 286 BaseClass.run(self, **self.kwargs)
@@ -313,25 +293,28 def MPDecorator(BaseClass):
313 293 elif optype=='external':
314 294 self.publish(self.dataOut, opId)
315 295
316 if self.dataOut.flagNoData:
317 continue
318
319 296 self.publish(self.dataOut, self.id)
297 if self.dataIn.error:
298 break
299
300 time.sleep(1)
320 301
321 302 def runOp(self):
322 303 '''
323 Run function for operations
304 Run function for external operations (this operations just receive data
305 ex: plots, writers, publishers)
324 306 '''
325 307
326 308 while True:
327 309
328 310 dataOut = self.listen()
329 311
330 if dataOut == 'end':
331 break
332
333 312 BaseClass.run(self, dataOut, **self.kwargs)
334 313
314 if dataOut.error:
315 break
316 time.sleep(1)
317
335 318 def run(self):
336 319
337 320 if self.typeProc is "ProcUnit":
@@ -353,15 +336,41 def MPDecorator(BaseClass):
353 336 else:
354 337 raise ValueError("Unknown type")
355 338
356 print("%s done" % BaseClass.__name__)
357 339 self.close()
358 340
341 def event_monitor(self, monitor):
342
343 events = {}
344
345 for name in dir(zmq):
346 if name.startswith('EVENT_'):
347 value = getattr(zmq, name)
348 events[value] = name
349
350 while monitor.poll():
351 evt = recv_monitor_message(monitor)
352 if evt['event'] == 32:
353 self.connections += 1
354 if evt['event'] == 512:
355 pass
356
357 evt.update({'description': events[evt['event']]})
358
359 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
360 break
361 monitor.close()
362 print('event monitor thread done!')
363
359 364 def close(self):
360 365
366 BaseClass.close(self)
367
361 368 if self.sender:
362 369 self.sender.close()
363 370
364 371 if self.receiver:
365 372 self.receiver.close()
366 373
374 log.success('Done...(Time:{:4.2f} secs)'.format(time.time()-self.start_time), self.name)
375
367 376 return MPClass
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (566 lines changed) Show them Hide them
@@ -7,11 +7,9 import glob
7 7 import time
8 8 import json
9 9 import numpy
10 import paho.mqtt.client as mqtt
11 10 import zmq
12 11 import datetime
13 12 import ftplib
14 from zmq.utils.monitor import recv_monitor_message
15 13 from functools import wraps
16 14 from threading import Thread
17 15 from multiprocessing import Process
@@ -54,330 +52,30 def get_plot_code(s):
54 52 else:
55 53 return 24
56 54
57 def roundFloats(obj):
58 if isinstance(obj, list):
59 return list(map(roundFloats, obj))
60 elif isinstance(obj, float):
61 return round(obj, 2)
62
63 55 def decimate(z, MAXNUMY):
64 56 dy = int(len(z[0])/MAXNUMY) + 1
65 57
66 58 return z[::, ::dy]
67 59
68 class throttle(object):
69 '''
70 Decorator that prevents a function from being called more than once every
71 time period.
72 To create a function that cannot be called more than once a minute, but
73 will sleep until it can be called:
74 @throttle(minutes=1)
75 def foo():
76 pass
77
78 for i in range(10):
79 foo()
80 print "This function has run %s times." % i
81 '''
82
83 def __init__(self, seconds=0, minutes=0, hours=0):
84 self.throttle_period = datetime.timedelta(
85 seconds=seconds, minutes=minutes, hours=hours
86 )
87
88 self.time_of_last_call = datetime.datetime.min
89
90 def __call__(self, fn):
91 @wraps(fn)
92 def wrapper(*args, **kwargs):
93 coerce = kwargs.pop('coerce', None)
94 if coerce:
95 self.time_of_last_call = datetime.datetime.now()
96 return fn(*args, **kwargs)
97 else:
98 now = datetime.datetime.now()
99 time_since_last_call = now - self.time_of_last_call
100 time_left = self.throttle_period - time_since_last_call
101
102 if time_left > datetime.timedelta(seconds=0):
103 return
104
105 self.time_of_last_call = datetime.datetime.now()
106 return fn(*args, **kwargs)
107
108 return wrapper
109
110 class Data(object):
111 '''
112 Object to hold data to be plotted
113 '''
114
115 def __init__(self, plottypes, throttle_value, exp_code, buffering=True):
116 self.plottypes = plottypes
117 self.throttle = throttle_value
118 self.exp_code = exp_code
119 self.buffering = buffering
120 self.ended = False
121 self.localtime = False
122 self.meta = {}
123 self.__times = []
124 self.__heights = []
125
126 def __str__(self):
127 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
128 return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times))
129
130 def __len__(self):
131 return len(self.__times)
132
133 def __getitem__(self, key):
134 if key not in self.data:
135 raise KeyError(log.error('Missing key: {}'.format(key)))
136
137 if 'spc' in key or not self.buffering:
138 ret = self.data[key]
139 else:
140 ret = numpy.array([self.data[key][x] for x in self.times])
141 if ret.ndim > 1:
142 ret = numpy.swapaxes(ret, 0, 1)
143 return ret
144
145 def __contains__(self, key):
146 return key in self.data
147
148 def setup(self):
149 '''
150 Configure object
151 '''
152
153 self.type = ''
154 self.ended = False
155 self.data = {}
156 self.__times = []
157 self.__heights = []
158 self.__all_heights = set()
159 for plot in self.plottypes:
160 if 'snr' in plot:
161 plot = 'snr'
162 self.data[plot] = {}
163
164 def shape(self, key):
165 '''
166 Get the shape of the one-element data for the given key
167 '''
168
169 if len(self.data[key]):
170 if 'spc' in key or not self.buffering:
171 return self.data[key].shape
172 return self.data[key][self.__times[0]].shape
173 return (0,)
174
175 def update(self, dataOut, tm):
176 '''
177 Update data object with new dataOut
178 '''
179
180 if tm in self.__times:
181 return
182
183 self.type = dataOut.type
184 self.parameters = getattr(dataOut, 'parameters', [])
185 if hasattr(dataOut, 'pairsList'):
186 self.pairs = dataOut.pairsList
187 if hasattr(dataOut, 'meta'):
188 self.meta = dataOut.meta
189 self.channels = dataOut.channelList
190 self.interval = dataOut.getTimeInterval()
191 self.localtime = dataOut.useLocalTime
192 if 'spc' in self.plottypes or 'cspc' in self.plottypes:
193 self.xrange = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
194 self.__heights.append(dataOut.heightList)
195 self.__all_heights.update(dataOut.heightList)
196 self.__times.append(tm)
197
198 for plot in self.plottypes:
199 if plot == 'spc':
200 z = dataOut.data_spc/dataOut.normFactor
201 buffer = 10*numpy.log10(z)
202 if plot == 'cspc':
203 buffer = dataOut.data_cspc
204 if plot == 'noise':
205 buffer = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
206 if plot == 'rti':
207 buffer = dataOut.getPower()
208 if plot == 'snr_db':
209 buffer = dataOut.data_SNR
210 if plot == 'snr':
211 buffer = 10*numpy.log10(dataOut.data_SNR)
212 if plot == 'dop':
213 buffer = 10*numpy.log10(dataOut.data_DOP)
214 if plot == 'mean':
215 buffer = dataOut.data_MEAN
216 if plot == 'std':
217 buffer = dataOut.data_STD
218 if plot == 'coh':
219 buffer = dataOut.getCoherence()
220 if plot == 'phase':
221 buffer = dataOut.getCoherence(phase=True)
222 if plot == 'output':
223 buffer = dataOut.data_output
224 if plot == 'param':
225 buffer = dataOut.data_param
226
227 if 'spc' in plot:
228 self.data[plot] = buffer
229 else:
230 if self.buffering:
231 self.data[plot][tm] = buffer
232 else:
233 self.data[plot] = buffer
234
235 def normalize_heights(self):
236 '''
237 Ensure same-dimension of the data for different heighList
238 '''
239
240 H = numpy.array(list(self.__all_heights))
241 H.sort()
242 for key in self.data:
243 shape = self.shape(key)[:-1] + H.shape
244 for tm, obj in list(self.data[key].items()):
245 h = self.__heights[self.__times.index(tm)]
246 if H.size == h.size:
247 continue
248 index = numpy.where(numpy.in1d(H, h))[0]
249 dummy = numpy.zeros(shape) + numpy.nan
250 if len(shape) == 2:
251 dummy[:, index] = obj
252 else:
253 dummy[index] = obj
254 self.data[key][tm] = dummy
255
256 self.__heights = [H for tm in self.__times]
257
258 def jsonify(self, decimate=False):
259 '''
260 Convert data to json
261 '''
262
263 data = {}
264 tm = self.times[-1]
265 dy = int(self.heights.size/MAXNUMY) + 1
266 for key in self.data:
267 if key in ('spc', 'cspc') or not self.buffering:
268 dx = int(self.data[key].shape[1]/MAXNUMX) + 1
269 data[key] = roundFloats(self.data[key][::, ::dx, ::dy].tolist())
270 else:
271 data[key] = roundFloats(self.data[key][tm].tolist())
272
273 ret = {'data': data}
274 ret['exp_code'] = self.exp_code
275 ret['time'] = tm
276 ret['interval'] = self.interval
277 ret['localtime'] = self.localtime
278 ret['yrange'] = roundFloats(self.heights[::dy].tolist())
279 if 'spc' in self.data or 'cspc' in self.data:
280 ret['xrange'] = roundFloats(self.xrange[2][::dx].tolist())
281 else:
282 ret['xrange'] = []
283 if hasattr(self, 'pairs'):
284 ret['pairs'] = self.pairs
285 else:
286 ret['pairs'] = []
287
288 for key, value in list(self.meta.items()):
289 ret[key] = value
290
291 return json.dumps(ret)
292
293 @property
294 def times(self):
295 '''
296 Return the list of times of the current data
297 '''
298
299 ret = numpy.array(self.__times)
300 ret.sort()
301 return ret
302
303 @property
304 def heights(self):
305 '''
306 Return the list of heights of the current data
307 '''
308
309 return numpy.array(self.__heights[-1])
310 60
311 61 class PublishData(Operation):
312 62 '''
313 63 Operation to send data over zmq.
314 64 '''
315 65
316 __attrs__ = ['host', 'port', 'delay', 'zeromq', 'mqtt', 'verbose']
66 __attrs__ = ['host', 'port', 'delay', 'verbose']
317 67
318 68 def __init__(self, **kwargs):
319 69 """Inicio."""
320 70 Operation.__init__(self, **kwargs)
321 71 self.isConfig = False
322 self.client = None
323 self.zeromq = None
324 self.mqtt = None
325
326 def on_disconnect(self, client, userdata, rc):
327 if rc != 0:
328 log.warning('Unexpected disconnection.')
329 self.connect()
330 72
331 def connect(self):
332 log.warning('trying to connect')
333 try:
334 self.client.connect(
335 host=self.host,
336 port=self.port,
337 keepalive=60*10,
338 bind_address='')
339 self.client.loop_start()
340 # self.client.publish(
341 # self.topic + 'SETUP',
342 # json.dumps(setup),
343 # retain=True
344 # )
345 except:
346 log.error('MQTT Conection error.')
347 self.client = False
348
349 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, verbose=True, **kwargs):
73 def setup(self, server='zmq.pipe', delay=0, verbose=True, **kwargs):
350 74 self.counter = 0
351 self.topic = kwargs.get('topic', 'schain')
352 75 self.delay = kwargs.get('delay', 0)
353 self.plottype = kwargs.get('plottype', 'spectra')
354 self.host = kwargs.get('host', "10.10.10.82")
355 self.port = kwargs.get('port', 3000)
356 self.clientId = clientId
357 76 self.cnt = 0
358 self.zeromq = zeromq
359 self.mqtt = kwargs.get('plottype', 0)
360 self.client = None
361 77 self.verbose = verbose
362 78 setup = []
363 if mqtt is 1:
364 self.client = mqtt.Client(
365 client_id=self.clientId + self.topic + 'SCHAIN',
366 clean_session=True)
367 self.client.on_disconnect = self.on_disconnect
368 self.connect()
369 for plot in self.plottype:
370 setup.append({
371 'plot': plot,
372 'topic': self.topic + plot,
373 'title': getattr(self, plot + '_' + 'title', False),
374 'xlabel': getattr(self, plot + '_' + 'xlabel', False),
375 'ylabel': getattr(self, plot + '_' + 'ylabel', False),
376 'xrange': getattr(self, plot + '_' + 'xrange', False),
377 'yrange': getattr(self, plot + '_' + 'yrange', False),
378 'zrange': getattr(self, plot + '_' + 'zrange', False),
379 })
380 if zeromq is 1:
381 79 context = zmq.Context()
382 80 self.zmq_socket = context.socket(zmq.PUSH)
383 81 server = kwargs.get('server', 'zmq.pipe')
@@ -393,83 +91,7 class PublishData(Operation):
393 91
394 92 def publish_data(self):
395 93 self.dataOut.finished = False
396 if self.mqtt is 1:
397 yData = self.dataOut.heightList[:2].tolist()
398 if self.plottype == 'spectra':
399 data = getattr(self.dataOut, 'data_spc')
400 z = data/self.dataOut.normFactor
401 zdB = 10*numpy.log10(z)
402 xlen, ylen = zdB[0].shape
403 dx = int(xlen/MAXNUMX) + 1
404 dy = int(ylen/MAXNUMY) + 1
405 Z = [0 for i in self.dataOut.channelList]
406 for i in self.dataOut.channelList:
407 Z[i] = zdB[i][::dx, ::dy].tolist()
408 payload = {
409 'timestamp': self.dataOut.utctime,
410 'data': roundFloats(Z),
411 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
412 'interval': self.dataOut.getTimeInterval(),
413 'type': self.plottype,
414 'yData': yData
415 }
416
417 elif self.plottype in ('rti', 'power'):
418 data = getattr(self.dataOut, 'data_spc')
419 z = data/self.dataOut.normFactor
420 avg = numpy.average(z, axis=1)
421 avgdB = 10*numpy.log10(avg)
422 xlen, ylen = z[0].shape
423 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
424 AVG = [0 for i in self.dataOut.channelList]
425 for i in self.dataOut.channelList:
426 AVG[i] = avgdB[i][::dy].tolist()
427 payload = {
428 'timestamp': self.dataOut.utctime,
429 'data': roundFloats(AVG),
430 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
431 'interval': self.dataOut.getTimeInterval(),
432 'type': self.plottype,
433 'yData': yData
434 }
435 elif self.plottype == 'noise':
436 noise = self.dataOut.getNoise()/self.dataOut.normFactor
437 noisedB = 10*numpy.log10(noise)
438 payload = {
439 'timestamp': self.dataOut.utctime,
440 'data': roundFloats(noisedB.reshape(-1, 1).tolist()),
441 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
442 'interval': self.dataOut.getTimeInterval(),
443 'type': self.plottype,
444 'yData': yData
445 }
446 elif self.plottype == 'snr':
447 data = getattr(self.dataOut, 'data_SNR')
448 avgdB = 10*numpy.log10(data)
449
450 ylen = data[0].size
451 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
452 AVG = [0 for i in self.dataOut.channelList]
453 for i in self.dataOut.channelList:
454 AVG[i] = avgdB[i][::dy].tolist()
455 payload = {
456 'timestamp': self.dataOut.utctime,
457 'data': roundFloats(AVG),
458 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
459 'type': self.plottype,
460 'yData': yData
461 }
462 else:
463 print("Tipo de grafico invalido")
464 payload = {
465 'data': 'None',
466 'timestamp': 'None',
467 'type': None
468 }
469 94
470 self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
471
472 if self.zeromq is 1:
473 95 if self.verbose:
474 96 log.log(
475 97 'Sending {} - {}'.format(self.dataOut.type, self.dataOut.datatime),
@@ -487,14 +109,11 class PublishData(Operation):
487 109 time.sleep(self.delay)
488 110
489 111 def close(self):
490 if self.zeromq is 1:
112
491 113 self.dataOut.finished = True
492 114 self.zmq_socket.send_pyobj(self.dataOut)
493 115 time.sleep(0.1)
494 116 self.zmq_socket.close()
495 if self.client:
496 self.client.loop_stop()
497 self.client.disconnect()
498 117
499 118
500 119 class ReceiverData(ProcessingUnit):
@@ -536,185 +155,6 class ReceiverData(ProcessingUnit):
536 155 'Receiving')
537 156
538 157
539 class PlotterReceiver(ProcessingUnit, Process):
540
541 throttle_value = 5
542 __attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle',
543 'exp_code', 'web_server', 'buffering']
544
545 def __init__(self, **kwargs):
546
547 ProcessingUnit.__init__(self, **kwargs)
548 Process.__init__(self)
549 self.mp = False
550 self.isConfig = False
551 self.isWebConfig = False
552 self.connections = 0
553 server = kwargs.get('server', 'zmq.pipe')
554 web_server = kwargs.get('web_server', None)
555 if 'tcp://' in server:
556 address = server
557 else:
558 address = 'ipc:///tmp/%s' % server
559 self.address = address
560 self.web_address = web_server
561 self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
562 self.realtime = kwargs.get('realtime', False)
563 self.localtime = kwargs.get('localtime', True)
564 self.buffering = kwargs.get('buffering', True)
565 self.throttle_value = kwargs.get('throttle', 5)
566 self.exp_code = kwargs.get('exp_code', None)
567 self.sendData = self.initThrottle(self.throttle_value)
568 self.dates = []
569 self.setup()
570
571 def setup(self):
572
573 self.data = Data(self.plottypes, self.throttle_value, self.exp_code, self.buffering)
574 self.isConfig = True
575
576 def event_monitor(self, monitor):
577
578 events = {}
579
580 for name in dir(zmq):
581 if name.startswith('EVENT_'):
582 value = getattr(zmq, name)
583 events[value] = name
584
585 while monitor.poll():
586 evt = recv_monitor_message(monitor)
587 if evt['event'] == 32:
588 self.connections += 1
589 if evt['event'] == 512:
590 pass
591
592 evt.update({'description': events[evt['event']]})
593
594 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
595 break
596 monitor.close()
597 print('event monitor thread done!')
598
599 def initThrottle(self, throttle_value):
600
601 @throttle(seconds=throttle_value)
602 def sendDataThrottled(fn_sender, data):
603 fn_sender(data)
604
605 return sendDataThrottled
606
607 def send(self, data):
608 log.log('Sending {}'.format(data), self.name)
609 self.sender.send_pyobj(data)
610
611 def run(self):
612
613 log.log(
614 'Starting from {}'.format(self.address),
615 self.name
616 )
617
618 self.context = zmq.Context()
619 self.receiver = self.context.socket(zmq.PULL)
620 self.receiver.bind(self.address)
621 monitor = self.receiver.get_monitor_socket()
622 self.sender = self.context.socket(zmq.PUB)
623 if self.web_address:
624 log.success(
625 'Sending to web: {}'.format(self.web_address),
626 self.name
627 )
628 self.sender_web = self.context.socket(zmq.REQ)
629 self.sender_web.connect(self.web_address)
630 self.poll = zmq.Poller()
631 self.poll.register(self.sender_web, zmq.POLLIN)
632 time.sleep(1)
633
634 if 'server' in self.kwargs:
635 self.sender.bind("ipc:///tmp/{}.plots".format(self.kwargs['server']))
636 else:
637 self.sender.bind("ipc:///tmp/zmq.plots")
638
639 time.sleep(2)
640
641 t = Thread(target=self.event_monitor, args=(monitor,))
642 t.start()
643
644 while True:
645 dataOut = self.receiver.recv_pyobj()
646 if not dataOut.flagNoData:
647 if dataOut.type == 'Parameters':
648 tm = dataOut.utctimeInit
649 else:
650 tm = dataOut.utctime
651 if dataOut.useLocalTime:
652 if not self.localtime:
653 tm += time.timezone
654 dt = datetime.datetime.fromtimestamp(tm).date()
655 else:
656 if self.localtime:
657 tm -= time.timezone
658 dt = datetime.datetime.utcfromtimestamp(tm).date()
659 coerce = False
660 if dt not in self.dates:
661 if self.data:
662 self.data.ended = True
663 self.send(self.data)
664 coerce = True
665 self.data.setup()
666 self.dates.append(dt)
667
668 self.data.update(dataOut, tm)
669
670 if dataOut.finished is True:
671 self.connections -= 1
672 if self.connections == 0 and dt in self.dates:
673 self.data.ended = True
674 self.send(self.data)
675 # self.data.setup()
676 time.sleep(1)
677 break
678 else:
679 if self.realtime:
680 self.send(self.data)
681 if self.web_address:
682 retries = 5
683 while True:
684 self.sender_web.send(self.data.jsonify())
685 socks = dict(self.poll.poll(5000))
686 if socks.get(self.sender_web) == zmq.POLLIN:
687 reply = self.sender_web.recv_string()
688 if reply == 'ok':
689 log.log("Response from server ok", self.name)
690 break
691 else:
692 log.warning("Malformed reply from server: {}".format(reply), self.name)
693
694 else:
695 log.warning("No response from server, retrying...", self.name)
696 self.sender_web.setsockopt(zmq.LINGER, 0)
697 self.sender_web.close()
698 self.poll.unregister(self.sender_web)
699 retries -= 1
700 if retries == 0:
701 log.error("Server seems to be offline, abandoning", self.name)
702 self.sender_web = self.context.socket(zmq.REQ)
703 self.sender_web.connect(self.web_address)
704 self.poll.register(self.sender_web, zmq.POLLIN)
705 time.sleep(1)
706 break
707 self.sender_web = self.context.socket(zmq.REQ)
708 self.sender_web.connect(self.web_address)
709 self.poll.register(self.sender_web, zmq.POLLIN)
710 time.sleep(1)
711 else:
712 self.sendData(self.send, self.data, coerce=coerce)
713 coerce = False
714
715 return
716
717
718 158 class SendToFTP(Operation, Process):
719 159
720 160 '''
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now