##// END OF EJS Templates
schain-jc
RSS
Julia Reader Done!! Task #1085
jespinoza -
r1105:3ff9df258487
parent child
Show More
Side by Side Unified
Context file:
r1105:3ff9df258487 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,955 +1,955
1 1
2 2 import os
3 3 import time
4 4 import glob
5 5 import datetime
6 6 from multiprocessing import Process
7 7
8 8 import zmq
9 9 import numpy
10 10 import matplotlib
11 11 import matplotlib.pyplot as plt
12 12 from mpl_toolkits.axes_grid1 import make_axes_locatable
13 13 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
14 14
15 15 from schainpy.model.proc.jroproc_base import Operation
16 16 from schainpy.utils import log
17 17
18 18 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
19 19 blu_values = matplotlib.pyplot.get_cmap(
20 20 'seismic_r', 20)(numpy.arange(20))[10:15]
21 21 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
22 22 'jro', numpy.vstack((blu_values, jet_values)))
23 23 matplotlib.pyplot.register_cmap(cmap=ncmap)
24 24
25 25 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'RdBu_r', 'seismic')]
26 26
27 27
28 28 def figpause(interval):
29 29 backend = plt.rcParams['backend']
30 30 if backend in matplotlib.rcsetup.interactive_bk:
31 31 figManager = matplotlib._pylab_helpers.Gcf.get_active()
32 32 if figManager is not None:
33 33 canvas = figManager.canvas
34 34 if canvas.figure.stale:
35 35 canvas.draw()
36 36 canvas.start_event_loop(interval)
37 37 return
38 38
39 39
40 40 class PlotData(Operation, Process):
41 41 '''
42 42 Base class for Schain plotting operations
43 43 '''
44 44
45 45 CODE = 'Figure'
46 46 colormap = 'jro'
47 47 bgcolor = 'white'
48 48 CONFLATE = False
49 49 __MAXNUMX = 80
50 50 __missing = 1E30
51 51
52 52 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
53 53 'zlimits', 'xlabel', 'ylabel', 'cb_label', 'title', 'titles', 'colorbar',
54 54 'bgcolor', 'width', 'height', 'localtime', 'oneFigure', 'showprofile']
55 55
56 56 def __init__(self, **kwargs):
57 57
58 58 Operation.__init__(self, plot=True, **kwargs)
59 59 Process.__init__(self)
60 self.contador = 0
60
61 61 self.kwargs['code'] = self.CODE
62 62 self.mp = False
63 63 self.data = None
64 64 self.isConfig = False
65 65 self.figures = []
66 66 self.axes = []
67 67 self.cb_axes = []
68 68 self.localtime = kwargs.pop('localtime', True)
69 69 self.show = kwargs.get('show', True)
70 70 self.save = kwargs.get('save', False)
71 71 self.colormap = kwargs.get('colormap', self.colormap)
72 72 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
73 73 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
74 74 self.colormaps = kwargs.get('colormaps', None)
75 75 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
76 76 self.showprofile = kwargs.get('showprofile', False)
77 77 self.title = kwargs.get('wintitle', self.CODE.upper())
78 78 self.cb_label = kwargs.get('cb_label', None)
79 79 self.cb_labels = kwargs.get('cb_labels', None)
80 80 self.xaxis = kwargs.get('xaxis', 'frequency')
81 81 self.zmin = kwargs.get('zmin', None)
82 82 self.zmax = kwargs.get('zmax', None)
83 83 self.zlimits = kwargs.get('zlimits', None)
84 84 self.xmin = kwargs.get('xmin', None)
85 85 self.xmax = kwargs.get('xmax', None)
86 86 self.xrange = kwargs.get('xrange', 24)
87 87 self.ymin = kwargs.get('ymin', None)
88 88 self.ymax = kwargs.get('ymax', None)
89 89 self.xlabel = kwargs.get('xlabel', None)
90 90 self.__MAXNUMY = kwargs.get('decimation', 300)
91 91 self.showSNR = kwargs.get('showSNR', False)
92 92 self.oneFigure = kwargs.get('oneFigure', True)
93 93 self.width = kwargs.get('width', None)
94 94 self.height = kwargs.get('height', None)
95 95 self.colorbar = kwargs.get('colorbar', True)
96 96 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
97 self.titles = ['' for __ in range(16)]
97 self.titles = kwargs.get('titles', [])
98 98 self.polar = False
99 99
100 100 def __fmtTime(self, x, pos):
101 101 '''
102 102 '''
103 103
104 104 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
105 105
106 106 def __setup(self):
107 107 '''
108 108 Common setup for all figures, here figures and axes are created
109 109 '''
110 110
111 111 if self.CODE not in self.data:
112 112 raise ValueError(log.error('Missing data for {}'.format(self.CODE),
113 113 self.name))
114 114
115 115 self.setup()
116 116
117 117 self.time_label = 'LT' if self.localtime else 'UTC'
118 118 if self.data.localtime:
119 119 self.getDateTime = datetime.datetime.fromtimestamp
120 120 else:
121 121 self.getDateTime = datetime.datetime.utcfromtimestamp
122 122
123 123 if self.width is None:
124 124 self.width = 8
125 125
126 126 self.figures = []
127 127 self.axes = []
128 128 self.cb_axes = []
129 129 self.pf_axes = []
130 130 self.cmaps = []
131 131
132 132 size = '15%' if self.ncols == 1 else '30%'
133 133 pad = '4%' if self.ncols == 1 else '8%'
134 134
135 135 if self.oneFigure:
136 136 if self.height is None:
137 137 self.height = 1.4 * self.nrows + 1
138 138 fig = plt.figure(figsize=(self.width, self.height),
139 139 edgecolor='k',
140 140 facecolor='w')
141 141 self.figures.append(fig)
142 142 for n in range(self.nplots):
143 143 ax = fig.add_subplot(self.nrows, self.ncols,
144 144 n + 1, polar=self.polar)
145 145 ax.tick_params(labelsize=8)
146 146 ax.firsttime = True
147 147 ax.index = 0
148 148 ax.press = None
149 149 self.axes.append(ax)
150 150 if self.showprofile:
151 151 cax = self.__add_axes(ax, size=size, pad=pad)
152 152 cax.tick_params(labelsize=8)
153 153 self.pf_axes.append(cax)
154 154 else:
155 155 if self.height is None:
156 156 self.height = 3
157 157 for n in range(self.nplots):
158 158 fig = plt.figure(figsize=(self.width, self.height),
159 159 edgecolor='k',
160 160 facecolor='w')
161 161 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
162 162 ax.tick_params(labelsize=8)
163 163 ax.firsttime = True
164 164 ax.index = 0
165 165 ax.press = None
166 166 self.figures.append(fig)
167 167 self.axes.append(ax)
168 168 if self.showprofile:
169 169 cax = self.__add_axes(ax, size=size, pad=pad)
170 170 cax.tick_params(labelsize=8)
171 171 self.pf_axes.append(cax)
172 172
173 173 for n in range(self.nrows):
174 174 if self.colormaps is not None:
175 175 cmap = plt.get_cmap(self.colormaps[n])
176 176 else:
177 177 cmap = plt.get_cmap(self.colormap)
178 178 cmap.set_bad(self.bgcolor, 1.)
179 179 self.cmaps.append(cmap)
180 180
181 181 for fig in self.figures:
182 182 fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
183 183 fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
184 184 fig.canvas.mpl_connect('button_press_event', self.onBtnPress)
185 185 fig.canvas.mpl_connect('motion_notify_event', self.onMotion)
186 186 fig.canvas.mpl_connect('button_release_event', self.onBtnRelease)
187 187 if self.show:
188 188 fig.show()
189 189
190 190 def OnKeyPress(self, event):
191 191 '''
192 192 Event for pressing keys (up, down) change colormap
193 193 '''
194 194 ax = event.inaxes
195 195 if ax in self.axes:
196 196 if event.key == 'down':
197 197 ax.index += 1
198 198 elif event.key == 'up':
199 199 ax.index -= 1
200 200 if ax.index < 0:
201 201 ax.index = len(CMAPS) - 1
202 202 elif ax.index == len(CMAPS):
203 203 ax.index = 0
204 204 cmap = CMAPS[ax.index]
205 205 ax.cbar.set_cmap(cmap)
206 206 ax.cbar.draw_all()
207 207 ax.plt.set_cmap(cmap)
208 208 ax.cbar.patch.figure.canvas.draw()
209 209 self.colormap = cmap.name
210 210
211 211 def OnBtnScroll(self, event):
212 212 '''
213 213 Event for scrolling, scale figure
214 214 '''
215 215 cb_ax = event.inaxes
216 216 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
217 217 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
218 218 pt = ax.cbar.ax.bbox.get_points()[:, 1]
219 219 nrm = ax.cbar.norm
220 220 vmin, vmax, p0, p1, pS = (
221 221 nrm.vmin, nrm.vmax, pt[0], pt[1], event.y)
222 222 scale = 2 if event.step == 1 else 0.5
223 223 point = vmin + (vmax - vmin) / (p1 - p0) * (pS - p0)
224 224 ax.cbar.norm.vmin = point - scale * (point - vmin)
225 225 ax.cbar.norm.vmax = point - scale * (point - vmax)
226 226 ax.plt.set_norm(ax.cbar.norm)
227 227 ax.cbar.draw_all()
228 228 ax.cbar.patch.figure.canvas.draw()
229 229
230 230 def onBtnPress(self, event):
231 231 '''
232 232 Event for mouse button press
233 233 '''
234 234 cb_ax = event.inaxes
235 235 if cb_ax is None:
236 236 return
237 237
238 238 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
239 239 cb_ax.press = event.x, event.y
240 240 else:
241 241 cb_ax.press = None
242 242
243 243 def onMotion(self, event):
244 244 '''
245 245 Event for move inside colorbar
246 246 '''
247 247 cb_ax = event.inaxes
248 248 if cb_ax is None:
249 249 return
250 250 if cb_ax not in [ax.cbar.ax for ax in self.axes if ax.cbar]:
251 251 return
252 252 if cb_ax.press is None:
253 253 return
254 254
255 255 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
256 256 xprev, yprev = cb_ax.press
257 257 dx = event.x - xprev
258 258 dy = event.y - yprev
259 259 cb_ax.press = event.x, event.y
260 260 scale = ax.cbar.norm.vmax - ax.cbar.norm.vmin
261 261 perc = 0.03
262 262
263 263 if event.button == 1:
264 264 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
265 265 ax.cbar.norm.vmax -= (perc * scale) * numpy.sign(dy)
266 266 elif event.button == 3:
267 267 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
268 268 ax.cbar.norm.vmax += (perc * scale) * numpy.sign(dy)
269 269
270 270 ax.cbar.draw_all()
271 271 ax.plt.set_norm(ax.cbar.norm)
272 272 ax.cbar.patch.figure.canvas.draw()
273 273
274 274 def onBtnRelease(self, event):
275 275 '''
276 276 Event for mouse button release
277 277 '''
278 278 cb_ax = event.inaxes
279 279 if cb_ax is not None:
280 280 cb_ax.press = None
281 281
282 282 def __add_axes(self, ax, size='30%', pad='8%'):
283 283 '''
284 284 Add new axes to the given figure
285 285 '''
286 286 divider = make_axes_locatable(ax)
287 287 nax = divider.new_horizontal(size=size, pad=pad)
288 288 ax.figure.add_axes(nax)
289 289 return nax
290 290
291 291 self.setup()
292 292
293 293 def setup(self):
294 294 '''
295 295 This method should be implemented in the child class, the following
296 296 attributes should be set:
297 297
298 298 self.nrows: number of rows
299 299 self.ncols: number of cols
300 300 self.nplots: number of plots (channels or pairs)
301 301 self.ylabel: label for Y axes
302 302 self.titles: list of axes title
303 303
304 304 '''
305 305 raise(NotImplementedError, 'Implement this method in child class')
306 306
307 307 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
308 308 '''
309 309 Create a masked array for missing data
310 310 '''
311 311 if x_buffer.shape[0] < 2:
312 312 return x_buffer, y_buffer, z_buffer
313 313
314 314 deltas = x_buffer[1:] - x_buffer[0:-1]
315 315 x_median = numpy.median(deltas)
316 316
317 317 index = numpy.where(deltas > 5 * x_median)
318 318
319 319 if len(index[0]) != 0:
320 320 z_buffer[::, index[0], ::] = self.__missing
321 321 z_buffer = numpy.ma.masked_inside(z_buffer,
322 322 0.99 * self.__missing,
323 323 1.01 * self.__missing)
324 324
325 325 return x_buffer, y_buffer, z_buffer
326 326
327 327 def decimate(self):
328 328
329 329 # dx = int(len(self.x)/self.__MAXNUMX) + 1
330 330 dy = int(len(self.y) / self.__MAXNUMY) + 1
331 331
332 332 # x = self.x[::dx]
333 333 x = self.x
334 334 y = self.y[::dy]
335 335 z = self.z[::, ::, ::dy]
336 336
337 337 return x, y, z
338 338
339 339 def format(self):
340 340 '''
341 341 Set min and max values, labels, ticks and titles
342 342 '''
343 343
344 344 if self.xmin is None:
345 345 xmin = self.min_time
346 346 else:
347 347 if self.xaxis is 'time':
348 348 dt = self.getDateTime(self.min_time)
349 349 xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
350 350 datetime.datetime(1970, 1, 1)).total_seconds()
351 351 if self.data.localtime:
352 352 xmin += time.timezone
353 353 else:
354 354 xmin = self.xmin
355 355
356 356 if self.xmax is None:
357 357 xmax = xmin + self.xrange * 60 * 60
358 358 else:
359 359 if self.xaxis is 'time':
360 360 dt = self.getDateTime(self.max_time)
361 361 xmax = (dt.replace(hour=int(self.xmax), minute=59, second=59) -
362 362 datetime.datetime(1970, 1, 1)).total_seconds()
363 363 if self.data.localtime:
364 364 xmax += time.timezone
365 365 else:
366 366 xmax = self.xmax
367 367
368 368 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
369 369 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
370 370
371 371 Y = numpy.array([10, 20, 50, 100, 200, 500, 1000, 2000])
372 372 i = 1 if numpy.where(ymax < Y)[
373 373 0][0] < 0 else numpy.where(ymax < Y)[0][0]
374 374 ystep = Y[i - 1] / 5
375 375
376 376 for n, ax in enumerate(self.axes):
377 377 if ax.firsttime:
378 378 ax.set_facecolor(self.bgcolor)
379 379 ax.yaxis.set_major_locator(MultipleLocator(ystep))
380 380 if self.xaxis is 'time':
381 381 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
382 382 ax.xaxis.set_major_locator(LinearLocator(9))
383 383 if self.xlabel is not None:
384 384 ax.set_xlabel(self.xlabel)
385 385 ax.set_ylabel(self.ylabel)
386 386 ax.firsttime = False
387 387 if self.showprofile:
388 388 self.pf_axes[n].set_ylim(ymin, ymax)
389 389 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
390 390 self.pf_axes[n].set_xlabel('dB')
391 391 self.pf_axes[n].grid(b=True, axis='x')
392 392 [tick.set_visible(False)
393 393 for tick in self.pf_axes[n].get_yticklabels()]
394 394 if self.colorbar:
395 395 ax.cbar = plt.colorbar(
396 396 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
397 397 ax.cbar.ax.tick_params(labelsize=8)
398 398 ax.cbar.ax.press = None
399 399 if self.cb_label:
400 400 ax.cbar.set_label(self.cb_label, size=8)
401 401 elif self.cb_labels:
402 402 ax.cbar.set_label(self.cb_labels[n], size=8)
403 403 else:
404 404 ax.cbar = None
405 405
406 406 if not self.polar:
407 407 ax.set_xlim(xmin, xmax)
408 408 ax.set_ylim(ymin, ymax)
409 409 ax.set_title('{} - {} {}'.format(
410 410 self.titles[n],
411 411 self.getDateTime(self.max_time).strftime('%H:%M:%S'),
412 412 self.time_label),
413 413 size=8)
414 414 else:
415 415 ax.set_title('{}'.format(self.titles[n]), size=8)
416 416 ax.set_ylim(0, 90)
417 417 ax.set_yticks(numpy.arange(0, 90, 20))
418 418 ax.yaxis.labelpad = 40
419 419
420 420 def __plot(self):
421 421 '''
422 422 '''
423 423 log.success('Plotting', self.name)
424 424
425 425 self.plot()
426 426 self.format()
427 427
428 428 for n, fig in enumerate(self.figures):
429 429 if self.nrows == 0 or self.nplots == 0:
430 430 log.warning('No data', self.name)
431 431 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
432 fig.canvas.manager.set_window_title(self.CODE)
432 433 continue
433 434
434 435 fig.tight_layout()
435 436 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
436 437 self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
437 # fig.canvas.draw()
438 fig.canvas.draw()
438 439
439 if self.save: # and self.data.ended:
440 self.contador += 1
440 if self.save and self.data.ended:
441 441 channels = range(self.nrows)
442 442 if self.oneFigure:
443 443 label = ''
444 444 else:
445 445 label = '_{}'.format(channels[n])
446 446 figname = os.path.join(
447 447 self.save,
448 '{}{}_{}{}.png'.format(
448 '{}{}_{}.png'.format(
449 449 self.CODE,
450 450 label,
451 451 self.getDateTime(self.saveTime).strftime(
452 '%y%m%d_%H%M%S'),
453 str(self.contador),
452 '%y%m%d_%H%M%S'),
454 453 )
455 454 )
456 455 log.log('Saving figure: {}'.format(figname), self.name)
457 456 fig.savefig(figname)
458 457
459 458 def plot(self):
460 459 '''
461 460 '''
462 461 raise(NotImplementedError, 'Implement this method in child class')
463 462
464 463 def run(self):
465 464
466 465 log.success('Starting', self.name)
467 466
468 467 context = zmq.Context()
469 468 receiver = context.socket(zmq.SUB)
470 469 receiver.setsockopt(zmq.SUBSCRIBE, '')
471 470 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
472 471
473 472 if 'server' in self.kwargs['parent']:
474 473 receiver.connect(
475 474 'ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
476 475 else:
477 476 receiver.connect("ipc:///tmp/zmq.plots")
478 477
479 478 while True:
480 479 try:
481 480 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
482 481 if self.data.localtime and self.localtime:
483 482 self.times = self.data.times
484 483 elif self.data.localtime and not self.localtime:
485 484 self.times = self.data.times + time.timezone
486 485 elif not self.data.localtime and self.localtime:
487 486 self.times = self.data.times - time.timezone
488 487 else:
489 488 self.times = self.data.times
490 489
491 490 self.min_time = self.times[0]
492 491 self.max_time = self.times[-1]
493 492
494 493 if self.isConfig is False:
495 494 self.__setup()
496 495 self.isConfig = True
497 496
498 497 self.__plot()
499 498
500 499 except zmq.Again as e:
501 500 log.log('Waiting for data...')
502 501 if self.data:
503 502 figpause(self.data.throttle)
504 503 else:
505 504 time.sleep(2)
506 505
507 506 def close(self):
508 507 if self.data:
509 508 self.__plot()
510 509
511 510
512 511 class PlotSpectraData(PlotData):
513 512 '''
514 513 Plot for Spectra data
515 514 '''
516 515
517 516 CODE = 'spc'
518 517 colormap = 'jro'
519 518
520 519 def setup(self):
521 520 self.nplots = len(self.data.channels)
522 521 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
523 522 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
524 523 self.width = 3.4 * self.ncols
525 524 self.height = 3 * self.nrows
526 525 self.cb_label = 'dB'
527 526 if self.showprofile:
528 527 self.width += 0.8 * self.ncols
529 528
530 529 self.ylabel = 'Range [Km]'
531 530
532 531 def plot(self):
533 532 if self.xaxis == "frequency":
534 533 x = self.data.xrange[0]
535 534 self.xlabel = "Frequency (kHz)"
536 535 elif self.xaxis == "time":
537 536 x = self.data.xrange[1]
538 537 self.xlabel = "Time (ms)"
539 538 else:
540 539 x = self.data.xrange[2]
541 540 self.xlabel = "Velocity (m/s)"
542 541
543 542 if self.CODE == 'spc_mean':
544 543 x = self.data.xrange[2]
545 544 self.xlabel = "Velocity (m/s)"
546 545
547 546 self.titles = []
548 547
549 548 y = self.data.heights
550 549 self.y = y
551 550 z = self.data['spc']
552 551
553 552 for n, ax in enumerate(self.axes):
554 553 noise = self.data['noise'][n][-1]
555 554 if self.CODE == 'spc_mean':
556 555 mean = self.data['mean'][n][-1]
557 556 if ax.firsttime:
558 557 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
559 558 self.xmin = self.xmin if self.xmin else -self.xmax
560 559 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
561 560 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
562 561 ax.plt = ax.pcolormesh(x, y, z[n].T,
563 562 vmin=self.zmin,
564 563 vmax=self.zmax,
565 564 cmap=plt.get_cmap(self.colormap)
566 565 )
567 566
568 567 if self.showprofile:
569 568 ax.plt_profile = self.pf_axes[n].plot(
570 569 self.data['rti'][n][-1], y)[0]
571 570 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
572 571 color="k", linestyle="dashed", lw=1)[0]
573 572 if self.CODE == 'spc_mean':
574 573 ax.plt_mean = ax.plot(mean, y, color='k')[0]
575 574 else:
576 575 ax.plt.set_array(z[n].T.ravel())
577 576 if self.showprofile:
578 577 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
579 578 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
580 579 if self.CODE == 'spc_mean':
581 580 ax.plt_mean.set_data(mean, y)
582 581
583 582 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
584 583 self.saveTime = self.max_time
585 584
586 585
587 586 class PlotCrossSpectraData(PlotData):
588 587
589 588 CODE = 'cspc'
590 589 zmin_coh = None
591 590 zmax_coh = None
592 591 zmin_phase = None
593 592 zmax_phase = None
594 593
595 594 def setup(self):
596 595
597 596 self.ncols = 4
598 597 self.nrows = len(self.data.pairs)
599 598 self.nplots = self.nrows * 4
600 599 self.width = 3.4 * self.ncols
601 600 self.height = 3 * self.nrows
602 601 self.ylabel = 'Range [Km]'
603 602 self.showprofile = False
604 603
605 604 def plot(self):
606 605
607 606 if self.xaxis == "frequency":
608 607 x = self.data.xrange[0]
609 608 self.xlabel = "Frequency (kHz)"
610 609 elif self.xaxis == "time":
611 610 x = self.data.xrange[1]
612 611 self.xlabel = "Time (ms)"
613 612 else:
614 613 x = self.data.xrange[2]
615 614 self.xlabel = "Velocity (m/s)"
616 615
617 616 self.titles = []
618 617
619 618 y = self.data.heights
620 619 self.y = y
621 620 spc = self.data['spc']
622 621 cspc = self.data['cspc']
623 622
624 623 for n in range(self.nrows):
625 624 noise = self.data['noise'][n][-1]
626 625 pair = self.data.pairs[n]
627 626 ax = self.axes[4 * n]
628 627 ax3 = self.axes[4 * n + 3]
629 628 if ax.firsttime:
630 629 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
631 630 self.xmin = self.xmin if self.xmin else -self.xmax
632 631 self.zmin = self.zmin if self.zmin else numpy.nanmin(spc)
633 632 self.zmax = self.zmax if self.zmax else numpy.nanmax(spc)
634 633 ax.plt = ax.pcolormesh(x, y, spc[pair[0]].T,
635 634 vmin=self.zmin,
636 635 vmax=self.zmax,
637 636 cmap=plt.get_cmap(self.colormap)
638 637 )
639 638 else:
640 639 ax.plt.set_array(spc[pair[0]].T.ravel())
641 640 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
642 641
643 642 ax = self.axes[4 * n + 1]
644 643 if ax.firsttime:
645 644 ax.plt = ax.pcolormesh(x, y, spc[pair[1]].T,
646 645 vmin=self.zmin,
647 646 vmax=self.zmax,
648 647 cmap=plt.get_cmap(self.colormap)
649 648 )
650 649 else:
651 650 ax.plt.set_array(spc[pair[1]].T.ravel())
652 651 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
653 652
654 653 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
655 654 coh = numpy.abs(out)
656 655 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
657 656
658 657 ax = self.axes[4 * n + 2]
659 658 if ax.firsttime:
660 659 ax.plt = ax.pcolormesh(x, y, coh.T,
661 660 vmin=0,
662 661 vmax=1,
663 662 cmap=plt.get_cmap(self.colormap_coh)
664 663 )
665 664 else:
666 665 ax.plt.set_array(coh.T.ravel())
667 666 self.titles.append(
668 667 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
669 668
670 669 ax = self.axes[4 * n + 3]
671 670 if ax.firsttime:
672 671 ax.plt = ax.pcolormesh(x, y, phase.T,
673 672 vmin=-180,
674 673 vmax=180,
675 674 cmap=plt.get_cmap(self.colormap_phase)
676 675 )
677 676 else:
678 677 ax.plt.set_array(phase.T.ravel())
679 678 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
680 679
681 680 self.saveTime = self.max_time
682 681
683 682
684 683 class PlotSpectraMeanData(PlotSpectraData):
685 684 '''
686 685 Plot for Spectra and Mean
687 686 '''
688 687 CODE = 'spc_mean'
689 688 colormap = 'jro'
690 689
691 690
692 691 class PlotRTIData(PlotData):
693 692 '''
694 693 Plot for RTI data
695 694 '''
696 695
697 696 CODE = 'rti'
698 697 colormap = 'jro'
699 698
700 699 def setup(self):
701 700 self.xaxis = 'time'
702 701 self.ncols = 1
703 702 self.nrows = len(self.data.channels)
704 703 self.nplots = len(self.data.channels)
705 704 self.ylabel = 'Range [Km]'
706 705 self.cb_label = 'dB'
707 706 self.titles = ['{} Channel {}'.format(
708 707 self.CODE.upper(), x) for x in range(self.nrows)]
709 708
710 709 def plot(self):
711 710 self.x = self.times
712 711 self.y = self.data.heights
713 712 self.z = self.data[self.CODE]
714 713 self.z = numpy.ma.masked_invalid(self.z)
715 714
716 715 for n, ax in enumerate(self.axes):
717 716 x, y, z = self.fill_gaps(*self.decimate())
718 717 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
719 718 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
720 719 if ax.firsttime:
721 720 ax.plt = ax.pcolormesh(x, y, z[n].T,
722 721 vmin=self.zmin,
723 722 vmax=self.zmax,
724 723 cmap=plt.get_cmap(self.colormap)
725 724 )
726 725 if self.showprofile:
727 726 ax.plot_profile = self.pf_axes[n].plot(
728 727 self.data['rti'][n][-1], self.y)[0]
729 728 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
730 729 color="k", linestyle="dashed", lw=1)[0]
731 730 else:
732 731 ax.collections.remove(ax.collections[0])
733 732 ax.plt = ax.pcolormesh(x, y, z[n].T,
734 733 vmin=self.zmin,
735 734 vmax=self.zmax,
736 735 cmap=plt.get_cmap(self.colormap)
737 736 )
738 737 if self.showprofile:
739 738 ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
740 739 ax.plot_noise.set_data(numpy.repeat(
741 740 self.data['noise'][n][-1], len(self.y)), self.y)
742 741
743 742 self.saveTime = self.min_time
744 743
745 744
746 745 class PlotCOHData(PlotRTIData):
747 746 '''
748 747 Plot for Coherence data
749 748 '''
750 749
751 750 CODE = 'coh'
752 751
753 752 def setup(self):
754 753 self.xaxis = 'time'
755 754 self.ncols = 1
756 755 self.nrows = len(self.data.pairs)
757 756 self.nplots = len(self.data.pairs)
758 757 self.ylabel = 'Range [Km]'
759 758 if self.CODE == 'coh':
760 759 self.cb_label = ''
761 760 self.titles = [
762 761 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
763 762 else:
764 763 self.cb_label = 'Degrees'
765 764 self.titles = [
766 765 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
767 766
768 767
769 768 class PlotPHASEData(PlotCOHData):
770 769 '''
771 770 Plot for Phase map data
772 771 '''
773 772
774 773 CODE = 'phase'
775 774 colormap = 'seismic'
776 775
777 776
778 777 class PlotNoiseData(PlotData):
779 778 '''
780 779 Plot for noise
781 780 '''
782 781
783 782 CODE = 'noise'
784 783
785 784 def setup(self):
786 785 self.xaxis = 'time'
787 786 self.ncols = 1
788 787 self.nrows = 1
789 788 self.nplots = 1
790 789 self.ylabel = 'Intensity [dB]'
791 790 self.titles = ['Noise']
792 791 self.colorbar = False
793 792
794 793 def plot(self):
795 794
796 795 x = self.times
797 796 xmin = self.min_time
798 797 xmax = xmin + self.xrange * 60 * 60
799 798 Y = self.data[self.CODE]
800 799
801 800 if self.axes[0].firsttime:
802 801 for ch in self.data.channels:
803 802 y = Y[ch]
804 803 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
805 804 plt.legend()
806 805 else:
807 806 for ch in self.data.channels:
808 807 y = Y[ch]
809 808 self.axes[0].lines[ch].set_data(x, y)
810 809
811 810 self.ymin = numpy.nanmin(Y) - 5
812 811 self.ymax = numpy.nanmax(Y) + 5
813 812 self.saveTime = self.min_time
814 813
815 814
816 815 class PlotSNRData(PlotRTIData):
817 816 '''
818 817 Plot for SNR Data
819 818 '''
820 819
821 820 CODE = 'snr'
822 821 colormap = 'jet'
823 822
824 823
825 824 class PlotDOPData(PlotRTIData):
826 825 '''
827 826 Plot for DOPPLER Data
828 827 '''
829 828
830 829 CODE = 'dop'
831 830 colormap = 'jet'
832 831
833 832
834 833 class PlotSkyMapData(PlotData):
835 834 '''
836 835 Plot for meteors detection data
837 836 '''
838 837
839 838 CODE = 'param'
840 839
841 840 def setup(self):
842 841
843 842 self.ncols = 1
844 843 self.nrows = 1
845 844 self.width = 7.2
846 845 self.height = 7.2
847 846 self.nplots = 1
848 847 self.xlabel = 'Zonal Zenith Angle (deg)'
849 848 self.ylabel = 'Meridional Zenith Angle (deg)'
850 849 self.polar = True
851 850 self.ymin = -180
852 851 self.ymax = 180
853 852 self.colorbar = False
854 853
855 854 def plot(self):
856 855
857 856 arrayParameters = numpy.concatenate(self.data['param'])
858 857 error = arrayParameters[:, -1]
859 858 indValid = numpy.where(error == 0)[0]
860 859 finalMeteor = arrayParameters[indValid, :]
861 860 finalAzimuth = finalMeteor[:, 3]
862 861 finalZenith = finalMeteor[:, 4]
863 862
864 863 x = finalAzimuth * numpy.pi / 180
865 864 y = finalZenith
866 865
867 866 ax = self.axes[0]
868 867
869 868 if ax.firsttime:
870 869 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
871 870 else:
872 871 ax.plot.set_data(x, y)
873 872
874 873 dt1 = self.getDateTime(self.min_time).strftime('%y/%m/%d %H:%M:%S')
875 874 dt2 = self.getDateTime(self.max_time).strftime('%y/%m/%d %H:%M:%S')
876 875 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
877 876 dt2,
878 877 len(x))
879 878 self.titles[0] = title
880 879 self.saveTime = self.max_time
881 880
882 881
883 882 class PlotParamData(PlotRTIData):
884 883 '''
885 884 Plot for data_param object
886 885 '''
887 886
888 887 CODE = 'param'
889 888 colormap = 'seismic'
890 889
891 890 def setup(self):
892 891 self.xaxis = 'time'
893 892 self.ncols = 1
894 893 self.nrows = self.data.shape(self.CODE)[0]
895 894 self.nplots = self.nrows
896 895 if self.showSNR:
897 896 self.nrows += 1
898 897 self.nplots += 1
899 898
900 899 self.ylabel = 'Height [Km]'
901 self.titles = self.data.parameters \
902 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
903 if self.showSNR:
904 self.titles.append('SNR')
900 if not self.titles:
901 self.titles = self.data.parameters \
902 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
903 if self.showSNR:
904 self.titles.append('SNR')
905 905
906 906 def plot(self):
907 907 self.data.normalize_heights()
908 908 self.x = self.times
909 909 self.y = self.data.heights
910 910 if self.showSNR:
911 911 self.z = numpy.concatenate(
912 912 (self.data[self.CODE], self.data['snr'])
913 913 )
914 914 else:
915 915 self.z = self.data[self.CODE]
916 916
917 917 self.z = numpy.ma.masked_invalid(self.z)
918 918
919 919 for n, ax in enumerate(self.axes):
920 920
921 921 x, y, z = self.fill_gaps(*self.decimate())
922 922 self.zmax = self.zmax if self.zmax is not None else numpy.max(
923 923 self.z[n])
924 924 self.zmin = self.zmin if self.zmin is not None else numpy.min(
925 925 self.z[n])
926 926
927 927 if ax.firsttime:
928 928 if self.zlimits is not None:
929 929 self.zmin, self.zmax = self.zlimits[n]
930 930
931 931 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
932 932 vmin=self.zmin,
933 933 vmax=self.zmax,
934 934 cmap=self.cmaps[n]
935 935 )
936 936 else:
937 937 if self.zlimits is not None:
938 938 self.zmin, self.zmax = self.zlimits[n]
939 939 ax.collections.remove(ax.collections[0])
940 940 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
941 941 vmin=self.zmin,
942 942 vmax=self.zmax,
943 943 cmap=self.cmaps[n]
944 944 )
945 945
946 946 self.saveTime = self.min_time
947 947
948 948
949 949 class PlotOutputData(PlotParamData):
950 950 '''
951 951 Plot data_output object
952 952 '''
953 953
954 954 CODE = 'output'
955 955 colormap = 'seismic'
Show file before | Show file after | Hide comments
@@ -1,385 +1,343
1 1 '''
2 2 Created on Set 10, 2017
3 3
4 4 @author: Juan C. Espinoza
5 5 '''
6 6
7 7
8 8 import os
9 9 import sys
10 10 import time
11 11 import glob
12 12 import datetime
13 13
14 14 import numpy
15 15
16 16 from schainpy.model.proc.jroproc_base import ProcessingUnit
17 17 from schainpy.model.data.jrodata import Parameters
18 18 from schainpy.model.io.jroIO_base import JRODataReader, isNumber
19 19 from schainpy.utils import log
20 20
21 21 FILE_HEADER_STRUCTURE = numpy.dtype([
22 22 ('year', 'f'),
23 23 ('doy', 'f'),
24 24 ('nint', 'f'),
25 25 ('navg', 'f'),
26 26 ('fh', 'f'),
27 27 ('dh', 'f'),
28 28 ('nheights', 'f'),
29 29 ('ipp', 'f')
30 30 ])
31 31
32 32 REC_HEADER_STRUCTURE = numpy.dtype([
33 33 ('magic', 'f'),
34 34 ('hours', 'f'),
35 ('timedelta', 'f'),
35 ('interval', 'f'),
36 36 ('h0', 'f'),
37 37 ('nheights', 'f'),
38 38 ('snr1', 'f'),
39 39 ('snr2', 'f'),
40 40 ('snr', 'f'),
41 41 ])
42 42
43 43 DATA_STRUCTURE = numpy.dtype([
44 44 ('range', '<u4'),
45 45 ('status', '<u4'),
46 46 ('zonal', '<f4'),
47 47 ('meridional', '<f4'),
48 48 ('vertical', '<f4'),
49 49 ('zonal_a', '<f4'),
50 50 ('meridional_a', '<f4'),
51 51 ('corrected_fading', '<f4'), # seconds
52 52 ('uncorrected_fading', '<f4'), # seconds
53 53 ('time_diff', '<f4'),
54 54 ('major_axis', '<f4'),
55 55 ('axial_ratio', '<f4'),
56 56 ('orientation', '<f4'),
57 57 ('sea_power', '<u4'),
58 58 ('sea_algorithm', '<u4')
59 59 ])
60 60
61 61
62 62 class JULIAParamReader(JRODataReader, ProcessingUnit):
63 63 '''
64 64 Julia data (eej, spf, 150km) *.dat files
65 65 '''
66 66
67 67 ext = '.dat'
68 68
69 69 def __init__(self, **kwargs):
70 70
71 71 ProcessingUnit.__init__(self, **kwargs)
72 72
73 73 self.dataOut = Parameters()
74 74 self.counter_records = 0
75 75 self.flagNoMoreFiles = 0
76 76 self.isConfig = False
77 77 self.filename = None
78 78 self.clockpulse = 0.15
79 79 self.kd = 213.6
80 80
81 81 def setup(self,
82 82 path=None,
83 83 startDate=None,
84 84 endDate=None,
85 85 ext=None,
86 86 startTime=datetime.time(0, 0, 0),
87 87 endTime=datetime.time(23, 59, 59),
88 88 timezone=0,
89 89 format=None,
90 90 **kwargs):
91 91
92 92 self.path = path
93 93 self.startDate = startDate
94 94 self.endDate = endDate
95 95 self.startTime = startTime
96 96 self.endTime = endTime
97 97 self.datatime = datetime.datetime(1900, 1, 1)
98 self.format = format
98 self.format = format
99 99
100 100 if self.path is None:
101 101 raise ValueError, "The path is not valid"
102 102
103 103 if ext is None:
104 104 ext = self.ext
105 105
106 106 self.search_files(self.path, startDate, endDate, ext)
107 107 self.timezone = timezone
108 108 self.fileIndex = 0
109 109
110 110 if not self.fileList:
111 111 log.warning('There is no files matching these date in the folder: {}'.format(
112 112 path), self.name)
113 113
114 114 self.setNextFile()
115 115
116 116 def search_files(self, path, startDate, endDate, ext):
117 117 '''
118 118 Searching for BLTR rawdata file in path
119 119 Creating a list of file to proces included in [startDate,endDate]
120 120
121 121 Input:
122 122 path - Path to find BLTR rawdata files
123 123 startDate - Select file from this date
124 124 enDate - Select file until this date
125 125 ext - Extension of the file to read
126 126 '''
127 127
128 128 log.success('Searching files in {} '.format(path), self.name)
129 129 fileList0 = glob.glob1(path, '{}*{}'.format(self.format.upper(), ext))
130 130 fileList0.sort()
131 131
132 132 self.fileList = []
133 133 self.dateFileList = []
134 134
135 135 for thisFile in fileList0:
136 136 year = thisFile[2:4]
137 137 if not isNumber(year):
138 138 continue
139 139
140 140 month = thisFile[4:6]
141 141 if not isNumber(month):
142 142 continue
143 143
144 144 day = thisFile[6:8]
145 145 if not isNumber(day):
146 146 continue
147 147
148 148 year, month, day = int(year), int(month), int(day)
149 149 dateFile = datetime.date(year+2000, month, day)
150 150
151 151 if (startDate > dateFile) or (endDate < dateFile):
152 152 continue
153 153
154 154 self.fileList.append(thisFile)
155 155 self.dateFileList.append(dateFile)
156 156
157 157 return
158 158
159 159 def setNextFile(self):
160 160
161 161 file_id = self.fileIndex
162 162
163 163 if file_id == len(self.fileList):
164 164 log.success('No more files in the folder', self.name)
165 165 self.flagNoMoreFiles = 1
166 166 return 0
167 167
168 168 log.success('Opening {}'.format(self.fileList[file_id]), self.name)
169 169 filename = os.path.join(self.path, self.fileList[file_id])
170 170
171 171 dirname, name = os.path.split(filename)
172 172 self.siteFile = name.split('.')[0]
173 173 if self.filename is not None:
174 174 self.fp.close()
175 175 self.filename = filename
176 176 self.fp = open(self.filename, 'rb')
177 self.t0 = [7, 0, 0]
178 self.tf = [18, 0, 0]
179 177
180 178 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
181 179 yy = self.header_file['year'] - 1900 * (self.header_file['year'] > 3000)
182 180 self.year = int(yy + 1900 * (yy < 1000))
183 181 self.doy = int(self.header_file['doy'])
184 self.H0 = self.clockpulse*numpy.round(self.header_file['fh']/self.clockpulse)
185 self.dH = self.clockpulse*numpy.round(self.header_file['dh']/self.clockpulse)
186 self.ipp = self.clockpulse*numpy.round(self.header_file['ipp']/self.clockpulse)
187
188 tau = self.ipp / 1.5E5
189
190 self.nheights = int(self.header_file['nheights'])
191 self.heights = numpy.arange(self.nheights) * self.dH + self.H0
192
182 self.dH = round(self.header_file['dh'], 2)
183 self.ipp = round(self.header_file['ipp'], 2)
193 184 self.sizeOfFile = os.path.getsize(self.filename)
194 185 self.counter_records = 0
195 186 self.flagIsNewFile = 0
196 187 self.fileIndex += 1
197 188
198 189 return 1
199 190
200 191 def readNextBlock(self):
201 192
202 193 while True:
203 if self.fp.tell() == self.sizeOfFile:
194 if not self.readBlock():
204 195 self.flagIsNewFile = 1
205 196 if not self.setNextFile():
206 return 0
207
208 self.readBlock()
197 return 0
209 198
210 199 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
211 200 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
212 201 log.warning(
213 202 'Reading Record No. {} -> {} [Skipping]'.format(
214 203 self.counter_records,
215 204 self.datatime.ctime()),
216 205 self.name)
217 206 continue
218 207 break
219
208
220 209 log.log('Reading Record No. {} -> {}'.format(
221 210 self.counter_records,
222 211 self.datatime.ctime()), self.name)
223 212
224 213 return 1
225 214
226 215 def readBlock(self):
227 216
228 header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
229 self.timedelta = header_rec['timedelta']
230 if header_rec['magic'] == 888.:
231 h0 = self.clockpulse * numpy.round(header_rec['h0'] / self.clockpulse)
232 nheights = int(header_rec['nheights'])
233 hours = float(header_rec['hours'][0])
234 self.datatime = datetime.datetime(self.year, 1, 1) + datetime.timedelta(days=self.doy-1, hours=hours)
235 self.time = (self.datatime - datetime.datetime(1970,1,1)).total_seconds()
236
237 buffer = numpy.fromfile(self.fp, 'f', 8*nheights).reshape(nheights, 8)
238 idh0 = int(numpy.round((self.H0 - h0) / self.dH))
239 if idh0 == 0 and buffer[0,0] > 1E8:
240 buffer[0,0] = buffer[1,0]
241
242 pow0 = buffer[:, 0]
243 pow1 = buffer[:, 1]
244 acf0 = (buffer[:,2] + buffer[:,3]*1j) / pow0
245 acf1 = (buffer[:,4] + buffer[:,5]*1j) / pow1
246 dccf = (buffer[:,6] + buffer[:,7]*1j) / (pow0*pow1)
247
248 ### SNR
249 sno = (pow0 + pow1 - header_rec['snr']) / header_rec['snr']
250 sno10 = numpy.log10(sno)
251 dsno = 1.0 / numpy.sqrt(self.header_file['nint'] * self.header_file['navg']) * (1 + (1 / sno))
252
253 ### Vertical Drift
254 sp = numpy.sqrt(numpy.abs(acf0)*numpy.abs(acf1))
255 sp[numpy.where(numpy.abs(sp) >= 1.0)] = numpy.sqrt(0.9999)
256
257 vzo = -numpy.arctan2(acf0.imag + acf1.imag,acf0.real + acf1.real)*1.5E5*1.5/(self.ipp*numpy.pi)
258 dvzo = numpy.sqrt(1.0 - sp*sp)*0.338*1.5E5/(numpy.sqrt(self.header_file['nint']*self.header_file['navg'])*sp*self.ipp)
259 err = numpy.where(dvzo <= 0.1)
260 dvzo[err] = 0.1
261
262 #Zonal Drifts
263 dt = self.header_file['nint']*self.ipp / 1.5E5
264 coh = numpy.sqrt(numpy.abs(dccf))
265 err = numpy.where(coh >= 1.0)
266 coh[err] = numpy.sqrt(0.99999)
267
268 err = numpy.where(coh <= 0.1)
269 coh[err] = numpy.sqrt(0.1)
270
271 vxo = numpy.arctan2(dccf.imag, dccf.real)*self.heights[idh0]*1.0E3/(self.kd*dt)
272 dvxo = numpy.sqrt(1.0 - coh*coh)*self.heights[idh0]*1.0E3/(numpy.sqrt(self.header_file['nint']*self.header_file['navg'])*coh*self.kd*dt)
273
274 err = numpy.where(dvxo <= 0.1)
275 dvxo[err] = 0.1
276
277 N = range(len(pow0))
278
279 self.buffer = numpy.zeros((6, self.nheights)) + numpy.nan
280
281 self.buffer[0, idh0+numpy.array(N)] = sno10
282 self.buffer[1, idh0+numpy.array(N)] = vzo
283 self.buffer[2, idh0+numpy.array(N)] = vxo
284 self.buffer[3, idh0+numpy.array(N)] = dvzo
285 self.buffer[4, idh0+numpy.array(N)] = dvxo
286 self.buffer[5, idh0+numpy.array(N)] = dsno
287
288 self.counter_records += 1
217 pointer = self.fp.tell()
218 heights, dt = self.readHeader()
219 self.fp.seek(pointer)
220 buffer_h = []
221 buffer_d = []
222 while True:
223 pointer = self.fp.tell()
224 if pointer == self.sizeOfFile:
225 return 0
226 heights, datatime = self.readHeader()
227 if dt == datatime:
228 buffer_h.append(heights)
229 buffer_d.append(self.readData(len(heights)))
230 continue
231 self.fp.seek(pointer)
232 break
233
234 if dt.date() > self.datatime.date():
235 self.flagDiscontinuousBlock = 1
236 self.datatime = dt
237 self.time = (dt - datetime.datetime(1970, 1, 1)).total_seconds() + time.timezone
238 self.heights = numpy.concatenate(buffer_h)
239 self.buffer = numpy.zeros((5, len(self.heights))) + numpy.nan
240 self.buffer[0, :] = numpy.concatenate([buf[0] for buf in buffer_d])
241 self.buffer[1, :] = numpy.concatenate([buf[1] for buf in buffer_d])
242 self.buffer[2, :] = numpy.concatenate([buf[2] for buf in buffer_d])
243 self.buffer[3, :] = numpy.concatenate([buf[3] for buf in buffer_d])
244 self.buffer[4, :] = numpy.concatenate([buf[4] for buf in buffer_d])
245
246 self.counter_records += 1
289 247
290 return
248 return 1
291 249
292 250 def readHeader(self):
293 251 '''
294 RecordHeader of BLTR rawdata file
252 Parse recordHeader
295 253 '''
296
297 header_structure = numpy.dtype(
298 REC_HEADER_STRUCTURE.descr + [
299 ('antenna_coord', 'f4', (2, self.nchannels)),
300 ('rx_gains', 'u4', (self.nchannels,)),
301 ('rx_analysis', 'u4', (self.nchannels,))
302 ]
303 )
304
305 self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
306 self.lat = self.header_rec['lat'][0]
307 self.lon = self.header_rec['lon'][0]
308 self.delta = self.header_rec['delta_r'][0]
309 self.correction = self.header_rec['dmode_rngcorr'][0]
310 self.imode = self.header_rec['dmode_index'][0]
311 self.antenna = self.header_rec['antenna_coord']
312 self.rx_gains = self.header_rec['rx_gains']
313 self.time = self.header_rec['time'][0]
314 dt = datetime.datetime.utcfromtimestamp(self.time)
315 if dt.date()>self.datatime.date():
316 self.flagDiscontinuousBlock = 1
317 self.datatime = dt
318 254
319 def readData(self):
255 self.header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
256 self.interval = self.header_rec['interval']
257 if self.header_rec['magic'] == 888.:
258 self.header_rec['h0'] = round(self.header_rec['h0'], 2)
259 nheights = int(self.header_rec['nheights'])
260 hours = float(self.header_rec['hours'][0])
261 heights = numpy.arange(nheights) * self.dH + self.header_rec['h0']
262 datatime = datetime.datetime(self.year, 1, 1) + datetime.timedelta(days=self.doy-1, hours=hours)
263 return heights, datatime
264 else:
265 return False
266
267 def readData(self, N):
320 268 '''
321 Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value.
322
323 Input:
324 status_value - Array data is set to NAN for values that are not equal to status_value
325
269 Parse data
326 270 '''
327 271
328 data_structure = numpy.dtype(
329 DATA_STRUCTURE.descr + [
330 ('rx_saturation', 'u4', (self.nchannels,)),
331 ('chan_offset', 'u4', (2 * self.nchannels,)),
332 ('rx_amp', 'u4', (self.nchannels,)),
333 ('rx_snr', 'f4', (self.nchannels,)),
334 ('cross_snr', 'f4', (self.kchan,)),
335 ('sea_power_relative', 'f4', (self.kchan,))]
336 )
337
338 data = numpy.fromfile(self.fp, data_structure, self.nranges)
339
340 height = data['range']
341 winds = numpy.array(
342 (data['zonal'], data['meridional'], data['vertical']))
343 snr = data['rx_snr'].T
344
345 winds[numpy.where(winds == -9999.)] = numpy.nan
346 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
347 snr[numpy.where(snr == -9999.)] = numpy.nan
348 snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
349 snr = numpy.power(10, snr / 10)
272 buffer = numpy.fromfile(self.fp, 'f', 8*N).reshape(N, 8)
273
274 pow0 = buffer[:, 0]
275 pow1 = buffer[:, 1]
276 acf0 = (buffer[:,2] + buffer[:,3]*1j) / pow0
277 acf1 = (buffer[:,4] + buffer[:,5]*1j) / pow1
278 dccf = (buffer[:,6] + buffer[:,7]*1j) / (pow0*pow1)
279
280 ### SNR
281 sno = (pow0 + pow1 - self.header_rec['snr']) / self.header_rec['snr']
282 sno10 = numpy.log10(sno)
283 # dsno = 1.0 / numpy.sqrt(self.header_file['nint'] * self.header_file['navg']) * (1 + (1 / sno))
284
285 ### Vertical Drift
286 sp = numpy.sqrt(numpy.abs(acf0)*numpy.abs(acf1))
287 sp[numpy.where(numpy.abs(sp) >= 1.0)] = numpy.sqrt(0.9999)
288
289 vzo = -numpy.arctan2(acf0.imag + acf1.imag,acf0.real + acf1.real)*1.5E5*1.5/(self.ipp*numpy.pi)
290 dvzo = numpy.sqrt(1.0 - sp*sp)*0.338*1.5E5/(numpy.sqrt(self.header_file['nint']*self.header_file['navg'])*sp*self.ipp)
291 err = numpy.where(dvzo <= 0.1)
292 dvzo[err] = 0.1
293
294 #Zonal Drifts
295 dt = self.header_file['nint']*self.ipp / 1.5E5
296 coh = numpy.sqrt(numpy.abs(dccf))
297 err = numpy.where(coh >= 1.0)
298 coh[err] = numpy.sqrt(0.99999)
299
300 err = numpy.where(coh <= 0.1)
301 coh[err] = numpy.sqrt(0.1)
302
303 vxo = numpy.arctan2(dccf.imag, dccf.real)*self.header_rec['h0']*1.0E3/(self.kd*dt)
304 dvxo = numpy.sqrt(1.0 - coh*coh)*self.header_rec['h0']*1.0E3/(numpy.sqrt(self.header_file['nint']*self.header_file['navg'])*coh*self.kd*dt)
305
306 err = numpy.where(dvxo <= 0.1)
307 dvxo[err] = 0.1
350 308
351 return height, winds, snr
309 return vzo, dvzo, vxo, dvxo, sno10
352 310
353 311 def set_output(self):
354 312 '''
355 313 Storing data from databuffer to dataOut object
356 314 '''
357
358 self.dataOut.data_SNR = self.buffer[0]
359 self.dataOut.heights = self.heights
360 self.dataOut.data_param = self.buffer[1:,]
315
316 self.dataOut.data_SNR = self.buffer[4].reshape(1, -1)
317 self.dataOut.heightList = self.heights
318 self.dataOut.data_param = self.buffer[0:4,]
361 319 self.dataOut.utctimeInit = self.time
362 self.dataOut.utctime = self.dataOut.utctimeInit
363 self.dataOut.useLocalTime = False
364 self.dataOut.paramInterval = self.timedelta
365 self.dataOut.timezone = self.timezone
320 self.dataOut.utctime = self.time
321 self.dataOut.useLocalTime = True
322 self.dataOut.paramInterval = self.interval
323 self.dataOut.timezone = self.timezone
366 324 self.dataOut.sizeOfFile = self.sizeOfFile
367 325 self.dataOut.flagNoData = False
368 326 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
369 327
370 328 def getData(self):
371 329 '''
372 330 Storing data from databuffer to dataOut object
373 331 '''
374 332 if self.flagNoMoreFiles:
375 333 self.dataOut.flagNoData = True
376 334 log.success('No file left to process', self.name)
377 335 return 0
378 336
379 337 if not self.readNextBlock():
380 338 self.dataOut.flagNoData = True
381 339 return 0
382 340
383 341 self.set_output()
384 342
385 343 return 1
Show file before | Show file after | Hide comments
@@ -1,636 +1,635
1 1 '''
2 2 @author: Juan C. Espinoza
3 3 '''
4 4
5 5 import time
6 6 import json
7 7 import numpy
8 8 import paho.mqtt.client as mqtt
9 9 import zmq
10 10 import datetime
11 11 from zmq.utils.monitor import recv_monitor_message
12 12 from functools import wraps
13 13 from threading import Thread
14 14 from multiprocessing import Process
15 15
16 16 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
17 17 from schainpy.model.data.jrodata import JROData
18 18 from schainpy.utils import log
19 19
20 20 MAXNUMX = 100
21 21 MAXNUMY = 100
22 22
23 23 class PrettyFloat(float):
24 24 def __repr__(self):
25 25 return '%.2f' % self
26 26
27 27 def roundFloats(obj):
28 28 if isinstance(obj, list):
29 29 return map(roundFloats, obj)
30 30 elif isinstance(obj, float):
31 31 return round(obj, 2)
32 32
33 33 def decimate(z, MAXNUMY):
34 34 dy = int(len(z[0])/MAXNUMY) + 1
35 35
36 36 return z[::, ::dy]
37 37
38 38 class throttle(object):
39 39 '''
40 40 Decorator that prevents a function from being called more than once every
41 41 time period.
42 42 To create a function that cannot be called more than once a minute, but
43 43 will sleep until it can be called:
44 44 @throttle(minutes=1)
45 45 def foo():
46 46 pass
47 47
48 48 for i in range(10):
49 49 foo()
50 50 print "This function has run %s times." % i
51 51 '''
52 52
53 53 def __init__(self, seconds=0, minutes=0, hours=0):
54 54 self.throttle_period = datetime.timedelta(
55 55 seconds=seconds, minutes=minutes, hours=hours
56 56 )
57 57
58 58 self.time_of_last_call = datetime.datetime.min
59 59
60 60 def __call__(self, fn):
61 61 @wraps(fn)
62 62 def wrapper(*args, **kwargs):
63 63 coerce = kwargs.pop('coerce', None)
64 64 if coerce:
65 65 self.time_of_last_call = datetime.datetime.now()
66 66 return fn(*args, **kwargs)
67 67 else:
68 68 now = datetime.datetime.now()
69 69 time_since_last_call = now - self.time_of_last_call
70 70 time_left = self.throttle_period - time_since_last_call
71 71
72 72 if time_left > datetime.timedelta(seconds=0):
73 73 return
74 74
75 75 self.time_of_last_call = datetime.datetime.now()
76 76 return fn(*args, **kwargs)
77 77
78 78 return wrapper
79 79
80 80 class Data(object):
81 81 '''
82 82 Object to hold data to be plotted
83 83 '''
84 84
85 85 def __init__(self, plottypes, throttle_value):
86 86 self.plottypes = plottypes
87 87 self.throttle = throttle_value
88 88 self.ended = False
89 89 self.localtime = False
90 90 self.__times = []
91 91 self.__heights = []
92 92
93 93 def __str__(self):
94 94 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
95 95 return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times))
96 96
97 97 def __len__(self):
98 98 return len(self.__times)
99 99
100 100 def __getitem__(self, key):
101 101 if key not in self.data:
102 102 raise KeyError(log.error('Missing key: {}'.format(key)))
103 103
104 104 if 'spc' in key:
105 105 ret = self.data[key]
106 106 else:
107 107 ret = numpy.array([self.data[key][x] for x in self.times])
108 108 if ret.ndim > 1:
109 109 ret = numpy.swapaxes(ret, 0, 1)
110 110 return ret
111 111
112 112 def __contains__(self, key):
113 113 return key in self.data
114 114
115 115 def setup(self):
116 116 '''
117 117 Configure object
118 118 '''
119 119
120 120 self.ended = False
121 121 self.data = {}
122 122 self.__times = []
123 123 self.__heights = []
124 124 self.__all_heights = set()
125 125 for plot in self.plottypes:
126 126 if 'snr' in plot:
127 127 plot = 'snr'
128 128 self.data[plot] = {}
129 129
130 130 def shape(self, key):
131 131 '''
132 132 Get the shape of the one-element data for the given key
133 133 '''
134 134
135 135 if len(self.data[key]):
136 136 if 'spc' in key:
137 137 return self.data[key].shape
138 138 return self.data[key][self.__times[0]].shape
139 139 return (0,)
140 140
141 def update(self, dataOut):
141 def update(self, dataOut, tm):
142 142 '''
143 143 Update data object with new dataOut
144 144 '''
145
146 tm = dataOut.utctime
145
147 146 if tm in self.__times:
148 147 return
149 148
150 149 self.parameters = getattr(dataOut, 'parameters', [])
151 150 self.pairs = dataOut.pairsList
152 151 self.channels = dataOut.channelList
153 152 self.interval = dataOut.getTimeInterval()
154 153 self.localtime = dataOut.useLocalTime
155 154 if 'spc' in self.plottypes or 'cspc' in self.plottypes:
156 155 self.xrange = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
157 156 self.__heights.append(dataOut.heightList)
158 157 self.__all_heights.update(dataOut.heightList)
159 158 self.__times.append(tm)
160 159
161 160 for plot in self.plottypes:
162 161 if plot == 'spc':
163 162 z = dataOut.data_spc/dataOut.normFactor
164 163 self.data[plot] = 10*numpy.log10(z)
165 164 if plot == 'cspc':
166 165 self.data[plot] = dataOut.data_cspc
167 166 if plot == 'noise':
168 167 self.data[plot][tm] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
169 168 if plot == 'rti':
170 169 self.data[plot][tm] = dataOut.getPower()
171 170 if plot == 'snr_db':
172 171 self.data['snr'][tm] = dataOut.data_SNR
173 172 if plot == 'snr':
174 173 self.data[plot][tm] = 10*numpy.log10(dataOut.data_SNR)
175 174 if plot == 'dop':
176 175 self.data[plot][tm] = 10*numpy.log10(dataOut.data_DOP)
177 176 if plot == 'mean':
178 177 self.data[plot][tm] = dataOut.data_MEAN
179 178 if plot == 'std':
180 179 self.data[plot][tm] = dataOut.data_STD
181 180 if plot == 'coh':
182 181 self.data[plot][tm] = dataOut.getCoherence()
183 182 if plot == 'phase':
184 183 self.data[plot][tm] = dataOut.getCoherence(phase=True)
185 184 if plot == 'output':
186 185 self.data[plot][tm] = dataOut.data_output
187 186 if plot == 'param':
188 187 self.data[plot][tm] = dataOut.data_param
189 188
190 189 def normalize_heights(self):
191 190 '''
192 191 Ensure same-dimension of the data for different heighList
193 192 '''
194 193
195 194 H = numpy.array(list(self.__all_heights))
196 195 H.sort()
197 196 for key in self.data:
198 197 shape = self.shape(key)[:-1] + H.shape
199 198 for tm, obj in self.data[key].items():
200 199 h = self.__heights[self.__times.index(tm)]
201 200 if H.size == h.size:
202 201 continue
203 202 index = numpy.where(numpy.in1d(H, h))[0]
204 203 dummy = numpy.zeros(shape) + numpy.nan
205 204 if len(shape) == 2:
206 205 dummy[:, index] = obj
207 206 else:
208 207 dummy[index] = obj
209 208 self.data[key][tm] = dummy
210 209
211 210 self.__heights = [H for tm in self.__times]
212 211
213 212 def jsonify(self, decimate=False):
214 213 '''
215 214 Convert data to json
216 215 '''
217 216
218 217 ret = {}
219 218 tm = self.times[-1]
220 219
221 220 for key, value in self.data:
222 221 if key in ('spc', 'cspc'):
223 222 ret[key] = roundFloats(self.data[key].to_list())
224 223 else:
225 224 ret[key] = roundFloats(self.data[key][tm].to_list())
226 225
227 226 ret['timestamp'] = tm
228 227 ret['interval'] = self.interval
229 228
230 229 @property
231 230 def times(self):
232 231 '''
233 232 Return the list of times of the current data
234 233 '''
235 234
236 235 ret = numpy.array(self.__times)
237 236 ret.sort()
238 237 return ret
239 238
240 239 @property
241 240 def heights(self):
242 241 '''
243 242 Return the list of heights of the current data
244 243 '''
245 244
246 245 return numpy.array(self.__heights[-1])
247 246
248 247 class PublishData(Operation):
249 248 '''
250 249 Operation to send data over zmq.
251 250 '''
252 251
253 252 __attrs__ = ['host', 'port', 'delay', 'zeromq', 'mqtt', 'verbose']
254 253
255 254 def __init__(self, **kwargs):
256 255 """Inicio."""
257 256 Operation.__init__(self, **kwargs)
258 257 self.isConfig = False
259 258 self.client = None
260 259 self.zeromq = None
261 260 self.mqtt = None
262 261
263 262 def on_disconnect(self, client, userdata, rc):
264 263 if rc != 0:
265 264 log.warning('Unexpected disconnection.')
266 265 self.connect()
267 266
268 267 def connect(self):
269 268 log.warning('trying to connect')
270 269 try:
271 270 self.client.connect(
272 271 host=self.host,
273 272 port=self.port,
274 273 keepalive=60*10,
275 274 bind_address='')
276 275 self.client.loop_start()
277 276 # self.client.publish(
278 277 # self.topic + 'SETUP',
279 278 # json.dumps(setup),
280 279 # retain=True
281 280 # )
282 281 except:
283 282 log.error('MQTT Conection error.')
284 283 self.client = False
285 284
286 285 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, verbose=True, **kwargs):
287 286 self.counter = 0
288 287 self.topic = kwargs.get('topic', 'schain')
289 288 self.delay = kwargs.get('delay', 0)
290 289 self.plottype = kwargs.get('plottype', 'spectra')
291 290 self.host = kwargs.get('host', "10.10.10.82")
292 291 self.port = kwargs.get('port', 3000)
293 292 self.clientId = clientId
294 293 self.cnt = 0
295 294 self.zeromq = zeromq
296 295 self.mqtt = kwargs.get('plottype', 0)
297 296 self.client = None
298 297 self.verbose = verbose
299 298 setup = []
300 299 if mqtt is 1:
301 300 self.client = mqtt.Client(
302 301 client_id=self.clientId + self.topic + 'SCHAIN',
303 302 clean_session=True)
304 303 self.client.on_disconnect = self.on_disconnect
305 304 self.connect()
306 305 for plot in self.plottype:
307 306 setup.append({
308 307 'plot': plot,
309 308 'topic': self.topic + plot,
310 309 'title': getattr(self, plot + '_' + 'title', False),
311 310 'xlabel': getattr(self, plot + '_' + 'xlabel', False),
312 311 'ylabel': getattr(self, plot + '_' + 'ylabel', False),
313 312 'xrange': getattr(self, plot + '_' + 'xrange', False),
314 313 'yrange': getattr(self, plot + '_' + 'yrange', False),
315 314 'zrange': getattr(self, plot + '_' + 'zrange', False),
316 315 })
317 316 if zeromq is 1:
318 317 context = zmq.Context()
319 318 self.zmq_socket = context.socket(zmq.PUSH)
320 319 server = kwargs.get('server', 'zmq.pipe')
321 320
322 321 if 'tcp://' in server:
323 322 address = server
324 323 else:
325 324 address = 'ipc:///tmp/%s' % server
326 325
327 326 self.zmq_socket.connect(address)
328 327 time.sleep(1)
329 328
330 329
331 330 def publish_data(self):
332 331 self.dataOut.finished = False
333 332 if self.mqtt is 1:
334 333 yData = self.dataOut.heightList[:2].tolist()
335 334 if self.plottype == 'spectra':
336 335 data = getattr(self.dataOut, 'data_spc')
337 336 z = data/self.dataOut.normFactor
338 337 zdB = 10*numpy.log10(z)
339 338 xlen, ylen = zdB[0].shape
340 339 dx = int(xlen/MAXNUMX) + 1
341 340 dy = int(ylen/MAXNUMY) + 1
342 341 Z = [0 for i in self.dataOut.channelList]
343 342 for i in self.dataOut.channelList:
344 343 Z[i] = zdB[i][::dx, ::dy].tolist()
345 344 payload = {
346 345 'timestamp': self.dataOut.utctime,
347 346 'data': roundFloats(Z),
348 347 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
349 348 'interval': self.dataOut.getTimeInterval(),
350 349 'type': self.plottype,
351 350 'yData': yData
352 351 }
353 352
354 353 elif self.plottype in ('rti', 'power'):
355 354 data = getattr(self.dataOut, 'data_spc')
356 355 z = data/self.dataOut.normFactor
357 356 avg = numpy.average(z, axis=1)
358 357 avgdB = 10*numpy.log10(avg)
359 358 xlen, ylen = z[0].shape
360 359 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
361 360 AVG = [0 for i in self.dataOut.channelList]
362 361 for i in self.dataOut.channelList:
363 362 AVG[i] = avgdB[i][::dy].tolist()
364 363 payload = {
365 364 'timestamp': self.dataOut.utctime,
366 365 'data': roundFloats(AVG),
367 366 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
368 367 'interval': self.dataOut.getTimeInterval(),
369 368 'type': self.plottype,
370 369 'yData': yData
371 370 }
372 371 elif self.plottype == 'noise':
373 372 noise = self.dataOut.getNoise()/self.dataOut.normFactor
374 373 noisedB = 10*numpy.log10(noise)
375 374 payload = {
376 375 'timestamp': self.dataOut.utctime,
377 376 'data': roundFloats(noisedB.reshape(-1, 1).tolist()),
378 377 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
379 378 'interval': self.dataOut.getTimeInterval(),
380 379 'type': self.plottype,
381 380 'yData': yData
382 381 }
383 382 elif self.plottype == 'snr':
384 383 data = getattr(self.dataOut, 'data_SNR')
385 384 avgdB = 10*numpy.log10(data)
386 385
387 386 ylen = data[0].size
388 387 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
389 388 AVG = [0 for i in self.dataOut.channelList]
390 389 for i in self.dataOut.channelList:
391 390 AVG[i] = avgdB[i][::dy].tolist()
392 391 payload = {
393 392 'timestamp': self.dataOut.utctime,
394 393 'data': roundFloats(AVG),
395 394 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
396 395 'type': self.plottype,
397 396 'yData': yData
398 397 }
399 398 else:
400 399 print "Tipo de grafico invalido"
401 400 payload = {
402 401 'data': 'None',
403 402 'timestamp': 'None',
404 403 'type': None
405 404 }
406 405
407 406 self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
408 407
409 408 if self.zeromq is 1:
410 409 if self.verbose:
411 410 log.log(
412 411 'Sending {} - {}'.format(self.dataOut.type, self.dataOut.datatime),
413 412 self.name
414 413 )
415 414 self.zmq_socket.send_pyobj(self.dataOut)
416 415
417 416 def run(self, dataOut, **kwargs):
418 417 self.dataOut = dataOut
419 418 if not self.isConfig:
420 419 self.setup(**kwargs)
421 420 self.isConfig = True
422 421
423 422 self.publish_data()
424 423 time.sleep(self.delay)
425 424
426 425 def close(self):
427 426 if self.zeromq is 1:
428 427 self.dataOut.finished = True
429 428 self.zmq_socket.send_pyobj(self.dataOut)
430 429 time.sleep(0.1)
431 430 self.zmq_socket.close()
432 431 if self.client:
433 432 self.client.loop_stop()
434 433 self.client.disconnect()
435 434
436 435
437 436 class ReceiverData(ProcessingUnit):
438 437
439 438 __attrs__ = ['server']
440 439
441 440 def __init__(self, **kwargs):
442 441
443 442 ProcessingUnit.__init__(self, **kwargs)
444 443
445 444 self.isConfig = False
446 445 server = kwargs.get('server', 'zmq.pipe')
447 446 if 'tcp://' in server:
448 447 address = server
449 448 else:
450 449 address = 'ipc:///tmp/%s' % server
451 450
452 451 self.address = address
453 452 self.dataOut = JROData()
454 453
455 454 def setup(self):
456 455
457 456 self.context = zmq.Context()
458 457 self.receiver = self.context.socket(zmq.PULL)
459 458 self.receiver.bind(self.address)
460 459 time.sleep(0.5)
461 460 log.success('ReceiverData from {}'.format(self.address))
462 461
463 462
464 463 def run(self):
465 464
466 465 if not self.isConfig:
467 466 self.setup()
468 467 self.isConfig = True
469 468
470 469 self.dataOut = self.receiver.recv_pyobj()
471 470 log.log('{} - {}'.format(self.dataOut.type,
472 471 self.dataOut.datatime.ctime(),),
473 472 'Receiving')
474 473
475 474
476 475 class PlotterReceiver(ProcessingUnit, Process):
477 476
478 477 throttle_value = 5
479 478 __attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle']
480 479
481 480 def __init__(self, **kwargs):
482 481
483 482 ProcessingUnit.__init__(self, **kwargs)
484 483 Process.__init__(self)
485 484 self.mp = False
486 485 self.isConfig = False
487 486 self.isWebConfig = False
488 487 self.connections = 0
489 488 server = kwargs.get('server', 'zmq.pipe')
490 489 plot_server = kwargs.get('plot_server', 'zmq.web')
491 490 if 'tcp://' in server:
492 491 address = server
493 492 else:
494 493 address = 'ipc:///tmp/%s' % server
495 494
496 495 if 'tcp://' in plot_server:
497 496 plot_address = plot_server
498 497 else:
499 498 plot_address = 'ipc:///tmp/%s' % plot_server
500 499
501 500 self.address = address
502 501 self.plot_address = plot_address
503 502 self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
504 503 self.realtime = kwargs.get('realtime', False)
505 504 self.localtime = kwargs.get('localtime', True)
506 505 self.throttle_value = kwargs.get('throttle', 5)
507 506 self.sendData = self.initThrottle(self.throttle_value)
508 507 self.dates = []
509 508 self.setup()
510 509
511 510 def setup(self):
512 511
513 512 self.data = Data(self.plottypes, self.throttle_value)
514 513 self.isConfig = True
515 514
516 515 def event_monitor(self, monitor):
517 516
518 517 events = {}
519 518
520 519 for name in dir(zmq):
521 520 if name.startswith('EVENT_'):
522 521 value = getattr(zmq, name)
523 522 events[value] = name
524 523
525 524 while monitor.poll():
526 525 evt = recv_monitor_message(monitor)
527 526 if evt['event'] == 32:
528 527 self.connections += 1
529 528 if evt['event'] == 512:
530 529 pass
531 530
532 531 evt.update({'description': events[evt['event']]})
533 532
534 533 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
535 534 break
536 535 monitor.close()
537 536 print('event monitor thread done!')
538 537
539 538 def initThrottle(self, throttle_value):
540 539
541 540 @throttle(seconds=throttle_value)
542 541 def sendDataThrottled(fn_sender, data):
543 542 fn_sender(data)
544 543
545 544 return sendDataThrottled
546 545
547 546 def send(self, data):
548 547 log.success('Sending {}'.format(data), self.name)
549 548 self.sender.send_pyobj(data)
550 549
551 550 def run(self):
552 551
553 552 log.success(
554 553 'Starting from {}'.format(self.address),
555 554 self.name
556 555 )
557 556
558 557 self.context = zmq.Context()
559 558 self.receiver = self.context.socket(zmq.PULL)
560 559 self.receiver.bind(self.address)
561 560 monitor = self.receiver.get_monitor_socket()
562 561 self.sender = self.context.socket(zmq.PUB)
563 562 if self.realtime:
564 563 self.sender_web = self.context.socket(zmq.PUB)
565 564 self.sender_web.connect(self.plot_address)
566 565 time.sleep(1)
567 566
568 567 if 'server' in self.kwargs:
569 568 self.sender.bind("ipc:///tmp/{}.plots".format(self.kwargs['server']))
570 569 else:
571 570 self.sender.bind("ipc:///tmp/zmq.plots")
572 571
573 572 time.sleep(2)
574 573
575 574 t = Thread(target=self.event_monitor, args=(monitor,))
576 575 t.start()
577 576
578 577 while True:
579 578 dataOut = self.receiver.recv_pyobj()
580 579 if not dataOut.flagNoData:
581 if dataOut.type == 'Parameters':
580 if dataOut.type == 'Parameters':
582 581 tm = dataOut.utctimeInit
583 582 else:
584 583 tm = dataOut.utctime
585 584 if dataOut.useLocalTime:
586 585 if not self.localtime:
587 586 tm += time.timezone
588 587 dt = datetime.datetime.fromtimestamp(tm).date()
589 588 else:
590 589 if self.localtime:
591 590 tm -= time.timezone
592 591 dt = datetime.datetime.utcfromtimestamp(tm).date()
593 592 coerce = False
594 593 if dt not in self.dates:
595 594 if self.data:
596 595 self.data.ended = True
597 596 self.send(self.data)
598 597 coerce = True
599 598 self.data.setup()
600 599 self.dates.append(dt)
601 600
602 self.data.update(dataOut)
601 self.data.update(dataOut, tm)
603 602
604 603 if dataOut.finished is True:
605 604 self.connections -= 1
606 605 if self.connections == 0 and dt in self.dates:
607 606 self.data.ended = True
608 607 self.send(self.data)
609 608 self.data.setup()
610 609 else:
611 610 if self.realtime:
612 611 self.send(self.data)
613 612 # self.sender_web.send_string(self.data.jsonify())
614 613 else:
615 614 self.sendData(self.send, self.data, coerce=coerce)
616 615 coerce = False
617 616
618 617 return
619 618
620 619 def sendToWeb(self):
621 620
622 621 if not self.isWebConfig:
623 622 context = zmq.Context()
624 623 sender_web_config = context.socket(zmq.PUB)
625 624 if 'tcp://' in self.plot_address:
626 625 dum, address, port = self.plot_address.split(':')
627 626 conf_address = '{}:{}:{}'.format(dum, address, int(port)+1)
628 627 else:
629 628 conf_address = self.plot_address + '.config'
630 629 sender_web_config.bind(conf_address)
631 630 time.sleep(1)
632 631 for kwargs in self.operationKwargs.values():
633 632 if 'plot' in kwargs:
634 633 log.success('[Sending] Config data to web for {}'.format(kwargs['code'].upper()))
635 634 sender_web_config.send_string(json.dumps(kwargs))
636 635 self.isWebConfig = True
General Comments 0
You need to be logged in to leave comments. Login now