##// END OF EJS Templates
Fix xmin & xmax in plots, fix SendToFTP
jespinoza -
r1334:96199ee477a0
parent child
Show More
@@ -94,7 +94,7 op = proc_unit.addOperation(name='removeDC')
94 94 op = proc_unit.addOperation(name='SpectraPlot')
95 95 op.addParameter(name='wintitle', value='Spectra', format='str')
96 96
97 op = procUnitConfObj1.addOperation(name='RTIPlot')
97 op = proc_unit.addOperation(name='RTIPlot')
98 98 op.addParameter(name='wintitle', value='RTI', format='str')
99 99
100 100 prj.start()
@@ -1,3 +1,10
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
3 #
4 # Distributed under the terms of the BSD 3-clause license.
5 """Base class to create plot operations
6
7 """
1 8
2 9 import os
3 10 import sys
@@ -5,10 +12,7 import zmq
5 12 import time
6 13 import numpy
7 14 import datetime
8 try:
9 from queue import Queue
10 except:
11 from Queue import Queue
15 from multiprocessing import Queue
12 16 from functools import wraps
13 17 from threading import Thread
14 18 import matplotlib
@@ -145,9 +149,24 def apply_throttle(value):
145 149
146 150 @MPDecorator
147 151 class Plot(Operation):
148 '''
149 Base class for Schain plotting operations
150 '''
152 """Base class for Schain plotting operations
153
154 This class should never be use directtly you must subclass a new operation,
155 children classes must be defined as follow:
156
157 ExamplePlot(Plot):
158
159 CODE = 'code'
160 colormap = 'jet'
161 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
162
163 def setup(self):
164 pass
165
166 def plot(self):
167 pass
168
169 """
151 170
152 171 CODE = 'Figure'
153 172 colormap = 'jet'
@@ -163,7 +182,7 class Plot(Operation):
163 182 Operation.__init__(self)
164 183 self.isConfig = False
165 184 self.isPlotConfig = False
166 self.save_counter = 1
185 self.save_time = 0
167 186 self.sender_time = 0
168 187 self.data = None
169 188 self.firsttime = True
@@ -187,7 +206,7 class Plot(Operation):
187 206 self.localtime = kwargs.pop('localtime', True)
188 207 self.show = kwargs.get('show', True)
189 208 self.save = kwargs.get('save', False)
190 self.save_period = kwargs.get('save_period', 1)
209 self.save_period = kwargs.get('save_period', 60)
191 210 self.colormap = kwargs.get('colormap', self.colormap)
192 211 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
193 212 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
@@ -224,7 +243,7 class Plot(Operation):
224 243 self.type = kwargs.get('type', 'iq')
225 244 self.grid = kwargs.get('grid', False)
226 245 self.pause = kwargs.get('pause', False)
227 self.save_code = kwargs.get('save_code', None)
246 self.save_code = kwargs.get('save_code', self.CODE)
228 247 self.throttle = kwargs.get('throttle', 0)
229 248 self.exp_code = kwargs.get('exp_code', None)
230 249 self.plot_server = kwargs.get('plot_server', False)
@@ -242,8 +261,6 class Plot(Operation):
242 261 'Sending to server: {}'.format(self.plot_server),
243 262 self.name
244 263 )
245 if 'plot_name' in kwargs:
246 self.plot_name = kwargs['plot_name']
247 264
248 265 def __setup_plot(self):
249 266 '''
@@ -358,85 +375,29 class Plot(Operation):
358 375 Set min and max values, labels, ticks and titles
359 376 '''
360 377
361 if self.xmin is None:
362 xmin = self.data.min_time
363 else:
364 if self.xaxis is 'time':
365 dt = self.getDateTime(self.data.min_time)
366 xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
367 datetime.datetime(1970, 1, 1)).total_seconds()
368 if self.data.localtime:
369 xmin += time.timezone
370 else:
378 for n, ax in enumerate(self.axes):
379 if ax.firsttime:
380 if self.xaxis != 'time':
371 381 xmin = self.xmin
372
373 if self.xmax is None:
374 xmax = xmin + self.xrange * 60 * 60
375 else:
376 if self.xaxis is 'time':
377 dt = self.getDateTime(self.data.max_time)
378 xmax = self.xmax - 1
379 xmax = (dt.replace(hour=int(xmax), minute=59, second=59) -
380 datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds()
381 if self.data.localtime:
382 xmax += time.timezone
383 else:
384 382 xmax = self.xmax
385
383 else:
384 xmin = self.tmin
385 xmax = self.tmin + self.xrange*60*60
386 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
387 ax.xaxis.set_major_locator(LinearLocator(9))
386 388 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
387 389 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
388
389 for n, ax in enumerate(self.axes):
390 if ax.firsttime:
391
392 dig = int(numpy.log10(ymax))
393 if dig == 0:
394 digD = len(str(ymax)) - 2
395 ydec = ymax*(10**digD)
396
397 dig = int(numpy.log10(ydec))
398 ystep = ((ydec + (10**(dig)))//10**(dig))*(10**(dig))
399 ystep = ystep/5
400 ystep = ystep/(10**digD)
401
402 else:
403 ystep = ((ymax + (10**(dig)))//10**(dig))*(10**(dig))
404 ystep = ystep/5
405
406 if self.xaxis is not 'time':
407
408 dig = int(numpy.log10(xmax))
409
410 if dig <= 0:
411 digD = len(str(xmax)) - 2
412 xdec = xmax*(10**digD)
413
414 dig = int(numpy.log10(xdec))
415 xstep = ((xdec + (10**(dig)))//10**(dig))*(10**(dig))
416 xstep = xstep*0.5
417 xstep = xstep/(10**digD)
418
419 else:
420 xstep = ((xmax + (10**(dig)))//10**(dig))*(10**(dig))
421 xstep = xstep/5
422
423 390 ax.set_facecolor(self.bgcolor)
424 ax.yaxis.set_major_locator(MultipleLocator(ystep))
425 391 if self.xscale:
426 392 ax.xaxis.set_major_formatter(FuncFormatter(
427 393 lambda x, pos: '{0:g}'.format(x*self.xscale)))
428 if self.xscale:
394 if self.yscale:
429 395 ax.yaxis.set_major_formatter(FuncFormatter(
430 396 lambda x, pos: '{0:g}'.format(x*self.yscale)))
431 if self.xaxis is 'time':
432 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
433 ax.xaxis.set_major_locator(LinearLocator(9))
434 else:
435 ax.xaxis.set_major_locator(MultipleLocator(xstep))
436 397 if self.xlabel is not None:
437 398 ax.set_xlabel(self.xlabel)
399 if self.ylabel is not None:
438 400 ax.set_ylabel(self.ylabel)
439 ax.firsttime = False
440 401 if self.showprofile:
441 402 self.pf_axes[n].set_ylim(ymin, ymax)
442 403 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
@@ -455,12 +416,12 class Plot(Operation):
455 416 ax.cbar.set_label(self.cb_labels[n], size=8)
456 417 else:
457 418 ax.cbar = None
419 ax.set_xlim(xmin, xmax)
420 ax.set_ylim(ymin, ymax)
421 ax.firsttime = False
458 422 if self.grid:
459 423 ax.grid(True)
460
461 424 if not self.polar:
462 ax.set_xlim(xmin, xmax)
463 ax.set_ylim(ymin, ymax)
464 425 ax.set_title('{} {} {}'.format(
465 426 self.titles[n],
466 427 self.getDateTime(self.data.max_time).strftime(
@@ -521,27 +482,18 class Plot(Operation):
521 482 '''
522 483 '''
523 484
524 if self.save_counter < self.save_period:
525 self.save_counter += 1
485 if (self.data.tm - self.save_time) < self.sender_period:
526 486 return
527 487
528 self.save_counter = 1
488 self.save_time = self.data.tm
529 489
530 490 fig = self.figures[n]
531 491
532 if self.save_code:
533 if isinstance(self.save_code, str):
534 labels = [self.save_code for x in self.figures]
535 else:
536 labels = self.save_code
537 else:
538 labels = [self.CODE for x in self.figures]
539
540 492 figname = os.path.join(
541 493 self.save,
542 labels[n],
494 self.save_code,
543 495 '{}_{}.png'.format(
544 labels[n],
496 self.save_code,
545 497 self.getDateTime(self.data.max_time).strftime(
546 498 '%Y%m%d_%H%M%S'
547 499 ),
@@ -556,7 +508,7 class Plot(Operation):
556 508 figname = os.path.join(
557 509 self.save,
558 510 '{}_{}.png'.format(
559 labels[n],
511 self.save_code,
560 512 self.getDateTime(self.data.min_time).strftime(
561 513 '%Y%m%d'
562 514 ),
@@ -588,7 +540,7 class Plot(Operation):
588 540 else:
589 541 self.data.meta['colormap'] = 'Viridis'
590 542 self.data.meta['interval'] = int(interval)
591 # msg = self.data.jsonify(self.data.tm, self.plot_name, self.plot_type)
543
592 544 try:
593 545 self.sender_queue.put(self.data.tm, block=False)
594 546 except:
@@ -600,7 +552,7 class Plot(Operation):
600 552 break
601 553 tm = self.sender_queue.get()
602 554 try:
603 msg = self.data.jsonify(tm, self.plot_name, self.plot_type)
555 msg = self.data.jsonify(tm, self.save_code, self.plot_type)
604 556 except:
605 557 continue
606 558 self.socket.send_string(msg)
@@ -655,24 +607,11 class Plot(Operation):
655 607 if self.isConfig is False:
656 608 self.__setup(**kwargs)
657 609
658 t = getattr(dataOut, self.attr_time)
659
660 610 if self.localtime:
661 611 self.getDateTime = datetime.datetime.fromtimestamp
662 612 else:
663 613 self.getDateTime = datetime.datetime.utcfromtimestamp
664 614
665 if self.xmin is None:
666 self.tmin = t
667 if 'buffer' in self.plot_type:
668 self.xmin = self.getDateTime(t).hour
669 else:
670 self.tmin = (
671 self.getDateTime(t).replace(
672 hour=int(self.xmin),
673 minute=0,
674 second=0) - self.getDateTime(0)).total_seconds()
675
676 615 self.data.setup()
677 616 self.isConfig = True
678 617 if self.plot_server:
@@ -684,13 +623,9 class Plot(Operation):
684 623
685 624 tm = getattr(dataOut, self.attr_time)
686 625
687 if self.data and (tm - self.tmin) >= self.xrange*60*60:
626 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
688 627 self.save_counter = self.save_period
689 628 self.__plot()
690 if 'time' in self.xaxis:
691 self.xmin += self.xrange
692 if self.xmin >= 24:
693 self.xmin -= 24
694 629 self.tmin += self.xrange*60*60
695 630 self.data.setup()
696 631 self.clear_figures()
@@ -700,6 +635,20 class Plot(Operation):
700 635 if self.isPlotConfig is False:
701 636 self.__setup_plot()
702 637 self.isPlotConfig = True
638 if self.xaxis == 'time':
639 dt = self.getDateTime(tm)
640 if self.xmin is None:
641 self.tmin = tm
642 self.xmin = dt.hour
643 minutes = (self.xmin-int(self.xmin)) * 60
644 seconds = (minutes - int(minutes)) * 60
645 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
646 datetime.datetime(1970, 1, 1)).total_seconds()
647 if self.localtime:
648 self.tmin += time.timezone
649
650 if self.xmin is not None and self.xmax is not None:
651 self.xrange = self.xmax - self.xmin
703 652
704 653 if self.throttle == 0:
705 654 self.__plot()
@@ -36,7 +36,6 class SpectralMomentsPlot(SpectraPlot):
36 36 '''
37 37 CODE = 'spc_moments'
38 38 colormap = 'jet'
39 plot_name = 'SpectralMoments'
40 39 plot_type = 'pcolor'
41 40
42 41
@@ -47,7 +46,6 class SnrPlot(RTIPlot):
47 46
48 47 CODE = 'snr'
49 48 colormap = 'jet'
50 plot_name = 'SNR'
51 49
52 50
53 51 class DopplerPlot(RTIPlot):
@@ -57,7 +55,6 class DopplerPlot(RTIPlot):
57 55
58 56 CODE = 'dop'
59 57 colormap = 'jet'
60 plot_name = 'DopplerShift'
61 58
62 59
63 60 class PowerPlot(RTIPlot):
@@ -67,7 +64,6 class PowerPlot(RTIPlot):
67 64
68 65 CODE = 'pow'
69 66 colormap = 'jet'
70 plot_name = 'TotalPower'
71 67
72 68
73 69 class SpectralWidthPlot(RTIPlot):
@@ -77,7 +73,6 class SpectralWidthPlot(RTIPlot):
77 73
78 74 CODE = 'width'
79 75 colormap = 'jet'
80 plot_name = 'SpectralWidth'
81 76
82 77
83 78 class SkyMapPlot(Plot):
@@ -135,7 +130,6 class ParametersPlot(RTIPlot):
135 130
136 131 CODE = 'param'
137 132 colormap = 'seismic'
138 plot_name = 'Parameters'
139 133
140 134 def setup(self):
141 135 self.xaxis = 'time'
@@ -210,7 +204,6 class OutputPlot(ParametersPlot):
210 204
211 205 CODE = 'output'
212 206 colormap = 'seismic'
213 plot_name = 'Output'
214 207
215 208
216 209 class PolarMapPlot(Plot):
@@ -19,14 +19,13 class SpectraPlot(Plot):
19 19
20 20 CODE = 'spc'
21 21 colormap = 'jet'
22 plot_name = 'Spectra'
23 22 plot_type = 'pcolor'
24 23
25 24 def setup(self):
26 25 self.nplots = len(self.data.channels)
27 26 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
28 27 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
29 self.height = 3 * self.nrows
28 self.height = 2.6 * self.nrows
30 29 self.cb_label = 'dB'
31 30 if self.showprofile:
32 31 self.width = 4 * self.ncols
@@ -92,7 +91,6 class CrossSpectraPlot(Plot):
92 91
93 92 CODE = 'cspc'
94 93 colormap = 'jet'
95 plot_name = 'CrossSpectra'
96 94 plot_type = 'pcolor'
97 95 zmin_coh = None
98 96 zmax_coh = None
@@ -104,11 +102,11 class CrossSpectraPlot(Plot):
104 102 self.ncols = 4
105 103 self.nrows = len(self.data.pairs)
106 104 self.nplots = self.nrows * 4
107 self.width = 3.4 * self.ncols
108 self.height = 3 * self.nrows
105 self.width = 3.1 * self.ncols
106 self.height = 2.6 * self.nrows
109 107 self.ylabel = 'Range [km]'
110 108 self.showprofile = False
111 self.plots_adjust.update({'bottom': 0.08})
109 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
112 110
113 111 def plot(self):
114 112
@@ -194,7 +192,6 class RTIPlot(Plot):
194 192
195 193 CODE = 'rti'
196 194 colormap = 'jet'
197 plot_name = 'RTI'
198 195 plot_type = 'pcolorbuffer'
199 196
200 197 def setup(self):
@@ -253,7 +250,6 class CoherencePlot(RTIPlot):
253 250 '''
254 251
255 252 CODE = 'coh'
256 plot_name = 'Coherence'
257 253
258 254 def setup(self):
259 255 self.xaxis = 'time'
@@ -280,7 +276,6 class PhasePlot(CoherencePlot):
280 276
281 277 CODE = 'phase'
282 278 colormap = 'seismic'
283 plot_name = 'Phase'
284 279
285 280
286 281 class NoisePlot(Plot):
@@ -289,7 +284,6 class NoisePlot(Plot):
289 284 '''
290 285
291 286 CODE = 'noise'
292 plot_name = 'Noise'
293 287 plot_type = 'scatterbuffer'
294 288
295 289
@@ -327,7 +321,6 class NoisePlot(Plot):
327 321 class PowerProfilePlot(Plot):
328 322
329 323 CODE = 'spcprofile'
330 plot_name = 'Power Profile'
331 324 plot_type = 'scatter'
332 325 buffering = False
333 326
@@ -365,7 +358,6 class PowerProfilePlot(Plot):
365 358 class SpectraCutPlot(Plot):
366 359
367 360 CODE = 'spc_cut'
368 plot_name = 'Spectra Cut'
369 361 plot_type = 'scatter'
370 362 buffering = False
371 363
@@ -17,7 +17,6 class ScopePlot(Plot):
17 17 '''
18 18
19 19 CODE = 'scope'
20 plot_name = 'Scope'
21 20 plot_type = 'scatter'
22 21
23 22 def setup(self):
@@ -269,7 +268,6 class PulsepairPowerPlot(ScopePlot):
269 268 '''
270 269
271 270 CODE = 'pp_power'
272 plot_name = 'PulsepairPower'
273 271 plot_type = 'scatter'
274 272 buffering = False
275 273
@@ -278,7 +276,6 class PulsepairVelocityPlot(ScopePlot):
278 276 Plot for VELOCITY
279 277 '''
280 278 CODE = 'pp_velocity'
281 plot_name = 'PulsepairVelocity'
282 279 plot_type = 'scatter'
283 280 buffering = False
284 281
@@ -287,7 +284,6 class PulsepairSpecwidthPlot(ScopePlot):
287 284 Plot for WIDTH
288 285 '''
289 286 CODE = 'pp_specwidth'
290 plot_name = 'PulsepairSpecwidth'
291 287 plot_type = 'scatter'
292 288 buffering = False
293 289
@@ -297,6 +293,5 class PulsepairSignalPlot(ScopePlot):
297 293 '''
298 294
299 295 CODE = 'pp_signal'
300 plot_name = 'PulsepairSignal'
301 296 plot_type = 'scatter'
302 297 buffering = False
@@ -1,6 +1,9
1 '''
2 @author: Juan C. Espinoza
3 '''
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
3 #
4 # Distributed under the terms of the BSD 3-clause license.
5 """Utilities for publish/send data, files & plots over different protocols
6 """
4 7
5 8 import os
6 9 import glob
@@ -18,8 +21,6 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit, MPDecora
18 21 from schainpy.model.data.jrodata import JROData
19 22 from schainpy.utils import log
20 23
21 MAXNUMX = 500
22 MAXNUMY = 500
23 24
24 25 PLOT_CODES = {
25 26 'rti': 0, # Range time intensity (RTI).
@@ -52,11 +53,6 def get_plot_code(s):
52 53 else:
53 54 return 24
54 55
55 def decimate(z, MAXNUMY):
56 dy = int(len(z[0])/MAXNUMY) + 1
57
58 return z[::, ::dy]
59
60 56
61 57 class PublishData(Operation):
62 58 '''
@@ -150,11 +146,46 class ReceiverData(ProcessingUnit):
150 146
151 147 @MPDecorator
152 148 class SendToFTP(Operation):
149 """Operation for send files over FTP
150
151 This operation is used to send files over FTP, you can send different files
152 from different folders by adding as many `pattern` as you wish.
153
154 Parameters:
155 -----------
156 server : str
157 FTP server address.
158 username : str
159 FTP username
160 password : str
161 FTP password
162 timeout : int
163 timeout to restart the connection
164 patternX : list
165 detail of files to be send must have the following order: local, remote
166 ext, period, exp_code, sub_exp_code
167
168 Example:
169 --------
170
171 ftp = proc_unit.addOperation(name='SendToFTP', optype='external')
172 ftp.addParameter(name='server', value='jro-app.igp.gob.pe')
173 ftp.addParameter(name='username', value='wmaster')
174 ftp.addParameter(name='password', value='mst2010vhf')
175 ftp.addParameter(
176 name='pattern1',
177 value='/local/path/rti,/remote/path,png,300,11,0'
178 )
179 ftp.addParameter(
180 name='pattern2',
181 value='/local/path/spc,/remote/path,png,300,11,0'
182 )
183 ftp.addParameter(
184 name='pattern3',
185 value='/local/path/param,/remote/path,hdf5,300,,'
186 )
153 187
154 '''
155 Operation to send data over FTP.
156 patternX = 'local, remote, ext, period, exp_code, sub_exp_code'
157 '''
188 """
158 189
159 190 __attrs__ = ['server', 'username', 'password', 'timeout', 'patternX']
160 191
@@ -179,7 +210,7 class SendToFTP(Operation):
179 210 for arg, value in kwargs.items():
180 211 if 'pattern' in arg:
181 212 self.patterns.append(value)
182 self.times.append(time.time())
213 self.times.append(0)
183 214 self.latest.append('')
184 215
185 216 def connect(self):
@@ -224,7 +255,7 class SendToFTP(Operation):
224 255
225 256 def find_files(self, path, ext):
226 257
227 files = glob.glob1(path, '*{}'.format(ext))
258 files = glob.glob1(path.strip(), '*{}'.format(ext.strip()))
228 259 files.sort()
229 260 if files:
230 261 return files[-1]
@@ -256,18 +287,12 class SendToFTP(Operation):
256 287 self.ftp.storbinary(command, fp, blocksize=1024)
257 288 except Exception as e:
258 289 log.error('{}'.format(e), self.name)
259 if self.ftp is not None:
260 self.ftp.close()
261 self.ftp = None
262 290 return 0
263 291
264 292 try:
265 293 self.ftp.sendcmd('SITE CHMOD 755 {}'.format(dst))
266 294 except Exception as e:
267 295 log.error('{}'.format(e), self.name)
268 if self.ftp is not None:
269 self.ftp.close()
270 self.ftp = None
271 296 return 0
272 297
273 298 fp.close()
@@ -278,30 +303,30 class SendToFTP(Operation):
278 303
279 304 for x, pattern in enumerate(self.patterns):
280 305 local, remote, ext, period, exp_code, sub_exp_code = pattern
281 if time.time()-self.times[x] >= int(period):
306
307 if (self.dataOut.utctime - self.times[x]) < int(period):
308 continue
309
282 310 srcname = self.find_files(local, ext)
283 src = os.path.join(local, srcname)
284 if os.path.getmtime(src) < time.time() - 30*60:
285 log.warning('Skipping old file {}'.format(srcname))
311
312 if srcname is None:
286 313 continue
287 314
288 if srcname is None or srcname == self.latest[x]:
315 if srcname == self.latest[x]:
289 316 log.warning('File alreday uploaded {}'.format(srcname))
290 317 continue
291 318
292 if 'png' in ext:
319 if exp_code.strip():
293 320 dstname = self.getftpname(srcname, int(exp_code), int(sub_exp_code))
294 321 else:
295 322 dstname = srcname
296 323
297 dst = os.path.join(remote, dstname)
324 src = os.path.join(local, srcname)
325 dst = os.path.join(remote.strip(), dstname)
298 326
299 327 if self.upload(src, dst):
300 self.times[x] = time.time()
328 self.times[x] = self.dataOut.utctime
301 329 self.latest[x] = srcname
302 else:
303 self.ready = False
304 break
305 330
306 331 def run(self, dataOut, server, username, password, timeout=10, **kwargs):
307 332
@@ -314,9 +339,9 class SendToFTP(Operation):
314 339 **kwargs
315 340 )
316 341 self.isConfig = True
317 if not self.ready:
318 342 self.connect()
319 if self.ftp is not None:
343
344 self.dataOut = dataOut
320 345 self.check()
321 346 self.send_files()
322 347
General Comments 0
You need to be logged in to leave comments. Login now