##// END OF EJS Templates
schain
RSS
Fix xmin & xmax in plots, fix SendToFTP
jespinoza -
r1334:96199ee477a0
parent child
Show More
Side by Side Unified
Context file:
r1334:96199ee477a0 Loading diff...:
Show file before | Show file after | Hide comments
@@ -94,7 +94,7 op = proc_unit.addOperation(name='removeDC')
94 op = proc_unit.addOperation(name='SpectraPlot')
94 op = proc_unit.addOperation(name='SpectraPlot')
95 op.addParameter(name='wintitle', value='Spectra', format='str')
95 op.addParameter(name='wintitle', value='Spectra', format='str')
96
96
97 op = procUnitConfObj1.addOperation(name='RTIPlot')
97 op = proc_unit.addOperation(name='RTIPlot')
98 op.addParameter(name='wintitle', value='RTI', format='str')
98 op.addParameter(name='wintitle', value='RTI', format='str')
99
99
100 prj.start()
100 prj.start()
Show file before | Show file after | Hide comments
@@ -1,3 +1,10
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
3 #
4 # Distributed under the terms of the BSD 3-clause license.
5 """Base class to create plot operations
6
7 """
1
8
2 import os
9 import os
3 import sys
10 import sys
@@ -5,10 +12,7 import zmq
5 import time
12 import time
6 import numpy
13 import numpy
7 import datetime
14 import datetime
8 try:
15 from multiprocessing import Queue
9 from queue import Queue
10 except:
11 from Queue import Queue
12 from functools import wraps
16 from functools import wraps
13 from threading import Thread
17 from threading import Thread
14 import matplotlib
18 import matplotlib
@@ -145,9 +149,24 def apply_throttle(value):
145
149
146 @MPDecorator
150 @MPDecorator
147 class Plot(Operation):
151 class Plot(Operation):
148 '''
152 """Base class for Schain plotting operations
149 Base class for Schain plotting operations
153
150 '''
154 This class should never be use directtly you must subclass a new operation,
155 children classes must be defined as follow:
156
157 ExamplePlot(Plot):
158
159 CODE = 'code'
160 colormap = 'jet'
161 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
162
163 def setup(self):
164 pass
165
166 def plot(self):
167 pass
168
169 """
151
170
152 CODE = 'Figure'
171 CODE = 'Figure'
153 colormap = 'jet'
172 colormap = 'jet'
@@ -163,7 +182,7 class Plot(Operation):
163 Operation.__init__(self)
182 Operation.__init__(self)
164 self.isConfig = False
183 self.isConfig = False
165 self.isPlotConfig = False
184 self.isPlotConfig = False
166 self.save_counter = 1
185 self.save_time = 0
167 self.sender_time = 0
186 self.sender_time = 0
168 self.data = None
187 self.data = None
169 self.firsttime = True
188 self.firsttime = True
@@ -187,7 +206,7 class Plot(Operation):
187 self.localtime = kwargs.pop('localtime', True)
206 self.localtime = kwargs.pop('localtime', True)
188 self.show = kwargs.get('show', True)
207 self.show = kwargs.get('show', True)
189 self.save = kwargs.get('save', False)
208 self.save = kwargs.get('save', False)
190 self.save_period = kwargs.get('save_period', 1)
209 self.save_period = kwargs.get('save_period', 60)
191 self.colormap = kwargs.get('colormap', self.colormap)
210 self.colormap = kwargs.get('colormap', self.colormap)
192 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
211 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
193 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
212 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
@@ -224,7 +243,7 class Plot(Operation):
224 self.type = kwargs.get('type', 'iq')
243 self.type = kwargs.get('type', 'iq')
225 self.grid = kwargs.get('grid', False)
244 self.grid = kwargs.get('grid', False)
226 self.pause = kwargs.get('pause', False)
245 self.pause = kwargs.get('pause', False)
227 self.save_code = kwargs.get('save_code', None)
246 self.save_code = kwargs.get('save_code', self.CODE)
228 self.throttle = kwargs.get('throttle', 0)
247 self.throttle = kwargs.get('throttle', 0)
229 self.exp_code = kwargs.get('exp_code', None)
248 self.exp_code = kwargs.get('exp_code', None)
230 self.plot_server = kwargs.get('plot_server', False)
249 self.plot_server = kwargs.get('plot_server', False)
@@ -242,8 +261,6 class Plot(Operation):
242 'Sending to server: {}'.format(self.plot_server),
261 'Sending to server: {}'.format(self.plot_server),
243 self.name
262 self.name
244 )
263 )
245 if 'plot_name' in kwargs:
246 self.plot_name = kwargs['plot_name']
247
264
248 def __setup_plot(self):
265 def __setup_plot(self):
249 '''
266 '''
@@ -357,86 +374,30 class Plot(Operation):
357 '''
374 '''
358 Set min and max values, labels, ticks and titles
375 Set min and max values, labels, ticks and titles
359 '''
376 '''
360
361 if self.xmin is None:
362 xmin = self.data.min_time
363 else:
364 if self.xaxis is 'time':
365 dt = self.getDateTime(self.data.min_time)
366 xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
367 datetime.datetime(1970, 1, 1)).total_seconds()
368 if self.data.localtime:
369 xmin += time.timezone
370 else:
371 xmin = self.xmin
372
373 if self.xmax is None:
374 xmax = xmin + self.xrange * 60 * 60
375 else:
376 if self.xaxis is 'time':
377 dt = self.getDateTime(self.data.max_time)
378 xmax = self.xmax - 1
379 xmax = (dt.replace(hour=int(xmax), minute=59, second=59) -
380 datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds()
381 if self.data.localtime:
382 xmax += time.timezone
383 else:
384 xmax = self.xmax
385
386 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
387 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
388
377
389 for n, ax in enumerate(self.axes):
378 for n, ax in enumerate(self.axes):
390 if ax.firsttime:
379 if ax.firsttime:
391
380 if self.xaxis != 'time':
392 dig = int(numpy.log10(ymax))
381 xmin = self.xmin
393 if dig == 0:
382 xmax = self.xmax
394 digD = len(str(ymax)) - 2
383 else:
395 ydec = ymax*(10**digD)
384 xmin = self.tmin
396
385 xmax = self.tmin + self.xrange*60*60
397 dig = int(numpy.log10(ydec))
386 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
398 ystep = ((ydec + (10**(dig)))//10**(dig))*(10**(dig))
387 ax.xaxis.set_major_locator(LinearLocator(9))
399 ystep = ystep/5
388 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
400 ystep = ystep/(10**digD)
389 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
401
402 else:
403 ystep = ((ymax + (10**(dig)))//10**(dig))*(10**(dig))
404 ystep = ystep/5
405
406 if self.xaxis is not 'time':
407
408 dig = int(numpy.log10(xmax))
409
410 if dig <= 0:
411 digD = len(str(xmax)) - 2
412 xdec = xmax*(10**digD)
413
414 dig = int(numpy.log10(xdec))
415 xstep = ((xdec + (10**(dig)))//10**(dig))*(10**(dig))
416 xstep = xstep*0.5
417 xstep = xstep/(10**digD)
418
419 else:
420 xstep = ((xmax + (10**(dig)))//10**(dig))*(10**(dig))
421 xstep = xstep/5
422
423 ax.set_facecolor(self.bgcolor)
390 ax.set_facecolor(self.bgcolor)
424 ax.yaxis.set_major_locator(MultipleLocator(ystep))
425 if self.xscale:
391 if self.xscale:
426 ax.xaxis.set_major_formatter(FuncFormatter(
392 ax.xaxis.set_major_formatter(FuncFormatter(
427 lambda x, pos: '{0:g}'.format(x*self.xscale)))
393 lambda x, pos: '{0:g}'.format(x*self.xscale)))
428 if self.xscale:
394 if self.yscale:
429 ax.yaxis.set_major_formatter(FuncFormatter(
395 ax.yaxis.set_major_formatter(FuncFormatter(
430 lambda x, pos: '{0:g}'.format(x*self.yscale)))
396 lambda x, pos: '{0:g}'.format(x*self.yscale)))
431 if self.xaxis is 'time':
432 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
433 ax.xaxis.set_major_locator(LinearLocator(9))
434 else:
435 ax.xaxis.set_major_locator(MultipleLocator(xstep))
436 if self.xlabel is not None:
397 if self.xlabel is not None:
437 ax.set_xlabel(self.xlabel)
398 ax.set_xlabel(self.xlabel)
438 ax.set_ylabel(self.ylabel)
399 if self.ylabel is not None:
439 ax.firsttime = False
400 ax.set_ylabel(self.ylabel)
440 if self.showprofile:
401 if self.showprofile:
441 self.pf_axes[n].set_ylim(ymin, ymax)
402 self.pf_axes[n].set_ylim(ymin, ymax)
442 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
403 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
@@ -455,12 +416,12 class Plot(Operation):
455 ax.cbar.set_label(self.cb_labels[n], size=8)
416 ax.cbar.set_label(self.cb_labels[n], size=8)
456 else:
417 else:
457 ax.cbar = None
418 ax.cbar = None
458 if self.grid:
459 ax.grid(True)
460
461 if not self.polar:
462 ax.set_xlim(xmin, xmax)
419 ax.set_xlim(xmin, xmax)
463 ax.set_ylim(ymin, ymax)
420 ax.set_ylim(ymin, ymax)
421 ax.firsttime = False
422 if self.grid:
423 ax.grid(True)
424 if not self.polar:
464 ax.set_title('{} {} {}'.format(
425 ax.set_title('{} {} {}'.format(
465 self.titles[n],
426 self.titles[n],
466 self.getDateTime(self.data.max_time).strftime(
427 self.getDateTime(self.data.max_time).strftime(
@@ -521,27 +482,18 class Plot(Operation):
521 '''
482 '''
522 '''
483 '''
523
484
524 if self.save_counter < self.save_period:
485 if (self.data.tm - self.save_time) < self.sender_period:
525 self.save_counter += 1
526 return
486 return
527
487
528 self.save_counter = 1
488 self.save_time = self.data.tm
529
489
530 fig = self.figures[n]
490 fig = self.figures[n]
531
491
532 if self.save_code:
533 if isinstance(self.save_code, str):
534 labels = [self.save_code for x in self.figures]
535 else:
536 labels = self.save_code
537 else:
538 labels = [self.CODE for x in self.figures]
539
540 figname = os.path.join(
492 figname = os.path.join(
541 self.save,
493 self.save,
542 labels[n],
494 self.save_code,
543 '{}_{}.png'.format(
495 '{}_{}.png'.format(
544 labels[n],
496 self.save_code,
545 self.getDateTime(self.data.max_time).strftime(
497 self.getDateTime(self.data.max_time).strftime(
546 '%Y%m%d_%H%M%S'
498 '%Y%m%d_%H%M%S'
547 ),
499 ),
@@ -556,7 +508,7 class Plot(Operation):
556 figname = os.path.join(
508 figname = os.path.join(
557 self.save,
509 self.save,
558 '{}_{}.png'.format(
510 '{}_{}.png'.format(
559 labels[n],
511 self.save_code,
560 self.getDateTime(self.data.min_time).strftime(
512 self.getDateTime(self.data.min_time).strftime(
561 '%Y%m%d'
513 '%Y%m%d'
562 ),
514 ),
@@ -588,7 +540,7 class Plot(Operation):
588 else:
540 else:
589 self.data.meta['colormap'] = 'Viridis'
541 self.data.meta['colormap'] = 'Viridis'
590 self.data.meta['interval'] = int(interval)
542 self.data.meta['interval'] = int(interval)
591 # msg = self.data.jsonify(self.data.tm, self.plot_name, self.plot_type)
543
592 try:
544 try:
593 self.sender_queue.put(self.data.tm, block=False)
545 self.sender_queue.put(self.data.tm, block=False)
594 except:
546 except:
@@ -600,7 +552,7 class Plot(Operation):
600 break
552 break
601 tm = self.sender_queue.get()
553 tm = self.sender_queue.get()
602 try:
554 try:
603 msg = self.data.jsonify(tm, self.plot_name, self.plot_type)
555 msg = self.data.jsonify(tm, self.save_code, self.plot_type)
604 except:
556 except:
605 continue
557 continue
606 self.socket.send_string(msg)
558 self.socket.send_string(msg)
@@ -654,24 +606,11 class Plot(Operation):
654
606
655 if self.isConfig is False:
607 if self.isConfig is False:
656 self.__setup(**kwargs)
608 self.__setup(**kwargs)
657
658 t = getattr(dataOut, self.attr_time)
659
609
660 if self.localtime:
610 if self.localtime:
661 self.getDateTime = datetime.datetime.fromtimestamp
611 self.getDateTime = datetime.datetime.fromtimestamp
662 else:
612 else:
663 self.getDateTime = datetime.datetime.utcfromtimestamp
613 self.getDateTime = datetime.datetime.utcfromtimestamp
664
665 if self.xmin is None:
666 self.tmin = t
667 if 'buffer' in self.plot_type:
668 self.xmin = self.getDateTime(t).hour
669 else:
670 self.tmin = (
671 self.getDateTime(t).replace(
672 hour=int(self.xmin),
673 minute=0,
674 second=0) - self.getDateTime(0)).total_seconds()
675
614
676 self.data.setup()
615 self.data.setup()
677 self.isConfig = True
616 self.isConfig = True
@@ -684,13 +623,9 class Plot(Operation):
684
623
685 tm = getattr(dataOut, self.attr_time)
624 tm = getattr(dataOut, self.attr_time)
686
625
687 if self.data and (tm - self.tmin) >= self.xrange*60*60:
626 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
688 self.save_counter = self.save_period
627 self.save_counter = self.save_period
689 self.__plot()
628 self.__plot()
690 if 'time' in self.xaxis:
691 self.xmin += self.xrange
692 if self.xmin >= 24:
693 self.xmin -= 24
694 self.tmin += self.xrange*60*60
629 self.tmin += self.xrange*60*60
695 self.data.setup()
630 self.data.setup()
696 self.clear_figures()
631 self.clear_figures()
@@ -700,6 +635,20 class Plot(Operation):
700 if self.isPlotConfig is False:
635 if self.isPlotConfig is False:
701 self.__setup_plot()
636 self.__setup_plot()
702 self.isPlotConfig = True
637 self.isPlotConfig = True
638 if self.xaxis == 'time':
639 dt = self.getDateTime(tm)
640 if self.xmin is None:
641 self.tmin = tm
642 self.xmin = dt.hour
643 minutes = (self.xmin-int(self.xmin)) * 60
644 seconds = (minutes - int(minutes)) * 60
645 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
646 datetime.datetime(1970, 1, 1)).total_seconds()
647 if self.localtime:
648 self.tmin += time.timezone
649
650 if self.xmin is not None and self.xmax is not None:
651 self.xrange = self.xmax - self.xmin
703
652
704 if self.throttle == 0:
653 if self.throttle == 0:
705 self.__plot()
654 self.__plot()
Show file before | Show file after | Hide comments
@@ -36,7 +36,6 class SpectralMomentsPlot(SpectraPlot):
36 '''
36 '''
37 CODE = 'spc_moments'
37 CODE = 'spc_moments'
38 colormap = 'jet'
38 colormap = 'jet'
39 plot_name = 'SpectralMoments'
40 plot_type = 'pcolor'
39 plot_type = 'pcolor'
41
40
42
41
@@ -47,7 +46,6 class SnrPlot(RTIPlot):
47
46
48 CODE = 'snr'
47 CODE = 'snr'
49 colormap = 'jet'
48 colormap = 'jet'
50 plot_name = 'SNR'
51
49
52
50
53 class DopplerPlot(RTIPlot):
51 class DopplerPlot(RTIPlot):
@@ -57,7 +55,6 class DopplerPlot(RTIPlot):
57
55
58 CODE = 'dop'
56 CODE = 'dop'
59 colormap = 'jet'
57 colormap = 'jet'
60 plot_name = 'DopplerShift'
61
58
62
59
63 class PowerPlot(RTIPlot):
60 class PowerPlot(RTIPlot):
@@ -67,7 +64,6 class PowerPlot(RTIPlot):
67
64
68 CODE = 'pow'
65 CODE = 'pow'
69 colormap = 'jet'
66 colormap = 'jet'
70 plot_name = 'TotalPower'
71
67
72
68
73 class SpectralWidthPlot(RTIPlot):
69 class SpectralWidthPlot(RTIPlot):
@@ -77,7 +73,6 class SpectralWidthPlot(RTIPlot):
77
73
78 CODE = 'width'
74 CODE = 'width'
79 colormap = 'jet'
75 colormap = 'jet'
80 plot_name = 'SpectralWidth'
81
76
82
77
83 class SkyMapPlot(Plot):
78 class SkyMapPlot(Plot):
@@ -135,7 +130,6 class ParametersPlot(RTIPlot):
135
130
136 CODE = 'param'
131 CODE = 'param'
137 colormap = 'seismic'
132 colormap = 'seismic'
138 plot_name = 'Parameters'
139
133
140 def setup(self):
134 def setup(self):
141 self.xaxis = 'time'
135 self.xaxis = 'time'
@@ -210,7 +204,6 class OutputPlot(ParametersPlot):
210
204
211 CODE = 'output'
205 CODE = 'output'
212 colormap = 'seismic'
206 colormap = 'seismic'
213 plot_name = 'Output'
214
207
215
208
216 class PolarMapPlot(Plot):
209 class PolarMapPlot(Plot):
Show file before | Show file after | Hide comments
@@ -19,14 +19,13 class SpectraPlot(Plot):
19
19
20 CODE = 'spc'
20 CODE = 'spc'
21 colormap = 'jet'
21 colormap = 'jet'
22 plot_name = 'Spectra'
23 plot_type = 'pcolor'
22 plot_type = 'pcolor'
24
23
25 def setup(self):
24 def setup(self):
26 self.nplots = len(self.data.channels)
25 self.nplots = len(self.data.channels)
27 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
26 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
28 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
27 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
29 self.height = 3 * self.nrows
28 self.height = 2.6 * self.nrows
30 self.cb_label = 'dB'
29 self.cb_label = 'dB'
31 if self.showprofile:
30 if self.showprofile:
32 self.width = 4 * self.ncols
31 self.width = 4 * self.ncols
@@ -92,7 +91,6 class CrossSpectraPlot(Plot):
92
91
93 CODE = 'cspc'
92 CODE = 'cspc'
94 colormap = 'jet'
93 colormap = 'jet'
95 plot_name = 'CrossSpectra'
96 plot_type = 'pcolor'
94 plot_type = 'pcolor'
97 zmin_coh = None
95 zmin_coh = None
98 zmax_coh = None
96 zmax_coh = None
@@ -104,11 +102,11 class CrossSpectraPlot(Plot):
104 self.ncols = 4
102 self.ncols = 4
105 self.nrows = len(self.data.pairs)
103 self.nrows = len(self.data.pairs)
106 self.nplots = self.nrows * 4
104 self.nplots = self.nrows * 4
107 self.width = 3.4 * self.ncols
105 self.width = 3.1 * self.ncols
108 self.height = 3 * self.nrows
106 self.height = 2.6 * self.nrows
109 self.ylabel = 'Range [km]'
107 self.ylabel = 'Range [km]'
110 self.showprofile = False
108 self.showprofile = False
111 self.plots_adjust.update({'bottom': 0.08})
109 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
112
110
113 def plot(self):
111 def plot(self):
114
112
@@ -194,7 +192,6 class RTIPlot(Plot):
194
192
195 CODE = 'rti'
193 CODE = 'rti'
196 colormap = 'jet'
194 colormap = 'jet'
197 plot_name = 'RTI'
198 plot_type = 'pcolorbuffer'
195 plot_type = 'pcolorbuffer'
199
196
200 def setup(self):
197 def setup(self):
@@ -253,7 +250,6 class CoherencePlot(RTIPlot):
253 '''
250 '''
254
251
255 CODE = 'coh'
252 CODE = 'coh'
256 plot_name = 'Coherence'
257
253
258 def setup(self):
254 def setup(self):
259 self.xaxis = 'time'
255 self.xaxis = 'time'
@@ -280,7 +276,6 class PhasePlot(CoherencePlot):
280
276
281 CODE = 'phase'
277 CODE = 'phase'
282 colormap = 'seismic'
278 colormap = 'seismic'
283 plot_name = 'Phase'
284
279
285
280
286 class NoisePlot(Plot):
281 class NoisePlot(Plot):
@@ -289,7 +284,6 class NoisePlot(Plot):
289 '''
284 '''
290
285
291 CODE = 'noise'
286 CODE = 'noise'
292 plot_name = 'Noise'
293 plot_type = 'scatterbuffer'
287 plot_type = 'scatterbuffer'
294
288
295
289
@@ -327,7 +321,6 class NoisePlot(Plot):
327 class PowerProfilePlot(Plot):
321 class PowerProfilePlot(Plot):
328
322
329 CODE = 'spcprofile'
323 CODE = 'spcprofile'
330 plot_name = 'Power Profile'
331 plot_type = 'scatter'
324 plot_type = 'scatter'
332 buffering = False
325 buffering = False
333
326
@@ -365,7 +358,6 class PowerProfilePlot(Plot):
365 class SpectraCutPlot(Plot):
358 class SpectraCutPlot(Plot):
366
359
367 CODE = 'spc_cut'
360 CODE = 'spc_cut'
368 plot_name = 'Spectra Cut'
369 plot_type = 'scatter'
361 plot_type = 'scatter'
370 buffering = False
362 buffering = False
371
363
Show file before | Show file after | Hide comments
@@ -17,7 +17,6 class ScopePlot(Plot):
17 '''
17 '''
18
18
19 CODE = 'scope'
19 CODE = 'scope'
20 plot_name = 'Scope'
21 plot_type = 'scatter'
20 plot_type = 'scatter'
22
21
23 def setup(self):
22 def setup(self):
@@ -269,7 +268,6 class PulsepairPowerPlot(ScopePlot):
269 '''
268 '''
270
269
271 CODE = 'pp_power'
270 CODE = 'pp_power'
272 plot_name = 'PulsepairPower'
273 plot_type = 'scatter'
271 plot_type = 'scatter'
274 buffering = False
272 buffering = False
275
273
@@ -278,7 +276,6 class PulsepairVelocityPlot(ScopePlot):
278 Plot for VELOCITY
276 Plot for VELOCITY
279 '''
277 '''
280 CODE = 'pp_velocity'
278 CODE = 'pp_velocity'
281 plot_name = 'PulsepairVelocity'
282 plot_type = 'scatter'
279 plot_type = 'scatter'
283 buffering = False
280 buffering = False
284
281
@@ -287,7 +284,6 class PulsepairSpecwidthPlot(ScopePlot):
287 Plot for WIDTH
284 Plot for WIDTH
288 '''
285 '''
289 CODE = 'pp_specwidth'
286 CODE = 'pp_specwidth'
290 plot_name = 'PulsepairSpecwidth'
291 plot_type = 'scatter'
287 plot_type = 'scatter'
292 buffering = False
288 buffering = False
293
289
@@ -297,6 +293,5 class PulsepairSignalPlot(ScopePlot):
297 '''
293 '''
298
294
299 CODE = 'pp_signal'
295 CODE = 'pp_signal'
300 plot_name = 'PulsepairSignal'
301 plot_type = 'scatter'
296 plot_type = 'scatter'
302 buffering = False
297 buffering = False
Show file before | Show file after | Hide comments
@@ -1,6 +1,9
1 '''
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 @author: Juan C. Espinoza
2 # All rights reserved.
3 '''
3 #
4 # Distributed under the terms of the BSD 3-clause license.
5 """Utilities for publish/send data, files & plots over different protocols
6 """
4
7
5 import os
8 import os
6 import glob
9 import glob
@@ -18,8 +21,6 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit, MPDecora
18 from schainpy.model.data.jrodata import JROData
21 from schainpy.model.data.jrodata import JROData
19 from schainpy.utils import log
22 from schainpy.utils import log
20
23
21 MAXNUMX = 500
22 MAXNUMY = 500
23
24
24 PLOT_CODES = {
25 PLOT_CODES = {
25 'rti': 0, # Range time intensity (RTI).
26 'rti': 0, # Range time intensity (RTI).
@@ -52,11 +53,6 def get_plot_code(s):
52 else:
53 else:
53 return 24
54 return 24
54
55
55 def decimate(z, MAXNUMY):
56 dy = int(len(z[0])/MAXNUMY) + 1
57
58 return z[::, ::dy]
59
60
56
61 class PublishData(Operation):
57 class PublishData(Operation):
62 '''
58 '''
@@ -150,12 +146,47 class ReceiverData(ProcessingUnit):
150
146
151 @MPDecorator
147 @MPDecorator
152 class SendToFTP(Operation):
148 class SendToFTP(Operation):
153
149 """Operation for send files over FTP
154 '''
150
155 Operation to send data over FTP.
151 This operation is used to send files over FTP, you can send different files
156 patternX = 'local, remote, ext, period, exp_code, sub_exp_code'
152 from different folders by adding as many `pattern` as you wish.
157 '''
153
158
154 Parameters:
155 -----------
156 server : str
157 FTP server address.
158 username : str
159 FTP username
160 password : str
161 FTP password
162 timeout : int
163 timeout to restart the connection
164 patternX : list
165 detail of files to be send must have the following order: local, remote
166 ext, period, exp_code, sub_exp_code
167
168 Example:
169 --------
170
171 ftp = proc_unit.addOperation(name='SendToFTP', optype='external')
172 ftp.addParameter(name='server', value='jro-app.igp.gob.pe')
173 ftp.addParameter(name='username', value='wmaster')
174 ftp.addParameter(name='password', value='mst2010vhf')
175 ftp.addParameter(
176 name='pattern1',
177 value='/local/path/rti,/remote/path,png,300,11,0'
178 )
179 ftp.addParameter(
180 name='pattern2',
181 value='/local/path/spc,/remote/path,png,300,11,0'
182 )
183 ftp.addParameter(
184 name='pattern3',
185 value='/local/path/param,/remote/path,hdf5,300,,'
186 )
187
188 """
189
159 __attrs__ = ['server', 'username', 'password', 'timeout', 'patternX']
190 __attrs__ = ['server', 'username', 'password', 'timeout', 'patternX']
160
191
161 def __init__(self):
192 def __init__(self):
@@ -179,7 +210,7 class SendToFTP(Operation):
179 for arg, value in kwargs.items():
210 for arg, value in kwargs.items():
180 if 'pattern' in arg:
211 if 'pattern' in arg:
181 self.patterns.append(value)
212 self.patterns.append(value)
182 self.times.append(time.time())
213 self.times.append(0)
183 self.latest.append('')
214 self.latest.append('')
184
215
185 def connect(self):
216 def connect(self):
@@ -224,7 +255,7 class SendToFTP(Operation):
224
255
225 def find_files(self, path, ext):
256 def find_files(self, path, ext):
226
257
227 files = glob.glob1(path, '*{}'.format(ext))
258 files = glob.glob1(path.strip(), '*{}'.format(ext.strip()))
228 files.sort()
259 files.sort()
229 if files:
260 if files:
230 return files[-1]
261 return files[-1]
@@ -256,18 +287,12 class SendToFTP(Operation):
256 self.ftp.storbinary(command, fp, blocksize=1024)
287 self.ftp.storbinary(command, fp, blocksize=1024)
257 except Exception as e:
288 except Exception as e:
258 log.error('{}'.format(e), self.name)
289 log.error('{}'.format(e), self.name)
259 if self.ftp is not None:
260 self.ftp.close()
261 self.ftp = None
262 return 0
290 return 0
263
291
264 try:
292 try:
265 self.ftp.sendcmd('SITE CHMOD 755 {}'.format(dst))
293 self.ftp.sendcmd('SITE CHMOD 755 {}'.format(dst))
266 except Exception as e:
294 except Exception as e:
267 log.error('{}'.format(e), self.name)
295 log.error('{}'.format(e), self.name)
268 if self.ftp is not None:
269 self.ftp.close()
270 self.ftp = None
271 return 0
296 return 0
272
297
273 fp.close()
298 fp.close()
@@ -278,30 +303,30 class SendToFTP(Operation):
278
303
279 for x, pattern in enumerate(self.patterns):
304 for x, pattern in enumerate(self.patterns):
280 local, remote, ext, period, exp_code, sub_exp_code = pattern
305 local, remote, ext, period, exp_code, sub_exp_code = pattern
281 if time.time()-self.times[x] >= int(period):
306
282 srcname = self.find_files(local, ext)
307 if (self.dataOut.utctime - self.times[x]) < int(period):
283 src = os.path.join(local, srcname)
308 continue
284 if os.path.getmtime(src) < time.time() - 30*60:
309
285 log.warning('Skipping old file {}'.format(srcname))
310 srcname = self.find_files(local, ext)
286 continue
311
287
312 if srcname is None:
288 if srcname is None or srcname == self.latest[x]:
313 continue
289 log.warning('File alreday uploaded {}'.format(srcname))
314
290 continue
315 if srcname == self.latest[x]:
291
316 log.warning('File alreday uploaded {}'.format(srcname))
292 if 'png' in ext:
317 continue
293 dstname = self.getftpname(srcname, int(exp_code), int(sub_exp_code))
318
294 else:
319 if exp_code.strip():
295 dstname = srcname
320 dstname = self.getftpname(srcname, int(exp_code), int(sub_exp_code))
296
321 else:
297 dst = os.path.join(remote, dstname)
322 dstname = srcname
298
323
299 if self.upload(src, dst):
324 src = os.path.join(local, srcname)
300 self.times[x] = time.time()
325 dst = os.path.join(remote.strip(), dstname)
301 self.latest[x] = srcname
326
302 else:
327 if self.upload(src, dst):
303 self.ready = False
328 self.times[x] = self.dataOut.utctime
304 break
329 self.latest[x] = srcname
305
330
306 def run(self, dataOut, server, username, password, timeout=10, **kwargs):
331 def run(self, dataOut, server, username, password, timeout=10, **kwargs):
307
332
@@ -314,11 +339,11 class SendToFTP(Operation):
314 **kwargs
339 **kwargs
315 )
340 )
316 self.isConfig = True
341 self.isConfig = True
317 if not self.ready:
318 self.connect()
342 self.connect()
319 if self.ftp is not None:
343
320 self.check()
344 self.dataOut = dataOut
321 self.send_files()
345 self.check()
346 self.send_files()
322
347
323 def close(self):
348 def close(self):
324
349
General Comments 0
You need to be logged in to leave comments. Login now