From 96199ee477a0f93be1be2c6ef78a49d73d27792c 2020-09-06 05:16:34 From: Juan C. Espinoza Date: 2020-09-06 05:16:34 Subject: [PATCH] Fix xmin & xmax in plots, fix SendToFTP --- diff --git a/README.md b/README.md index a9de87c..fdc0d00 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ op = proc_unit.addOperation(name='removeDC') op = proc_unit.addOperation(name='SpectraPlot') op.addParameter(name='wintitle', value='Spectra', format='str') -op = procUnitConfObj1.addOperation(name='RTIPlot') +op = proc_unit.addOperation(name='RTIPlot') op.addParameter(name='wintitle', value='RTI', format='str') prj.start() diff --git a/schainpy/model/graphics/jroplot_base.py b/schainpy/model/graphics/jroplot_base.py index 2f271e2..3cdf0b0 100644 --- a/schainpy/model/graphics/jroplot_base.py +++ b/schainpy/model/graphics/jroplot_base.py @@ -1,3 +1,10 @@ +# Copyright (c) 2012-2020 Jicamarca Radio Observatory +# All rights reserved. +# +# Distributed under the terms of the BSD 3-clause license. +"""Base class to create plot operations + +""" import os import sys @@ -5,10 +12,7 @@ import zmq import time import numpy import datetime -try: - from queue import Queue -except: - from Queue import Queue +from multiprocessing import Queue from functools import wraps from threading import Thread import matplotlib @@ -145,9 +149,24 @@ def apply_throttle(value): @MPDecorator class Plot(Operation): - ''' - Base class for Schain plotting operations - ''' + """Base class for Schain plotting operations + + This class should never be use directtly you must subclass a new operation, + children classes must be defined as follow: + + ExamplePlot(Plot): + + CODE = 'code' + colormap = 'jet' + plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer') + + def setup(self): + pass + + def plot(self): + pass + + """ CODE = 'Figure' colormap = 'jet' @@ -163,7 +182,7 @@ class Plot(Operation): Operation.__init__(self) self.isConfig = False self.isPlotConfig = False - self.save_counter = 1 + self.save_time = 0 self.sender_time = 0 self.data = None self.firsttime = True @@ -187,7 +206,7 @@ class Plot(Operation): self.localtime = kwargs.pop('localtime', True) self.show = kwargs.get('show', True) self.save = kwargs.get('save', False) - self.save_period = kwargs.get('save_period', 1) + self.save_period = kwargs.get('save_period', 60) self.colormap = kwargs.get('colormap', self.colormap) self.colormap_coh = kwargs.get('colormap_coh', 'jet') self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r') @@ -224,7 +243,7 @@ class Plot(Operation): self.type = kwargs.get('type', 'iq') self.grid = kwargs.get('grid', False) self.pause = kwargs.get('pause', False) - self.save_code = kwargs.get('save_code', None) + self.save_code = kwargs.get('save_code', self.CODE) self.throttle = kwargs.get('throttle', 0) self.exp_code = kwargs.get('exp_code', None) self.plot_server = kwargs.get('plot_server', False) @@ -242,8 +261,6 @@ class Plot(Operation): 'Sending to server: {}'.format(self.plot_server), self.name ) - if 'plot_name' in kwargs: - self.plot_name = kwargs['plot_name'] def __setup_plot(self): ''' @@ -357,86 +374,30 @@ class Plot(Operation): ''' Set min and max values, labels, ticks and titles ''' - - if self.xmin is None: - xmin = self.data.min_time - else: - if self.xaxis is 'time': - dt = self.getDateTime(self.data.min_time) - xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) - - datetime.datetime(1970, 1, 1)).total_seconds() - if self.data.localtime: - xmin += time.timezone - else: - xmin = self.xmin - - if self.xmax is None: - xmax = xmin + self.xrange * 60 * 60 - else: - if self.xaxis is 'time': - dt = self.getDateTime(self.data.max_time) - xmax = self.xmax - 1 - xmax = (dt.replace(hour=int(xmax), minute=59, second=59) - - datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds() - if self.data.localtime: - xmax += time.timezone - else: - xmax = self.xmax - - ymin = self.ymin if self.ymin else numpy.nanmin(self.y) - ymax = self.ymax if self.ymax else numpy.nanmax(self.y) for n, ax in enumerate(self.axes): if ax.firsttime: - - dig = int(numpy.log10(ymax)) - if dig == 0: - digD = len(str(ymax)) - 2 - ydec = ymax*(10**digD) - - dig = int(numpy.log10(ydec)) - ystep = ((ydec + (10**(dig)))//10**(dig))*(10**(dig)) - ystep = ystep/5 - ystep = ystep/(10**digD) - - else: - ystep = ((ymax + (10**(dig)))//10**(dig))*(10**(dig)) - ystep = ystep/5 - - if self.xaxis is not 'time': - - dig = int(numpy.log10(xmax)) - - if dig <= 0: - digD = len(str(xmax)) - 2 - xdec = xmax*(10**digD) - - dig = int(numpy.log10(xdec)) - xstep = ((xdec + (10**(dig)))//10**(dig))*(10**(dig)) - xstep = xstep*0.5 - xstep = xstep/(10**digD) - - else: - xstep = ((xmax + (10**(dig)))//10**(dig))*(10**(dig)) - xstep = xstep/5 - + if self.xaxis != 'time': + xmin = self.xmin + xmax = self.xmax + else: + xmin = self.tmin + xmax = self.tmin + self.xrange*60*60 + ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime)) + ax.xaxis.set_major_locator(LinearLocator(9)) + ymin = self.ymin if self.ymin else numpy.nanmin(self.y) + ymax = self.ymax if self.ymax else numpy.nanmax(self.y) ax.set_facecolor(self.bgcolor) - ax.yaxis.set_major_locator(MultipleLocator(ystep)) if self.xscale: ax.xaxis.set_major_formatter(FuncFormatter( lambda x, pos: '{0:g}'.format(x*self.xscale))) - if self.xscale: + if self.yscale: ax.yaxis.set_major_formatter(FuncFormatter( lambda x, pos: '{0:g}'.format(x*self.yscale))) - if self.xaxis is 'time': - ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime)) - ax.xaxis.set_major_locator(LinearLocator(9)) - else: - ax.xaxis.set_major_locator(MultipleLocator(xstep)) if self.xlabel is not None: ax.set_xlabel(self.xlabel) - ax.set_ylabel(self.ylabel) - ax.firsttime = False + if self.ylabel is not None: + ax.set_ylabel(self.ylabel) if self.showprofile: self.pf_axes[n].set_ylim(ymin, ymax) self.pf_axes[n].set_xlim(self.zmin, self.zmax) @@ -455,12 +416,12 @@ class Plot(Operation): ax.cbar.set_label(self.cb_labels[n], size=8) else: ax.cbar = None - if self.grid: - ax.grid(True) - - if not self.polar: ax.set_xlim(xmin, xmax) ax.set_ylim(ymin, ymax) + ax.firsttime = False + if self.grid: + ax.grid(True) + if not self.polar: ax.set_title('{} {} {}'.format( self.titles[n], self.getDateTime(self.data.max_time).strftime( @@ -521,27 +482,18 @@ class Plot(Operation): ''' ''' - if self.save_counter < self.save_period: - self.save_counter += 1 + if (self.data.tm - self.save_time) < self.sender_period: return - self.save_counter = 1 + self.save_time = self.data.tm fig = self.figures[n] - if self.save_code: - if isinstance(self.save_code, str): - labels = [self.save_code for x in self.figures] - else: - labels = self.save_code - else: - labels = [self.CODE for x in self.figures] - figname = os.path.join( self.save, - labels[n], + self.save_code, '{}_{}.png'.format( - labels[n], + self.save_code, self.getDateTime(self.data.max_time).strftime( '%Y%m%d_%H%M%S' ), @@ -556,7 +508,7 @@ class Plot(Operation): figname = os.path.join( self.save, '{}_{}.png'.format( - labels[n], + self.save_code, self.getDateTime(self.data.min_time).strftime( '%Y%m%d' ), @@ -588,7 +540,7 @@ class Plot(Operation): else: self.data.meta['colormap'] = 'Viridis' self.data.meta['interval'] = int(interval) - # msg = self.data.jsonify(self.data.tm, self.plot_name, self.plot_type) + try: self.sender_queue.put(self.data.tm, block=False) except: @@ -600,7 +552,7 @@ class Plot(Operation): break tm = self.sender_queue.get() try: - msg = self.data.jsonify(tm, self.plot_name, self.plot_type) + msg = self.data.jsonify(tm, self.save_code, self.plot_type) except: continue self.socket.send_string(msg) @@ -654,24 +606,11 @@ class Plot(Operation): if self.isConfig is False: self.__setup(**kwargs) - - t = getattr(dataOut, self.attr_time) if self.localtime: self.getDateTime = datetime.datetime.fromtimestamp else: self.getDateTime = datetime.datetime.utcfromtimestamp - - if self.xmin is None: - self.tmin = t - if 'buffer' in self.plot_type: - self.xmin = self.getDateTime(t).hour - else: - self.tmin = ( - self.getDateTime(t).replace( - hour=int(self.xmin), - minute=0, - second=0) - self.getDateTime(0)).total_seconds() self.data.setup() self.isConfig = True @@ -684,13 +623,9 @@ class Plot(Operation): tm = getattr(dataOut, self.attr_time) - if self.data and (tm - self.tmin) >= self.xrange*60*60: + if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60: self.save_counter = self.save_period self.__plot() - if 'time' in self.xaxis: - self.xmin += self.xrange - if self.xmin >= 24: - self.xmin -= 24 self.tmin += self.xrange*60*60 self.data.setup() self.clear_figures() @@ -700,6 +635,20 @@ class Plot(Operation): if self.isPlotConfig is False: self.__setup_plot() self.isPlotConfig = True + if self.xaxis == 'time': + dt = self.getDateTime(tm) + if self.xmin is None: + self.tmin = tm + self.xmin = dt.hour + minutes = (self.xmin-int(self.xmin)) * 60 + seconds = (minutes - int(minutes)) * 60 + self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) - + datetime.datetime(1970, 1, 1)).total_seconds() + if self.localtime: + self.tmin += time.timezone + + if self.xmin is not None and self.xmax is not None: + self.xrange = self.xmax - self.xmin if self.throttle == 0: self.__plot() diff --git a/schainpy/model/graphics/jroplot_parameters.py b/schainpy/model/graphics/jroplot_parameters.py index ed29a71..f2311aa 100644 --- a/schainpy/model/graphics/jroplot_parameters.py +++ b/schainpy/model/graphics/jroplot_parameters.py @@ -36,7 +36,6 @@ class SpectralMomentsPlot(SpectraPlot): ''' CODE = 'spc_moments' colormap = 'jet' - plot_name = 'SpectralMoments' plot_type = 'pcolor' @@ -47,7 +46,6 @@ class SnrPlot(RTIPlot): CODE = 'snr' colormap = 'jet' - plot_name = 'SNR' class DopplerPlot(RTIPlot): @@ -57,7 +55,6 @@ class DopplerPlot(RTIPlot): CODE = 'dop' colormap = 'jet' - plot_name = 'DopplerShift' class PowerPlot(RTIPlot): @@ -67,7 +64,6 @@ class PowerPlot(RTIPlot): CODE = 'pow' colormap = 'jet' - plot_name = 'TotalPower' class SpectralWidthPlot(RTIPlot): @@ -77,7 +73,6 @@ class SpectralWidthPlot(RTIPlot): CODE = 'width' colormap = 'jet' - plot_name = 'SpectralWidth' class SkyMapPlot(Plot): @@ -135,7 +130,6 @@ class ParametersPlot(RTIPlot): CODE = 'param' colormap = 'seismic' - plot_name = 'Parameters' def setup(self): self.xaxis = 'time' @@ -210,7 +204,6 @@ class OutputPlot(ParametersPlot): CODE = 'output' colormap = 'seismic' - plot_name = 'Output' class PolarMapPlot(Plot): diff --git a/schainpy/model/graphics/jroplot_spectra.py b/schainpy/model/graphics/jroplot_spectra.py index 3ef5c6e..f120647 100644 --- a/schainpy/model/graphics/jroplot_spectra.py +++ b/schainpy/model/graphics/jroplot_spectra.py @@ -19,14 +19,13 @@ class SpectraPlot(Plot): CODE = 'spc' colormap = 'jet' - plot_name = 'Spectra' plot_type = 'pcolor' def setup(self): self.nplots = len(self.data.channels) self.ncols = int(numpy.sqrt(self.nplots) + 0.9) self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9) - self.height = 3 * self.nrows + self.height = 2.6 * self.nrows self.cb_label = 'dB' if self.showprofile: self.width = 4 * self.ncols @@ -92,7 +91,6 @@ class CrossSpectraPlot(Plot): CODE = 'cspc' colormap = 'jet' - plot_name = 'CrossSpectra' plot_type = 'pcolor' zmin_coh = None zmax_coh = None @@ -104,11 +102,11 @@ class CrossSpectraPlot(Plot): self.ncols = 4 self.nrows = len(self.data.pairs) self.nplots = self.nrows * 4 - self.width = 3.4 * self.ncols - self.height = 3 * self.nrows + self.width = 3.1 * self.ncols + self.height = 2.6 * self.nrows self.ylabel = 'Range [km]' self.showprofile = False - self.plots_adjust.update({'bottom': 0.08}) + self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08}) def plot(self): @@ -194,7 +192,6 @@ class RTIPlot(Plot): CODE = 'rti' colormap = 'jet' - plot_name = 'RTI' plot_type = 'pcolorbuffer' def setup(self): @@ -253,7 +250,6 @@ class CoherencePlot(RTIPlot): ''' CODE = 'coh' - plot_name = 'Coherence' def setup(self): self.xaxis = 'time' @@ -280,7 +276,6 @@ class PhasePlot(CoherencePlot): CODE = 'phase' colormap = 'seismic' - plot_name = 'Phase' class NoisePlot(Plot): @@ -289,7 +284,6 @@ class NoisePlot(Plot): ''' CODE = 'noise' - plot_name = 'Noise' plot_type = 'scatterbuffer' @@ -327,7 +321,6 @@ class NoisePlot(Plot): class PowerProfilePlot(Plot): CODE = 'spcprofile' - plot_name = 'Power Profile' plot_type = 'scatter' buffering = False @@ -365,7 +358,6 @@ class PowerProfilePlot(Plot): class SpectraCutPlot(Plot): CODE = 'spc_cut' - plot_name = 'Spectra Cut' plot_type = 'scatter' buffering = False diff --git a/schainpy/model/graphics/jroplot_voltage.py b/schainpy/model/graphics/jroplot_voltage.py index 7b98340..76287ce 100644 --- a/schainpy/model/graphics/jroplot_voltage.py +++ b/schainpy/model/graphics/jroplot_voltage.py @@ -17,7 +17,6 @@ class ScopePlot(Plot): ''' CODE = 'scope' - plot_name = 'Scope' plot_type = 'scatter' def setup(self): @@ -269,7 +268,6 @@ class PulsepairPowerPlot(ScopePlot): ''' CODE = 'pp_power' - plot_name = 'PulsepairPower' plot_type = 'scatter' buffering = False @@ -278,7 +276,6 @@ class PulsepairVelocityPlot(ScopePlot): Plot for VELOCITY ''' CODE = 'pp_velocity' - plot_name = 'PulsepairVelocity' plot_type = 'scatter' buffering = False @@ -287,7 +284,6 @@ class PulsepairSpecwidthPlot(ScopePlot): Plot for WIDTH ''' CODE = 'pp_specwidth' - plot_name = 'PulsepairSpecwidth' plot_type = 'scatter' buffering = False @@ -297,6 +293,5 @@ class PulsepairSignalPlot(ScopePlot): ''' CODE = 'pp_signal' - plot_name = 'PulsepairSignal' plot_type = 'scatter' buffering = False diff --git a/schainpy/model/utils/jroutils_publish.py b/schainpy/model/utils/jroutils_publish.py index d631130..808f2b7 100644 --- a/schainpy/model/utils/jroutils_publish.py +++ b/schainpy/model/utils/jroutils_publish.py @@ -1,6 +1,9 @@ -''' -@author: Juan C. Espinoza -''' +# Copyright (c) 2012-2020 Jicamarca Radio Observatory +# All rights reserved. +# +# Distributed under the terms of the BSD 3-clause license. +"""Utilities for publish/send data, files & plots over different protocols +""" import os import glob @@ -18,8 +21,6 @@ from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit, MPDecora from schainpy.model.data.jrodata import JROData from schainpy.utils import log -MAXNUMX = 500 -MAXNUMY = 500 PLOT_CODES = { 'rti': 0, # Range time intensity (RTI). @@ -52,11 +53,6 @@ def get_plot_code(s): else: return 24 -def decimate(z, MAXNUMY): - dy = int(len(z[0])/MAXNUMY) + 1 - - return z[::, ::dy] - class PublishData(Operation): ''' @@ -150,12 +146,47 @@ class ReceiverData(ProcessingUnit): @MPDecorator class SendToFTP(Operation): - - ''' - Operation to send data over FTP. - patternX = 'local, remote, ext, period, exp_code, sub_exp_code' - ''' - + """Operation for send files over FTP + + This operation is used to send files over FTP, you can send different files + from different folders by adding as many `pattern` as you wish. + + Parameters: + ----------- + server : str + FTP server address. + username : str + FTP username + password : str + FTP password + timeout : int + timeout to restart the connection + patternX : list + detail of files to be send must have the following order: local, remote + ext, period, exp_code, sub_exp_code + + Example: + -------- + + ftp = proc_unit.addOperation(name='SendToFTP', optype='external') + ftp.addParameter(name='server', value='jro-app.igp.gob.pe') + ftp.addParameter(name='username', value='wmaster') + ftp.addParameter(name='password', value='mst2010vhf') + ftp.addParameter( + name='pattern1', + value='/local/path/rti,/remote/path,png,300,11,0' + ) + ftp.addParameter( + name='pattern2', + value='/local/path/spc,/remote/path,png,300,11,0' + ) + ftp.addParameter( + name='pattern3', + value='/local/path/param,/remote/path,hdf5,300,,' + ) + + """ + __attrs__ = ['server', 'username', 'password', 'timeout', 'patternX'] def __init__(self): @@ -179,7 +210,7 @@ class SendToFTP(Operation): for arg, value in kwargs.items(): if 'pattern' in arg: self.patterns.append(value) - self.times.append(time.time()) + self.times.append(0) self.latest.append('') def connect(self): @@ -224,7 +255,7 @@ class SendToFTP(Operation): def find_files(self, path, ext): - files = glob.glob1(path, '*{}'.format(ext)) + files = glob.glob1(path.strip(), '*{}'.format(ext.strip())) files.sort() if files: return files[-1] @@ -256,18 +287,12 @@ class SendToFTP(Operation): self.ftp.storbinary(command, fp, blocksize=1024) except Exception as e: log.error('{}'.format(e), self.name) - if self.ftp is not None: - self.ftp.close() - self.ftp = None return 0 try: self.ftp.sendcmd('SITE CHMOD 755 {}'.format(dst)) except Exception as e: log.error('{}'.format(e), self.name) - if self.ftp is not None: - self.ftp.close() - self.ftp = None return 0 fp.close() @@ -278,30 +303,30 @@ class SendToFTP(Operation): for x, pattern in enumerate(self.patterns): local, remote, ext, period, exp_code, sub_exp_code = pattern - if time.time()-self.times[x] >= int(period): - srcname = self.find_files(local, ext) - src = os.path.join(local, srcname) - if os.path.getmtime(src) < time.time() - 30*60: - log.warning('Skipping old file {}'.format(srcname)) - continue - - if srcname is None or srcname == self.latest[x]: - log.warning('File alreday uploaded {}'.format(srcname)) - continue - - if 'png' in ext: - dstname = self.getftpname(srcname, int(exp_code), int(sub_exp_code)) - else: - dstname = srcname - - dst = os.path.join(remote, dstname) - - if self.upload(src, dst): - self.times[x] = time.time() - self.latest[x] = srcname - else: - self.ready = False - break + + if (self.dataOut.utctime - self.times[x]) < int(period): + continue + + srcname = self.find_files(local, ext) + + if srcname is None: + continue + + if srcname == self.latest[x]: + log.warning('File alreday uploaded {}'.format(srcname)) + continue + + if exp_code.strip(): + dstname = self.getftpname(srcname, int(exp_code), int(sub_exp_code)) + else: + dstname = srcname + + src = os.path.join(local, srcname) + dst = os.path.join(remote.strip(), dstname) + + if self.upload(src, dst): + self.times[x] = self.dataOut.utctime + self.latest[x] = srcname def run(self, dataOut, server, username, password, timeout=10, **kwargs): @@ -314,11 +339,11 @@ class SendToFTP(Operation): **kwargs ) self.isConfig = True - if not self.ready: self.connect() - if self.ftp is not None: - self.check() - self.send_files() + + self.dataOut = dataOut + self.check() + self.send_files() def close(self):