schain-jc Commit - r1138:c6cf5a83f48e

Fix px1000 reading online

jespinoza -

r1138:c6cf5a83f48e

parent child

Context file:

r1138:c6cf5a83f48e

Collapse all files

schainpy/model/graphics/jroplot_data.py +1 -1

             import os
             import time
             import glob
             import datetime
             from multiprocessing import Process
             import zmq
             import numpy
             import matplotlib
             import matplotlib.pyplot as plt
             from mpl_toolkits.axes_grid1 import make_axes_locatable
             from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
             from schainpy.model.proc.jroproc_base import Operation
             from schainpy.utils import log
             jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
             blu_values = matplotlib.pyplot.get_cmap(
                 'seismic_r', 20)(numpy.arange(20))[10:15]
             ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
                 'jro', numpy.vstack((blu_values, jet_values)))
             matplotlib.pyplot.register_cmap(cmap=ncmap)
             CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm', 'spectral')]
             def figpause(interval):
                 backend = plt.rcParams['backend']
                 if backend in matplotlib.rcsetup.interactive_bk:
                     figManager = matplotlib._pylab_helpers.Gcf.get_active()
                     if figManager is not None:
                         canvas = figManager.canvas
                         if canvas.figure.stale:
                             canvas.draw()
                         canvas.start_event_loop(interval)
                         return
             class PlotData(Operation, Process):
                 '''
                 Base class for Schain plotting operations
                 '''
                 CODE = 'Figure'
                 colormap = 'jro'
                 bgcolor = 'white'
                 CONFLATE = False
                 __missing = 1E30
                 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
                              'zlimits', 'xlabel', 'ylabel', 'xaxis','cb_label', 'title',
                              'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
                              'showprofile', 'decimation', 'ftp']
                 def __init__(self, **kwargs):
                     Operation.__init__(self, plot=True, **kwargs)
                     Process.__init__(self)
                     self.kwargs['code'] = self.CODE
                     self.mp = False
                     self.data = None
                     self.isConfig = False
                     self.figures = []
                     self.axes = []
                     self.cb_axes = []
                     self.localtime = kwargs.pop('localtime', True)
                     self.show = kwargs.get('show', True)
                     self.save = kwargs.get('save', False)
                     self.ftp = kwargs.get('ftp', False)
                     self.colormap = kwargs.get('colormap', self.colormap)
                     self.colormap_coh = kwargs.get('colormap_coh', 'jet')
                     self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
                     self.colormaps = kwargs.get('colormaps', None)
                     self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
                     self.showprofile = kwargs.get('showprofile', False)
                     self.title = kwargs.get('wintitle', self.CODE.upper())
                     self.cb_label = kwargs.get('cb_label', None)
                     self.cb_labels = kwargs.get('cb_labels', None)
                     self.labels = kwargs.get('labels', None)
                     self.xaxis = kwargs.get('xaxis', 'frequency')
                     self.zmin = kwargs.get('zmin', None)
                     self.zmax = kwargs.get('zmax', None)
                     self.zlimits = kwargs.get('zlimits', None)
                     self.xmin = kwargs.get('xmin', None)
                     self.xmax = kwargs.get('xmax', None)
                     self.xrange = kwargs.get('xrange', 24)
                     self.xscale = kwargs.get('xscale', None)
                     self.ymin = kwargs.get('ymin', None)
                     self.ymax = kwargs.get('ymax', None)
                     self.yscale = kwargs.get('yscale', None)
                     self.xlabel = kwargs.get('xlabel', None)
                     self.decimation = kwargs.get('decimation', None)
                     self.showSNR = kwargs.get('showSNR', False)
                     self.oneFigure = kwargs.get('oneFigure', True)
                     self.width = kwargs.get('width', None)
                     self.height = kwargs.get('height', None)
                     self.colorbar = kwargs.get('colorbar', True)
                     self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
                     self.channels = kwargs.get('channels', None)
                     self.titles = kwargs.get('titles', [])
                     self.polar = False
                 def __fmtTime(self, x, pos):
                     '''
                     '''
                     return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
                 def __setup(self):
                     '''
                     Common setup for all figures, here figures and axes are created
                     '''
                     if self.CODE not in self.data:
                         raise ValueError(log.error('Missing data for {}'.format(self.CODE),
                                                    self.name))
                     self.setup()
                     self.time_label = 'LT' if self.localtime else 'UTC'
                     if self.data.localtime:
                         self.getDateTime = datetime.datetime.fromtimestamp
                     else:
                         self.getDateTime = datetime.datetime.utcfromtimestamp
                     if self.width is None:
                         self.width = 8
                     self.figures = []
                     self.axes = []
                     self.cb_axes = []
                     self.pf_axes = []
                     self.cmaps = []
                     size = '15%' if self.ncols == 1 else '30%'
                     pad = '4%' if self.ncols == 1 else '8%'
                     if self.oneFigure:
                         if self.height is None:
                             self.height = 1.4 * self.nrows + 1
                         fig = plt.figure(figsize=(self.width, self.height),
                                          edgecolor='k',
                                          facecolor='w')
                         self.figures.append(fig)
                         for n in range(self.nplots):
                             ax = fig.add_subplot(self.nrows, self.ncols,
                                                  n + 1, polar=self.polar)
                             ax.tick_params(labelsize=8)
                             ax.firsttime = True
                             ax.index = 0
                             ax.press = None
                             self.axes.append(ax)
                             if self.showprofile:
                                 cax = self.__add_axes(ax, size=size, pad=pad)
                                 cax.tick_params(labelsize=8)
                                 self.pf_axes.append(cax)
                     else:
                         if self.height is None:
                             self.height = 3
                         for n in range(self.nplots):
                             fig = plt.figure(figsize=(self.width, self.height),
                                              edgecolor='k',
                                              facecolor='w')
                             ax = fig.add_subplot(1, 1, 1, polar=self.polar)
                             ax.tick_params(labelsize=8)
                             ax.firsttime = True
                             ax.index = 0
                             ax.press = None
                             self.figures.append(fig)
                             self.axes.append(ax)
                             if self.showprofile:
                                 cax = self.__add_axes(ax, size=size, pad=pad)
                                 cax.tick_params(labelsize=8)
                                 self.pf_axes.append(cax)
                     for n in range(self.nrows):
                         if self.colormaps is not None:
                             cmap = plt.get_cmap(self.colormaps[n])
                         else:
                             cmap = plt.get_cmap(self.colormap)
                         cmap.set_bad(self.bgcolor, 1.)
                         self.cmaps.append(cmap)
                     for fig in self.figures:
                         fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
                         fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
                         fig.canvas.mpl_connect('button_press_event', self.onBtnPress)
                         fig.canvas.mpl_connect('motion_notify_event', self.onMotion)
                         fig.canvas.mpl_connect('button_release_event', self.onBtnRelease)
                         if self.show:
                             fig.show()
                 def OnKeyPress(self, event):
                     '''
                     Event for pressing keys (up, down) change colormap
                     '''
                     ax = event.inaxes
                     if ax in self.axes:
                         if event.key == 'down':
                             ax.index += 1
                         elif event.key == 'up':
                             ax.index -= 1
                         if ax.index < 0:
                             ax.index = len(CMAPS) - 1
                         elif ax.index == len(CMAPS):
                             ax.index = 0
                         cmap = CMAPS[ax.index]
                         ax.cbar.set_cmap(cmap)
                         ax.cbar.draw_all()
                         ax.plt.set_cmap(cmap)
                         ax.cbar.patch.figure.canvas.draw()
                         self.colormap = cmap.name
                 def OnBtnScroll(self, event):
                     '''
                     Event for scrolling, scale figure
                     '''
                     cb_ax = event.inaxes
                     if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
                         ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
                         pt = ax.cbar.ax.bbox.get_points()[:, 1]
                         nrm = ax.cbar.norm
                         vmin, vmax, p0, p1, pS = (
                             nrm.vmin, nrm.vmax, pt[0], pt[1], event.y)
                         scale = 2 if event.step == 1 else 0.5
                         point = vmin + (vmax - vmin) / (p1 - p0) * (pS - p0)
                         ax.cbar.norm.vmin = point - scale * (point - vmin)
                         ax.cbar.norm.vmax = point - scale * (point - vmax)
                         ax.plt.set_norm(ax.cbar.norm)
                         ax.cbar.draw_all()
                         ax.cbar.patch.figure.canvas.draw()
                 def onBtnPress(self, event):
                     '''
                     Event for mouse button press
                     '''
                     cb_ax = event.inaxes
                     if cb_ax is None:
                         return
                     if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
                         cb_ax.press = event.x, event.y
                     else:
                         cb_ax.press = None
                 def onMotion(self, event):
                     '''
                     Event for move inside colorbar
                     '''
                     cb_ax = event.inaxes
                     if cb_ax is None:
                         return
                     if cb_ax not in [ax.cbar.ax for ax in self.axes if ax.cbar]:
                         return
                     if cb_ax.press is None:
                         return
                     ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
                     xprev, yprev = cb_ax.press
                     dx = event.x - xprev
                     dy = event.y - yprev
                     cb_ax.press = event.x, event.y
                     scale = ax.cbar.norm.vmax - ax.cbar.norm.vmin
                     perc = 0.03
                     if event.button == 1:
                         ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
                         ax.cbar.norm.vmax -= (perc * scale) * numpy.sign(dy)
                     elif event.button == 3:
                         ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
                         ax.cbar.norm.vmax += (perc * scale) * numpy.sign(dy)
                     ax.cbar.draw_all()
                     ax.plt.set_norm(ax.cbar.norm)
                     ax.cbar.patch.figure.canvas.draw()
                 def onBtnRelease(self, event):
                     '''
                     Event for mouse button release
                     '''
                     cb_ax = event.inaxes
                     if cb_ax is not None:
                         cb_ax.press = None
                 def __add_axes(self, ax, size='30%', pad='8%'):
                     '''
                     Add new axes to the given figure
                     '''
                     divider = make_axes_locatable(ax)
                     nax = divider.new_horizontal(size=size, pad=pad)
                     ax.figure.add_axes(nax)
                     return nax
                     self.setup()
                 def setup(self):
                     '''
                     This method should be implemented in the child class, the following
                     attributes should be set:
                     self.nrows: number of rows
                     self.ncols: number of cols
                     self.nplots: number of plots (channels or pairs)
                     self.ylabel: label for Y axes
                     self.titles: list of axes title
                     '''
                     raise(NotImplementedError, 'Implement this method in child class')
                 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
                     '''
                     Create a masked array for missing data
                     '''
                     if x_buffer.shape[0] < 2:
                         return x_buffer, y_buffer, z_buffer
                     deltas = x_buffer[1:] - x_buffer[0:-1]
                     x_median = numpy.median(deltas)
                     index = numpy.where(deltas > 5 * x_median)
                     if len(index[0]) != 0:
                         z_buffer[::, index[0], ::] = self.__missing
                         z_buffer = numpy.ma.masked_inside(z_buffer,
 .99 * self.__missing,
 .01 * self.__missing)
                     return x_buffer, y_buffer, z_buffer
                 def decimate(self):
                     # dx = int(len(self.x)/self.__MAXNUMX) + 1
                     dy = int(len(self.y) / self.decimation) + 1
                     # x = self.x[::dx]
                     x = self.x
                     y = self.y[::dy]
                     z = self.z[::, ::, ::dy]
                     return x, y, z
                 def format(self):
                     '''
                     Set min and max values, labels, ticks and titles
                     '''
                     if self.xmin is None:
                         xmin = self.min_time
                     else:
                         if self.xaxis is 'time':
                             dt = self.getDateTime(self.min_time)
                             xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
                                     datetime.datetime(1970, 1, 1)).total_seconds()
                             if self.data.localtime:
                                 xmin += time.timezone
                         else:
                             xmin = self.xmin
                     if self.xmax is None:
                         xmax = xmin + self.xrange * 60 * 60
                     else:
                         if self.xaxis is 'time':
                             dt = self.getDateTime(self.max_time)
                             xmax = (dt.replace(hour=int(self.xmax), minute=59, second=59) -
                                     datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds()
                             if self.data.localtime:
                                 xmax += time.timezone
                         else:
                             xmax = self.xmax
                     ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
                     ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
                     Y = numpy.array([5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000])
                     i = 1 if numpy.where(abs(ymax-ymin) <= Y)[0][0] < 0 else numpy.where(abs(ymax-ymin) <= Y)[0][0]
                     ystep = Y[i] / 5.
                     for n, ax in enumerate(self.axes):
                         if ax.firsttime:
                             ax.set_facecolor(self.bgcolor)
                             ax.yaxis.set_major_locator(MultipleLocator(ystep))
                             ax.xaxis.set_major_locator(MultipleLocator(ystep))
                             if self.xscale:
                                 ax.xaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.xscale)))
                             if self.xscale:
                                 ax.yaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.yscale)))
                             if self.xaxis is 'time':
                                 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
                                 ax.xaxis.set_major_locator(LinearLocator(9))
                             if self.xlabel is not None:
                                 ax.set_xlabel(self.xlabel)
                             ax.set_ylabel(self.ylabel)
                             ax.firsttime = False
                             if self.showprofile:
                                 self.pf_axes[n].set_ylim(ymin, ymax)
                                 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
                                 self.pf_axes[n].set_xlabel('dB')
                                 self.pf_axes[n].grid(b=True, axis='x')
                                 [tick.set_visible(False)
                                  for tick in self.pf_axes[n].get_yticklabels()]
                             if self.colorbar:
                                 ax.cbar = plt.colorbar(
                                     ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
                                 ax.cbar.ax.tick_params(labelsize=8)
                                 ax.cbar.ax.press = None
                                 if self.cb_label:
                                     ax.cbar.set_label(self.cb_label, size=8)
                                 elif self.cb_labels:
                                     ax.cbar.set_label(self.cb_labels[n], size=8)
                             else:
                                 ax.cbar = None
                         if not self.polar:
                             ax.set_xlim(xmin, xmax)
                             ax.set_ylim(ymin, ymax)
                             ax.set_title('{} - {} {}'.format(
                                 self.titles[n],
                                 self.getDateTime(self.max_time).strftime('%H:%M:%S'),
                                 self.time_label),
                                 size=8)
                         else:
                             ax.set_title('{}'.format(self.titles[n]), size=8)
                             ax.set_ylim(0, 90)
                             ax.set_yticks(numpy.arange(0, 90, 20))
                             ax.yaxis.labelpad = 40
                 def __plot(self):
                     '''
                     '''
                     log.log('Plotting', self.name)
                     try:
                         self.plot()
                         self.format()
                     except:
                         log.warning('{} Plot could not be updated... check data'.format(self.CODE), self.name)
                     for n, fig in enumerate(self.figures):
                         if self.nrows == 0 or self.nplots == 0:
                             log.warning('No data', self.name)
                             fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
                             fig.canvas.manager.set_window_title(self.CODE)
                             continue
                         fig.tight_layout()
                         fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
                                                                              self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
                         fig.canvas.draw()
                         if self.save and (self.data.ended or not self.data.buffering):
                             if self.labels:
                                 labels = self.labels
                             else:
                                 labels = range(self.nrows)
                             if self.oneFigure:
                                 label = ''
                             else:
                                 label = '-{}'.format(labels[n])
                             figname = os.path.join(
                                 self.save,
                                 self.CODE,
                                 '{}{}_{}.png'.format(
                                     self.CODE,
                                     label,
                                     self.getDateTime(self.saveTime).strftime(
                                         '%Y%m%d_%H%M%S'),
                                 )
                             )
                             log.log('Saving figure: {}'.format(figname), self.name)
                             if not os.path.isdir(os.path.dirname(figname)):
                                 os.makedirs(os.path.dirname(figname))
                             fig.savefig(figname)
                 def plot(self):
                     '''
                     '''
                     raise(NotImplementedError, 'Implement this method in child class')
                 def run(self):
                     log.log('Starting', self.name)
                     context = zmq.Context()
                     receiver = context.socket(zmq.SUB)
                     receiver.setsockopt(zmq.SUBSCRIBE, '')
                     receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
                     if 'server' in self.kwargs['parent']:
                         receiver.connect(
                             'ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
                     else:
                         receiver.connect("ipc:///tmp/zmq.plots")
                     while True:
                         try:
                             self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
                             if self.data.localtime and self.localtime:
                                 self.times = self.data.times
                             elif self.data.localtime and not self.localtime:
                                 self.times = self.data.times + time.timezone
                             elif not self.data.localtime and self.localtime:
                                 self.times = self.data.times - time.timezone
                             else:
                                 self.times = self.data.times
                             self.min_time = self.times[0]
                             self.max_time = self.times[-1]
                             if self.isConfig is False:
                                 self.__setup()
                                 self.isConfig = True
                             self.__plot()
                         except zmq.Again as e:
-                            log.log('.', tag='', nl=False)
+                            # log.log('.', tag='', nl=False)
                             if self.data:
                                 figpause(self.data.throttle)
                             else:
                                 time.sleep(2)
                 def close(self):
                     if self.data:
                         self.__plot()
             class PlotSpectraData(PlotData):
                 '''
                 Plot for Spectra data
                 '''
                 CODE = 'spc'
                 colormap = 'jro'
                 def setup(self):
                     self.nplots = len(self.data.channels)
                     self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
                     self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
                     self.width = 3.4 * self.ncols
                     self.height = 3 * self.nrows
                     self.cb_label = 'dB'
                     if self.showprofile:
                         self.width += 0.8 * self.ncols
                     self.ylabel = 'Range [km]'
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.data.xrange[0]
                         self.xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.data.xrange[1]
                         self.xlabel = "Time (ms)"
                     else:
                         x = self.data.xrange[2]
                         self.xlabel = "Velocity (m/s)"
                     if self.CODE == 'spc_mean':
                         x = self.data.xrange[2]
                         self.xlabel = "Velocity (m/s)"
                     self.titles = []
                     y = self.data.heights
                     self.y = y
                     z = self.data['spc']
                     for n, ax in enumerate(self.axes):
                         noise = self.data['noise'][n][-1]
                         if self.CODE == 'spc_mean':
                             mean = self.data['mean'][n][-1]
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
                             self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                             if self.showprofile:
                                 ax.plt_profile = self.pf_axes[n].plot(
                                     self.data['rti'][n][-1], y)[0]
                                 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
                                                                     color="k", linestyle="dashed", lw=1)[0]
                             if self.CODE == 'spc_mean':
                                 ax.plt_mean = ax.plot(mean, y, color='k')[0]
                         else:
                             ax.plt.set_array(z[n].T.ravel())
                             if self.showprofile:
                                 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
                                 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
                             if self.CODE == 'spc_mean':
                                 ax.plt_mean.set_data(mean, y)
                         self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
                         self.saveTime = self.max_time
             class PlotCrossSpectraData(PlotData):
                 CODE = 'cspc'
                 zmin_coh = None
                 zmax_coh = None
                 zmin_phase = None
                 zmax_phase = None
                 def setup(self):
                     self.ncols = 4
                     self.nrows = len(self.data.pairs)
                     self.nplots = self.nrows * 4
                     self.width = 3.4 * self.ncols
                     self.height = 3 * self.nrows
                     self.ylabel = 'Range [km]'
                     self.showprofile = False
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.data.xrange[0]
                         self.xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.data.xrange[1]
                         self.xlabel = "Time (ms)"
                     else:
                         x = self.data.xrange[2]
                         self.xlabel = "Velocity (m/s)"
                     self.titles = []
                     y = self.data.heights
                     self.y = y
                     spc = self.data['spc']
                     cspc = self.data['cspc']
                     for n in range(self.nrows):
                         noise = self.data['noise'][n][-1]
                         pair = self.data.pairs[n]
                         ax = self.axes[4 * n]
                         ax3 = self.axes[4 * n + 3]
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.zmin = self.zmin if self.zmin else numpy.nanmin(spc)
                             self.zmax = self.zmax if self.zmax else numpy.nanmax(spc)
                             ax.plt = ax.pcolormesh(x, y, spc[pair[0]].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                         else:
                             ax.plt.set_array(spc[pair[0]].T.ravel())
                         self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
                         ax = self.axes[4 * n + 1]
                         if ax.firsttime:
                             ax.plt = ax.pcolormesh(x, y, spc[pair[1]].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                         else:
                             ax.plt.set_array(spc[pair[1]].T.ravel())
                         self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
                         out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
                         coh = numpy.abs(out)
                         phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
                         ax = self.axes[4 * n + 2]
                         if ax.firsttime:
                             ax.plt = ax.pcolormesh(x, y, coh.T,
                                                    vmin=0,
                                                    vmax=1,
                                                    cmap=plt.get_cmap(self.colormap_coh)
                                                    )
                         else:
                             ax.plt.set_array(coh.T.ravel())
                         self.titles.append(
                             'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
                         ax = self.axes[4 * n + 3]
                         if ax.firsttime:
                             ax.plt = ax.pcolormesh(x, y, phase.T,
                                                    vmin=-180,
                                                    vmax=180,
                                                    cmap=plt.get_cmap(self.colormap_phase)
                                                    )
                         else:
                             ax.plt.set_array(phase.T.ravel())
                         self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
                         self.saveTime = self.max_time
             class PlotSpectraMeanData(PlotSpectraData):
                 '''
                 Plot for Spectra and Mean
                 '''
                 CODE = 'spc_mean'
                 colormap = 'jro'
             class PlotRTIData(PlotData):
                 '''
                 Plot for RTI data
                 '''
                 CODE = 'rti'
                 colormap = 'jro'
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     self.nrows = len(self.data.channels)
                     self.nplots = len(self.data.channels)
                     self.ylabel = 'Range [km]'
                     self.cb_label = 'dB'
                     self.titles = ['{} Channel {}'.format(
                         self.CODE.upper(), x) for x in range(self.nrows)]
                 def plot(self):
                     self.x = self.times
                     self.y = self.data.heights
                     self.z = self.data[self.CODE]
                     self.z = numpy.ma.masked_invalid(self.z)
                     if self.decimation is None:
                         x, y, z = self.fill_gaps(self.x, self.y, self.z)
                     else:
                         x, y, z = self.fill_gaps(*self.decimate())
                     for n, ax in enumerate(self.axes):
                         self.zmin = self.zmin if self.zmin else numpy.min(self.z)
                         self.zmax = self.zmax if self.zmax else numpy.max(self.z)
                         if ax.firsttime:
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                             if self.showprofile:
                                 ax.plot_profile = self.pf_axes[n].plot(
                                     self.data['rti'][n][-1], self.y)[0]
                                 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
                                                                      color="k", linestyle="dashed", lw=1)[0]
                         else:
                             ax.collections.remove(ax.collections[0])
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                             if self.showprofile:
                                 ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
                                 ax.plot_noise.set_data(numpy.repeat(
                                     self.data['noise'][n][-1], len(self.y)), self.y)
                     self.saveTime = self.min_time
             class PlotCOHData(PlotRTIData):
                 '''
                 Plot for Coherence data
                 '''
                 CODE = 'coh'
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     self.nrows = len(self.data.pairs)
                     self.nplots = len(self.data.pairs)
                     self.ylabel = 'Range [km]'
                     if self.CODE == 'coh':
                         self.cb_label = ''
                         self.titles = [
                             'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
                     else:
                         self.cb_label = 'Degrees'
                         self.titles = [
                             'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
             class PlotPHASEData(PlotCOHData):
                 '''
                 Plot for Phase map data
                 '''
                 CODE = 'phase'
                 colormap = 'seismic'
             class PlotNoiseData(PlotData):
                 '''
                 Plot for noise
                 '''
                 CODE = 'noise'
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     self.nrows = 1
                     self.nplots = 1
                     self.ylabel = 'Intensity [dB]'
                     self.titles = ['Noise']
                     self.colorbar = False
                 def plot(self):
                     x = self.times
                     xmin = self.min_time
                     xmax = xmin + self.xrange * 60 * 60
                     Y = self.data[self.CODE]
                     if self.axes[0].firsttime:
                         for ch in self.data.channels:
                             y = Y[ch]
                             self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
                         plt.legend()
                     else:
                         for ch in self.data.channels:
                             y = Y[ch]
                             self.axes[0].lines[ch].set_data(x, y)
                     self.ymin = numpy.nanmin(Y) - 5
                     self.ymax = numpy.nanmax(Y) + 5
                     self.saveTime = self.min_time
             class PlotSNRData(PlotRTIData):
                 '''
                 Plot for SNR Data
                 '''
                 CODE = 'snr'
                 colormap = 'jet'
             class PlotDOPData(PlotRTIData):
                 '''
                 Plot for DOPPLER Data
                 '''
                 CODE = 'dop'
                 colormap = 'jet'
             class PlotSkyMapData(PlotData):
                 '''
                 Plot for meteors detection data
                 '''
                 CODE = 'param'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = 1
                     self.width = 7.2
                     self.height = 7.2
                     self.nplots = 1
                     self.xlabel = 'Zonal Zenith Angle (deg)'
                     self.ylabel = 'Meridional Zenith Angle (deg)'
                     self.polar = True
                     self.ymin = -180
                     self.ymax = 180
                     self.colorbar = False
                 def plot(self):
                     arrayParameters = numpy.concatenate(self.data['param'])
                     error = arrayParameters[:, -1]
                     indValid = numpy.where(error == 0)[0]
                     finalMeteor = arrayParameters[indValid, :]
                     finalAzimuth = finalMeteor[:, 3]
                     finalZenith = finalMeteor[:, 4]
                     x = finalAzimuth * numpy.pi / 180
                     y = finalZenith
                     ax = self.axes[0]
                     if ax.firsttime:
                         ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
                     else:
                         ax.plot.set_data(x, y)
                     dt1 = self.getDateTime(self.min_time).strftime('%y/%m/%d %H:%M:%S')
                     dt2 = self.getDateTime(self.max_time).strftime('%y/%m/%d %H:%M:%S')
                     title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
                                                                                                  dt2,
                                                                                                  len(x))
                     self.titles[0] = title
                     self.saveTime = self.max_time
             class PlotParamData(PlotRTIData):
                 '''
                 Plot for data_param object
                 '''
                 CODE = 'param'
                 colormap = 'seismic'
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     self.nrows = self.data.shape(self.CODE)[0]
                     self.nplots = self.nrows
                     if self.showSNR:
                         self.nrows += 1
                         self.nplots += 1
                     self.ylabel = 'Height [km]'
                     if not self.titles:
                         self.titles = self.data.parameters \
                             if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
                         if self.showSNR:
                             self.titles.append('SNR')
                 def plot(self):
                     self.data.normalize_heights()
                     self.x = self.times
                     self.y = self.data.heights
                     if self.showSNR:
                         self.z = numpy.concatenate(
                             (self.data[self.CODE], self.data['snr'])
                         )
                     else:
                         self.z = self.data[self.CODE]
                     self.z = numpy.ma.masked_invalid(self.z)
                     if self.decimation is None:
                         x, y, z = self.fill_gaps(self.x, self.y, self.z)
                     else:
                         x, y, z = self.fill_gaps(*self.decimate())
                     for n, ax in enumerate(self.axes):
                         self.zmax = self.zmax if self.zmax is not None else numpy.max(
                             self.z[n])
                         self.zmin = self.zmin if self.zmin is not None else numpy.min(
                             self.z[n])
                         if ax.firsttime:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=self.cmaps[n]
                                                    )
                         else:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.collections.remove(ax.collections[0])
                             ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=self.cmaps[n]
                                                    )
                     self.saveTime = self.min_time
             class PlotOutputData(PlotParamData):
                 '''
                 Plot data_output object
                 '''
                 CODE = 'output'
                 colormap = 'seismic'
             class PlotPolarMapData(PlotData):
                 '''
                 Plot for meteors detection data
                 '''
                 CODE = 'param'
                 colormap = 'seismic'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = 1
                     self.width = 9
                     self.height = 8
                     if self.channels is not None:
                         self.nplots = len(self.channels)
                         self.nrows = len(self.channels)
                     else:
                         self.nplots = self.data.shape(self.CODE)[0]
                         self.nrows = self.nplots
                         self.channels = range(self.nplots)
                     self.xlabel = 'Zonal Distance (km)'
                     self.ylabel = 'Meridional Distance (km)'
                     self.bgcolor = 'white'
                 def plot(self):
                     for n, ax in enumerate(self.axes):
                         data = self.data['param'][self.channels[n]]
                         zeniths = numpy.arange(data.shape[1])
                         azimuths = -numpy.radians(self.data.heights)+numpy.pi/2
                         self.y = zeniths
                         r, theta = numpy.meshgrid(zeniths, azimuths)
                         x, y = r*numpy.cos(theta), r*numpy.sin(theta)
                         if ax.firsttime:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.plt = ax.pcolormesh(x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
                                                     vmin=self.zmin,
                                                     vmax=self.zmax,
                                                     cmap=self.cmaps[n])
                         else:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.collections.remove(ax.collections[0])
                             ax.plt = ax.pcolormesh(x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
                                                     vmin=self.zmin,
                                                     vmax=self.zmax,
                                                     cmap=self.cmaps[n])
                         title = ''
                     self.titles = [self.data.parameters[x] for x in self.channels]
                     self.saveTime = self.max_time

schainpy/model/io/pxIO_param.py +46 -43

             '''
             Created on Dec 27, 2017
             @author: Juan C. Espinoza
             '''
             import os
             import sys
             import time
             import json
             import glob
             import datetime
             import tarfile
             import numpy
             from netCDF4 import Dataset
             from schainpy.model.io.jroIO_base import JRODataReader
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             from schainpy.model.data.jrodata import Parameters
             from schainpy.utils import log
             UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
             def load_json(obj):
                 '''
                 Parse json as string instead of unicode
                 '''
                 if isinstance(obj, str):
                     iterable = json.loads(obj)
                 else:
                     iterable = obj
                 if isinstance(iterable, dict):
                     return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
                         for k, v in iterable.items()}
                 elif isinstance(iterable, (list, tuple)):
                     return [str(v) if isinstance(v, unicode) else v for v in iterable]
                 return iterable
             class NCDFReader(JRODataReader, ProcessingUnit):
                 def __init__(self, **kwargs):
                     ProcessingUnit.__init__(self, **kwargs)
                     self.dataOut = Parameters()
                     self.counter_records = 0
                     self.nrecords = None
                     self.flagNoMoreFiles = 0
                     self.isConfig = False
                     self.filename = None
                     self.intervals = set()
                     self.ext = ('.nc', '.tgz')
                     self.online_mode = False
                 def setup(self,
                           path=None,
                           startDate=None,
                           endDate=None,
                           format=None,
                           startTime=datetime.time(0, 0, 0),
                           endTime=datetime.time(23, 59, 59),
                           walk=False,
                           **kwargs):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.datatime = datetime.datetime(1900,1,1)
                     self.walk = walk
                     self.nTries = kwargs.get('nTries', 3)
                     self.online = kwargs.get('online', False)
                     self.delay = kwargs.get('delay', 30)
+                    self.ele = kwargs.get('ext', '')
                     if self.path is None:
                         raise ValueError, 'The path is not valid'
                     self.search_files(path, startDate, endDate, startTime, endTime, walk)
                     self.cursor = 0
                     self.counter_records = 0
                     if not self.files:
                         raise  Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
                 def search_files(self, path, startDate, endDate, startTime, endTime, walk):
                     '''
                      Searching for NCDF files in path
                      Creating a list of files to procces included in [startDate,endDate]
                      Input:
                          path - Path to find files
                     '''
                     log.log('Searching files {} in {} '.format(self.ext, path), 'NCDFReader')
                     if walk:
                         paths = [os.path.join(path, p) for p in os.listdir(path) if os.path.isdir(os.path.join(path, p))]
                         paths.sort()
                     else:
                         paths = [path]
                     fileList0 = []
                     for subpath in paths:
-                        fileList0 += [os.path.join(subpath, s) for s in glob.glob1(subpath, '*') if os.path.splitext(s)[-1] in self.ext]
+                        fileList0 += [os.path.join(subpath, s) for s in glob.glob1(subpath, '*') if os.path.splitext(s)[-1] in self.ext and 'E{}'.format(self.ele) in s]
                     fileList0.sort()
+                    if self.online:
+                        fileList0 = fileList0[-1:]
                     self.files = {}
                     startDate = startDate - datetime.timedelta(1)
                     endDate = endDate + datetime.timedelta(1)
                     for fullname in fileList0:
                         thisFile = fullname.split('/')[-1]
                         year = thisFile[3:7]
                         if not year.isdigit():
                             continue
                         month = thisFile[7:9]
                         if not month.isdigit():
                             continue
                         day = thisFile[9:11]
                         if not day.isdigit():
                             continue
                         year, month, day = int(year), int(month), int(day)
                         dateFile = datetime.date(year, month, day)
                         timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
                         if (startDate > dateFile) or (endDate < dateFile):
                             continue
                         dt = datetime.datetime.combine(dateFile, timeFile)
                         if dt not in self.files:
                             self.files[dt] = []
                         self.files[dt].append(fullname)
                     self.dates = self.files.keys()
                     self.dates.sort()
                     return
                 def search_files_online(self):
                     '''
                      Searching for NCDF files in online mode path
                      Creating a list of files to procces included in [startDate,endDate]
                      Input:
                          path - Path to find files
                     '''
-                    old_files = self.files[self.dt]
                     self.files = {}
                     for n in range(self.nTries):
                         if self.walk:
                             paths = [os.path.join(self.path, p) for p in os.listdir(self.path) if os.path.isdir(os.path.join(self.path, p))]
-                            paths = paths[-2:]
+                            path = paths[-1]
                         else:
-                            paths = [self.path]
+                            paths = self.path
-                        new_files = []
+                        new_files = [os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and 'E{}'.format(self.ele) in s]
-                        for path in paths:
-                            new_files += [os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and os.path.join(path, s not in old_files)]
                         new_files.sort()
-                        if new_files:
+                        for fullname in new_files:
+                            thisFile = fullname.split('/')[-1]
+                            year = thisFile[3:7]
+                            if not year.isdigit():
+                                continue
+                            month = thisFile[7:9]
+                            if not month.isdigit():
+                                continue
+                            day = thisFile[9:11]
+                            if not day.isdigit():
+                                continue
+                            year, month, day = int(year), int(month), int(day)
+                            dateFile = datetime.date(year, month, day)
+                            timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
+                            dt = datetime.datetime.combine(dateFile, timeFile)
+                            if self.dt >= dt:
+                                continue
+                            if dt not in self.files:
+                                self.dt = dt
+                                self.files[dt] = []
+                            self.files[dt].append(fullname)
+                            break
+                        if self.files:
                             break
                         else:
                             log.warning('Waiting {} seconds for the next file, try {} ...'.format(self.delay, n + 1), 'NCDFReader')
                             time.sleep(self.delay)
-                    if not new_files:
+                    if not self.files:
-                        log.error('No more files found', 'NCDFReader')
                         return 0
-                    startDate = self.dt - datetime.timedelta(seconds=1)
-                    for fullname in new_files:
-                        thisFile = fullname.split('/')[-1]
-                        year = thisFile[3:7]
-                        if not year.isdigit():
-                            continue
-                        month = thisFile[7:9]
-                        if not month.isdigit():
-                            continue
-                        day = thisFile[9:11]
-                        if not day.isdigit():
-                            continue
-                        year, month, day = int(year), int(month), int(day)
-                        dateFile = datetime.date(year, month, day)
-                        timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
-                        if (startDate > dateFile):
-                            continue
-                        dt = datetime.datetime.combine(dateFile, timeFile)
-                        if dt not in self.files:
-                            self.files[dt] = []
-                        self.files[dt].append(fullname)
                     self.dates = self.files.keys()
                     self.dates.sort()
                     self.cursor = 0
                     return 1
                 def parseFile(self):
                     '''
                     '''
                     self.header = {}
                     for attr in self.fp.ncattrs():
                         self.header[str(attr)] = getattr(self.fp, attr)
                     self.data[self.header['TypeName']] = numpy.array(self.fp.variables[self.header['TypeName']])
                     if 'Azimuth' not in self.data:
                         self.data['Azimuth'] = numpy.array(self.fp.variables['Azimuth'])
                 def setNextFile(self):
                     '''
                     Open next files for the current datetime
                     '''
                     cursor = self.cursor
                     if not self.online_mode:
-                        self.dt = self.dates[cursor]
                         if cursor == len(self.dates):
                             if self.online:
+                                cursor = 0
+                                self.dt = self.dates[cursor]
                                 self.online_mode = True
+                                if not self.search_files_online():
+                                    log.success('No more files', 'NCDFReader')
+                                    return 0
                             else:
                                 log.success('No more files', 'NCDFReader')
                                 self.flagNoMoreFiles = 1
                                 return 0
                     else:
                         if not self.search_files_online():
                             return 0
+                        cursor = self.cursor
                     log.log(
                         'Opening: {}\'s files'.format(self.dates[cursor]),
                         'NCDFReader'
                         )
                     self.data = {}
                     for fullname in self.files[self.dates[cursor]]:
                         if os.path.splitext(fullname)[-1] == '.tgz':
                             tar = tarfile.open(fullname, 'r:gz')
                             tar.extractall('/tmp')
                             files = [os.path.join('/tmp', member.name) for member in tar.getmembers()]
                         else:
                             files = [fullname]
                         for filename in files:
                             if self.filename is not None:
                                 self.fp.close()
                             self.filename = filename
                             self.filedate = self.dates[cursor]
                             self.fp = Dataset(self.filename, 'r')
                             self.parseFile()
                     self.counter_records += 1
                     self.cursor += 1
                     return 1
                 def readNextFile(self):
                     while True:
                         self.flagDiscontinuousBlock = 0
                         if not self.setNextFile():
                             return 0
                         self.datatime = datetime.datetime.utcfromtimestamp(self.header['Time'])
                         if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
                            (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
                             log.warning(
                                 'Reading Record No. {}/{} -> {} [Skipping]'.format(
                                     self.counter_records,
                                     self.nrecords,
                                     self.datatime.ctime()),
                                 'NCDFReader')
                             continue
                         break
                     log.log(
                         'Reading Record No. {}/{} -> {}'.format(
                             self.counter_records,
                             self.nrecords,
                             self.datatime.ctime()),
                         'NCDFReader')
                     return 1
                 def set_output(self):
                     '''
                     Storing data from buffer to dataOut object
                     '''
                     self.dataOut.heightList = self.data.pop('Azimuth')
                     log.log('Parameters found: {}'.format(','.join(self.data.keys())),
                                 'PXReader')
                     self.dataOut.data_param = numpy.array(self.data.values())
                     self.dataOut.data_param[self.dataOut.data_param == -99900.] = numpy.nan
                     self.dataOut.parameters = self.data.keys()
                     self.dataOut.utctime = self.header['Time']
                     self.dataOut.utctimeInit = self.dataOut.utctime
                     self.dataOut.useLocalTime = False
                     self.dataOut.flagNoData = False
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                 def getData(self):
                     '''
                     Storing data from databuffer to dataOut object
                     '''
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         log.error('No file left to process', 'NCDFReader')
                         return 0
                     if not self.readNextFile():
                         self.dataOut.flagNoData = True
                         return 0
                     self.set_output()
                     return 1

schainpy/model/utils/jroutils_publish.py +3 -3

             '''
             @author: Juan C. Espinoza
             '''
             import os
             import glob
             import time
             import json
             import numpy
             import paho.mqtt.client as mqtt
             import zmq
             import datetime
             import ftplib
             from zmq.utils.monitor import recv_monitor_message
             from functools import wraps
             from threading import Thread
             from multiprocessing import Process
             from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
             from schainpy.model.data.jrodata import JROData
             from schainpy.utils import log
             MAXNUMX = 500
             MAXNUMY = 500
             PLOT_CODES = {
                 'rti': 0,            #Range time intensity (RTI).
                 'spc': 1,           #Spectra (and Cross-spectra) information.
                 'cspc': 2,          #Cross-Correlation information.
                 'coh': 3,            #Coherence map.
                 'base': 4,           #Base lines graphic.
                 'row': 5,            #Row Spectra.
                 'total' : 6,          #Total Power.
                 'drift' : 7,         #Drifts graphics.
                 'height' : 8,         #Height profile.
                 'phase' : 9,          #Signal Phase.
                 'power' : 16,
                 'noise' : 17,
                 'beacon' : 18,
                 #USED IN jroplot_parameters.py
                 'wind' : 22,
                 'skymap' : 23,
                 # 'MPHASE_CODE' : 24,
-                'moments' : 25,
+                'V' : 25,
-                'param' : 26,
+                'Z' : 26,
                 'spc_fit' : 27,
                 'ew_drifts' : 28,
                 'reflectivity': 30
             }
             class PrettyFloat(float):
                 def __repr__(self):
                     return '%.2f' % self
             def roundFloats(obj):
                 if isinstance(obj, list):
                     return map(roundFloats, obj)
                 elif isinstance(obj, float):
                     return round(obj, 2)
             def decimate(z, MAXNUMY):
                 dy = int(len(z[0])/MAXNUMY) + 1
                 return z[::, ::dy]
             class throttle(object):
                 '''
                 Decorator that prevents a function from being called more than once every
                 time period.
                 To create a function that cannot be called more than once a minute, but
                 will sleep until it can be called:
                 @throttle(minutes=1)
                 def foo():
                   pass
                 for i in range(10):
                   foo()
                   print "This function has run %s times." % i
                 '''
                 def __init__(self, seconds=0, minutes=0, hours=0):
                     self.throttle_period = datetime.timedelta(
                         seconds=seconds, minutes=minutes, hours=hours
                     )
                     self.time_of_last_call = datetime.datetime.min
                 def __call__(self, fn):
                     @wraps(fn)
                     def wrapper(*args, **kwargs):
                         coerce = kwargs.pop('coerce', None)
                         if coerce:
                             self.time_of_last_call = datetime.datetime.now()
                             return fn(*args, **kwargs)
                         else:
                             now = datetime.datetime.now()
                             time_since_last_call = now - self.time_of_last_call
                             time_left = self.throttle_period - time_since_last_call
                             if time_left > datetime.timedelta(seconds=0):
                                 return
                         self.time_of_last_call = datetime.datetime.now()
                         return fn(*args, **kwargs)
                     return wrapper
             class Data(object):
                 '''
                 Object to hold data to be plotted
                 '''
                 def __init__(self, plottypes, throttle_value, exp_code, buffering=True):
                     self.plottypes = plottypes
                     self.throttle = throttle_value
                     self.exp_code = exp_code
                     self.buffering = buffering
                     self.ended = False
                     self.localtime = False
                     self.__times = []
                     self.__heights = []
                 def __str__(self):
                     dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
                     return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times))
                 def __len__(self):
                     return len(self.__times)
                 def __getitem__(self, key):
                     if key not in self.data:
                         raise KeyError(log.error('Missing key: {}'.format(key)))
                     if 'spc' in key or not self.buffering:
                         ret = self.data[key]
                     else:
                         ret = numpy.array([self.data[key][x] for x in self.times])
                         if ret.ndim > 1:
                             ret = numpy.swapaxes(ret, 0, 1)
                     return ret
                 def __contains__(self, key):
                     return key in self.data
                 def setup(self):
                     '''
                     Configure object
                     '''
                     self.type = ''
                     self.ended = False
                     self.data = {}
                     self.__times = []
                     self.__heights = []
                     self.__all_heights = set()
                     for plot in self.plottypes:
                         if 'snr' in plot:
                             plot = 'snr'
                         self.data[plot] = {}
                 def shape(self, key):
                     '''
                     Get the shape of the one-element data for the given key
                     '''
                     if len(self.data[key]):
                         if 'spc' in key or not self.buffering:
                             return self.data[key].shape
                         return self.data[key][self.__times[0]].shape
                     return (0,)
                 def update(self, dataOut, tm):
                     '''
                     Update data object with new dataOut
                     '''
                     if tm in self.__times:
                         return
                     self.type = dataOut.type
                     self.parameters = getattr(dataOut, 'parameters', [])
                     if hasattr(dataOut, 'pairsList'):
                         self.pairs = dataOut.pairsList
                     self.channels = dataOut.channelList
                     self.interval = dataOut.getTimeInterval()
                     self.localtime = dataOut.useLocalTime
                     if 'spc' in self.plottypes or 'cspc' in self.plottypes:
                         self.xrange = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
                     self.__heights.append(dataOut.heightList)
                     self.__all_heights.update(dataOut.heightList)
                     self.__times.append(tm)
                     for plot in self.plottypes:
                         if plot == 'spc':
                             z = dataOut.data_spc/dataOut.normFactor
                             self.data[plot] = 10*numpy.log10(z)
                         if plot == 'cspc':
                             self.data[plot] = dataOut.data_cspc
                         if plot == 'noise':
                             buffer = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
                         if plot == 'rti':
                             buffer = dataOut.getPower()
                         if plot == 'snr_db':
                             self.data['snr'][tm] = dataOut.data_SNR
                         if plot == 'snr':
                             buffer = 10*numpy.log10(dataOut.data_SNR)
                         if plot == 'dop':
                             buffer = 10*numpy.log10(dataOut.data_DOP)
                         if plot == 'mean':
                             buffer = dataOut.data_MEAN
                         if plot == 'std':
                             buffer = dataOut.data_STD
                         if plot == 'coh':
                             buffer = dataOut.getCoherence()
                         if plot == 'phase':
                             buffer = dataOut.getCoherence(phase=True)
                         if plot == 'output':
                             buffer = dataOut.data_output
                         if plot == 'param':
                             buffer = dataOut.data_param
                         if self.buffering:
                             self.data[plot][tm] = buffer
                         else:
                             self.data[plot] = buffer
                 def normalize_heights(self):
                     '''
                     Ensure same-dimension of the data for different heighList
                     '''
                     H = numpy.array(list(self.__all_heights))
                     H.sort()
                     for key in self.data:
                         shape = self.shape(key)[:-1] + H.shape
                         for tm, obj in self.data[key].items():
                             h = self.__heights[self.__times.index(tm)]
                             if H.size == h.size:
                                 continue
                             index = numpy.where(numpy.in1d(H, h))[0]
                             dummy = numpy.zeros(shape) + numpy.nan
                             if len(shape) == 2:
                                 dummy[:, index] = obj
                             else:
                                 dummy[index] = obj
                             self.data[key][tm] = dummy
                     self.__heights = [H for tm in self.__times]
                 def jsonify(self, decimate=False):
                     '''
                     Convert data to json
                     '''
                     data = {}
                     tm = self.times[-1]
                     for key in self.data:
                         if key in ('spc', 'cspc') or not self.buffering:
                             dx = int(self.data[key].shape[1]/MAXNUMX) + 1
                             dy = int(self.data[key].shape[2]/MAXNUMY) + 1
                             data[key] = roundFloats(self.data[key][::, ::dx, ::dy].tolist())
                         else:
                             data[key] = roundFloats(self.data[key][tm].tolist())
                     ret = {'data': data}
                     ret['exp_code'] = self.exp_code
                     ret['time'] = tm
                     ret['interval'] = self.interval
                     ret['localtime'] = self.localtime
                     ret['yrange'] = roundFloats(self.heights.tolist())
                     if key in ('spc', 'cspc'):
                         ret['xrange'] = roundFloats(self.xrange[2][::dx].tolist())
                     else:
                         ret['xrange'] = []
                     if hasattr(self, 'pairs'):
                         ret['pairs'] = self.pairs
                     return json.dumps(ret)
                 @property
                 def times(self):
                     '''
                     Return the list of times of the current data
                     '''
                     ret = numpy.array(self.__times)
                     ret.sort()
                     return ret
                 @property
                 def heights(self):
                     '''
                     Return the list of heights of the current data
                     '''
                     return numpy.array(self.__heights[-1])
             class PublishData(Operation):
                 '''
                 Operation to send data over zmq.
                 '''
                 __attrs__ = ['host', 'port', 'delay', 'zeromq', 'mqtt', 'verbose']
                 def __init__(self, **kwargs):
                     """Inicio."""
                     Operation.__init__(self, **kwargs)
                     self.isConfig = False
                     self.client = None
                     self.zeromq = None
                     self.mqtt = None
                 def on_disconnect(self, client, userdata, rc):
                     if rc != 0:
                         log.warning('Unexpected disconnection.')
                     self.connect()
                 def connect(self):
                     log.warning('trying to connect')
                     try:
                         self.client.connect(
                             host=self.host,
                             port=self.port,
                             keepalive=60*10,
                             bind_address='')
                         self.client.loop_start()
                         # self.client.publish(
                         #     self.topic + 'SETUP',
                         #     json.dumps(setup),
                         #     retain=True
                         #     )
                     except:
                         log.error('MQTT Conection error.')
                         self.client = False
                 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, verbose=True, **kwargs):
                     self.counter = 0
                     self.topic = kwargs.get('topic', 'schain')
                     self.delay = kwargs.get('delay', 0)
                     self.plottype = kwargs.get('plottype', 'spectra')
                     self.host = kwargs.get('host', "10.10.10.82")
                     self.port = kwargs.get('port', 3000)
                     self.clientId = clientId
                     self.cnt = 0
                     self.zeromq = zeromq
                     self.mqtt = kwargs.get('plottype', 0)
                     self.client = None
                     self.verbose = verbose
                     setup = []
                     if mqtt is 1:
                         self.client = mqtt.Client(
                             client_id=self.clientId + self.topic + 'SCHAIN',
                             clean_session=True)
                         self.client.on_disconnect = self.on_disconnect
                         self.connect()
                         for plot in self.plottype:
                             setup.append({
                                 'plot': plot,
                                 'topic': self.topic + plot,
                                 'title': getattr(self, plot + '_' + 'title', False),
                                 'xlabel': getattr(self, plot + '_' + 'xlabel', False),
                                 'ylabel': getattr(self, plot + '_' + 'ylabel', False),
                                 'xrange': getattr(self, plot + '_' + 'xrange', False),
                                 'yrange': getattr(self, plot + '_' + 'yrange', False),
                                 'zrange': getattr(self, plot + '_' + 'zrange', False),
                             })
                     if zeromq is 1:
                         context = zmq.Context()
                         self.zmq_socket = context.socket(zmq.PUSH)
                         server = kwargs.get('server', 'zmq.pipe')
                         if 'tcp://' in server:
                             address = server
                         else:
                             address = 'ipc:///tmp/%s' % server
                         self.zmq_socket.connect(address)
                         time.sleep(1)
                 def publish_data(self):
                     self.dataOut.finished = False
                     if self.mqtt is 1:
                         yData = self.dataOut.heightList[:2].tolist()
                         if self.plottype == 'spectra':
                             data = getattr(self.dataOut, 'data_spc')
                             z = data/self.dataOut.normFactor
                             zdB = 10*numpy.log10(z)
                             xlen, ylen = zdB[0].shape
                             dx = int(xlen/MAXNUMX) + 1
                             dy = int(ylen/MAXNUMY) + 1
                             Z = [0 for i in self.dataOut.channelList]
                             for i in self.dataOut.channelList:
                                 Z[i] = zdB[i][::dx, ::dy].tolist()
                             payload = {
                                 'timestamp': self.dataOut.utctime,
                                 'data': roundFloats(Z),
                                 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                 'interval': self.dataOut.getTimeInterval(),
                                 'type': self.plottype,
                                 'yData': yData
                             }
                         elif self.plottype in ('rti', 'power'):
                             data = getattr(self.dataOut, 'data_spc')
                             z = data/self.dataOut.normFactor
                             avg = numpy.average(z, axis=1)
                             avgdB = 10*numpy.log10(avg)
                             xlen, ylen = z[0].shape
                             dy = numpy.floor(ylen/self.__MAXNUMY) + 1
                             AVG = [0 for i in self.dataOut.channelList]
                             for i in self.dataOut.channelList:
                                 AVG[i] = avgdB[i][::dy].tolist()
                             payload = {
                                 'timestamp': self.dataOut.utctime,
                                 'data': roundFloats(AVG),
                                 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                 'interval': self.dataOut.getTimeInterval(),
                                 'type': self.plottype,
                                 'yData': yData
                             }
                         elif self.plottype == 'noise':
                             noise = self.dataOut.getNoise()/self.dataOut.normFactor
                             noisedB = 10*numpy.log10(noise)
                             payload = {
                                 'timestamp': self.dataOut.utctime,
                                 'data': roundFloats(noisedB.reshape(-1, 1).tolist()),
                                 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                 'interval': self.dataOut.getTimeInterval(),
                                 'type': self.plottype,
                                 'yData': yData
                             }
                         elif self.plottype == 'snr':
                             data = getattr(self.dataOut, 'data_SNR')
                             avgdB = 10*numpy.log10(data)
                             ylen = data[0].size
                             dy = numpy.floor(ylen/self.__MAXNUMY) + 1
                             AVG = [0 for i in self.dataOut.channelList]
                             for i in self.dataOut.channelList:
                                 AVG[i] = avgdB[i][::dy].tolist()
                             payload = {
                                 'timestamp': self.dataOut.utctime,
                                 'data': roundFloats(AVG),
                                 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                 'type': self.plottype,
                                 'yData': yData
                             }
                         else:
                             print "Tipo de grafico invalido"
                             payload = {
                                 'data': 'None',
                                 'timestamp': 'None',
                                 'type': None
                             }
                         self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
                     if self.zeromq is 1:
                         if self.verbose:
                             log.log(
                                 'Sending {} - {}'.format(self.dataOut.type, self.dataOut.datatime),
                                 self.name
                             )
                         self.zmq_socket.send_pyobj(self.dataOut)
                 def run(self, dataOut, **kwargs):
                     self.dataOut = dataOut
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.publish_data()
                     time.sleep(self.delay)
                 def close(self):
                     if self.zeromq is 1:
                         self.dataOut.finished = True
                         self.zmq_socket.send_pyobj(self.dataOut)
                         time.sleep(0.1)
                         self.zmq_socket.close()
                     if self.client:
                         self.client.loop_stop()
                         self.client.disconnect()
             class ReceiverData(ProcessingUnit):
                 __attrs__ = ['server']
                 def __init__(self, **kwargs):
                     ProcessingUnit.__init__(self, **kwargs)
                     self.isConfig = False
                     server = kwargs.get('server', 'zmq.pipe')
                     if 'tcp://' in server:
                         address = server
                     else:
                         address = 'ipc:///tmp/%s' % server
                     self.address = address
                     self.dataOut = JROData()
                 def setup(self):
                     self.context = zmq.Context()
                     self.receiver = self.context.socket(zmq.PULL)
                     self.receiver.bind(self.address)
                     time.sleep(0.5)
                     log.success('ReceiverData from {}'.format(self.address))
                 def run(self):
                     if not self.isConfig:
                         self.setup()
                         self.isConfig = True
                     self.dataOut = self.receiver.recv_pyobj()
                     log.log('{} - {}'.format(self.dataOut.type,
                                              self.dataOut.datatime.ctime(),),
                             'Receiving')
             class PlotterReceiver(ProcessingUnit, Process):
                 throttle_value = 5
                 __attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle',
                     'exp_code', 'web_server', 'buffering']
                 def __init__(self, **kwargs):
                     ProcessingUnit.__init__(self, **kwargs)
                     Process.__init__(self)
                     self.mp = False
                     self.isConfig = False
                     self.isWebConfig = False
                     self.connections = 0
                     server = kwargs.get('server', 'zmq.pipe')
                     web_server = kwargs.get('web_server', None)
                     if 'tcp://' in server:
                         address = server
                     else:
                         address = 'ipc:///tmp/%s' % server
                     self.address = address
                     self.web_address = web_server
                     self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
                     self.realtime = kwargs.get('realtime', False)
                     self.localtime = kwargs.get('localtime', True)
                     self.buffering = kwargs.get('buffering', True)
                     self.throttle_value = kwargs.get('throttle', 5)
                     self.exp_code = kwargs.get('exp_code', None)
                     self.sendData = self.initThrottle(self.throttle_value)
                     self.dates = []
                     self.setup()
                 def setup(self):
                     self.data = Data(self.plottypes, self.throttle_value, self.exp_code, self.buffering)
                     self.isConfig = True
                 def event_monitor(self, monitor):
                     events = {}
                     for name in dir(zmq):
                         if name.startswith('EVENT_'):
                             value = getattr(zmq, name)
                             events[value] = name
                     while monitor.poll():
                         evt = recv_monitor_message(monitor)
                         if evt['event'] == 32:
                             self.connections += 1
                         if evt['event'] == 512:
                             pass
                         evt.update({'description': events[evt['event']]})
                         if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
                             break
                     monitor.close()
                     print('event monitor thread done!')
                 def initThrottle(self, throttle_value):
                     @throttle(seconds=throttle_value)
                     def sendDataThrottled(fn_sender, data):
                         fn_sender(data)
                     return sendDataThrottled
                 def send(self, data):
                     log.log('Sending {}'.format(data), self.name)
                     self.sender.send_pyobj(data)
                 def run(self):
                     log.log(
                         'Starting from {}'.format(self.address),
                         self.name
                     )
                     self.context = zmq.Context()
                     self.receiver = self.context.socket(zmq.PULL)
                     self.receiver.bind(self.address)
                     monitor = self.receiver.get_monitor_socket()
                     self.sender = self.context.socket(zmq.PUB)
                     if self.web_address:
                         log.success(
                             'Sending to web: {}'.format(self.web_address),
                             self.name
                         )
                         self.sender_web = self.context.socket(zmq.PUSH)
                         self.sender_web.connect(self.web_address)
                         time.sleep(1)
                     if 'server' in self.kwargs:
                         self.sender.bind("ipc:///tmp/{}.plots".format(self.kwargs['server']))
                     else:
                         self.sender.bind("ipc:///tmp/zmq.plots")
                     time.sleep(2)
                     t = Thread(target=self.event_monitor, args=(monitor,))
                     t.start()
                     while True:
                         dataOut = self.receiver.recv_pyobj()
                         if not dataOut.flagNoData:
                             if dataOut.type == 'Parameters':
                                 tm = dataOut.utctimeInit
                             else:
                                 tm = dataOut.utctime
                             if dataOut.useLocalTime:
                                 if not self.localtime:
                                     tm += time.timezone
                                 dt = datetime.datetime.fromtimestamp(tm).date()
                             else:
                                 if self.localtime:
                                     tm -= time.timezone
                                 dt = datetime.datetime.utcfromtimestamp(tm).date()
                             coerce = False
                             if dt not in self.dates:
                                 if self.data:
                                     self.data.ended = True
                                     self.send(self.data)
                                     coerce = True
                                 self.data.setup()
                                 self.dates.append(dt)
                             self.data.update(dataOut, tm)
                         if dataOut.finished is True:
                             self.connections -= 1
                             if self.connections == 0 and dt in self.dates:
                                 self.data.ended = True
                                 self.send(self.data)
                                 self.data.setup()
                         else:
                             if self.realtime:
                                 self.send(self.data)
                                 if self.web_address:
                                     payload = self.data.jsonify()
                                     log.log('Sending to web... type:{}, size:{}'.format(dataOut.type, len(payload)), self.name)
                                     self.sender_web.send(payload)
                             else:
                                 self.sendData(self.send, self.data, coerce=coerce)
                                 coerce = False
                     return
             class SendToFTP(Operation, Process):
                 '''
                 Operation to send data over FTP.
                 '''
                 __attrs__ = ['server', 'username', 'password', 'patterns', 'timeout']
                 def __init__(self, **kwargs):
                     '''
                     patterns = [(local1, remote1, ext, delay, exp_code, sub_exp_code), ...]
                     '''
                     Operation.__init__(self, **kwargs)
                     Process.__init__(self)
                     self.server = kwargs.get('server')
                     self.username = kwargs.get('username')
                     self.password = kwargs.get('password')
                     self.patterns = kwargs.get('patterns')
                     self.timeout = kwargs.get('timeout', 10)
                     self.times = [time.time() for p in self.patterns]
                     self.latest = ['' for p in self.patterns]
                     self.mp = False
                     self.ftp = None
                 def setup(self):
                     log.log('Connecting to ftp://{}'.format(self.server), self.name)
                     try:
                         self.ftp = ftplib.FTP(self.server, timeout=self.timeout)
                     except ftplib.all_errors:
                         log.error('Server connection fail: {}'.format(self.server), self.name)
                         if self.ftp is not None:
                             self.ftp.close()
                         self.ftp = None
                         self.isConfig = False
                         return
                     try:
                         self.ftp.login(self.username, self.password)
                     except ftplib.all_errors:
                         log.error('The given username y/o password are incorrect', self.name)
                         if self.ftp is not None:
                             self.ftp.close()
                         self.ftp = None
                         self.isConfig = False
                         return
                     log.success('Connection success', self.name)
                     self.isConfig = True
                     return
                 def check(self):
                     try:
                         self.ftp.voidcmd("NOOP")
                     except:
                         log.warning('Connection lost... trying to reconnect', self.name)
                         if self.ftp is not None:
                             self.ftp.close()
                         self.ftp = None
                         self.setup()
                 def find_files(self, path, ext):
                     files = glob.glob1(path, '*{}'.format(ext))
                     files.sort()
                     if files:
                         return files[-1]
                     return None
                 def getftpname(self, filename, exp_code, sub_exp_code):
                     thisDatetime = datetime.datetime.strptime(filename.split('_')[1], '%Y%m%d')
                     YEAR_STR = '%4.4d'%thisDatetime.timetuple().tm_year
                     DOY_STR = '%3.3d'%thisDatetime.timetuple().tm_yday
                     exp_code = '%3.3d'%exp_code
                     sub_exp_code = '%2.2d'%sub_exp_code
-                    plot_code = '%2.2d'% PLOT_CODES[filename.split('_')[0].split('-')[0]]
+                    plot_code = '%2.2d'% PLOT_CODES[filename.split('_')[0].split('-')[1]]
                     name = YEAR_STR + DOY_STR + '00' + exp_code + sub_exp_code + plot_code + '00.png'
                     return name
                 def upload(self, src, dst):
                     log.log('Uploading {} '.format(src), self.name, nl=False)
                     fp = open(src, 'rb')
                     command = 'STOR {}'.format(dst)
                     try:
                         self.ftp.storbinary(command, fp, blocksize=1024)
                     except ftplib.all_errors, e:
                         log.error('{}'.format(e), self.name)
                         if self.ftp is not None:
                             self.ftp.close()
                         self.ftp = None
                         return
                     try:
                         self.ftp.sendcmd('SITE CHMOD 755 {}'.format(dst))
                     except ftplib.all_errors, e:
                         log.error('{}'.format(e), self.name)
                         if self.ftp is not None:
                             self.ftp.close()
                         self.ftp = None
                     fp.close()
                     log.success('OK', tag='')
                 def send_files(self):
                     for x, pattern in enumerate(self.patterns):
                         local, remote, ext, delay, exp_code, sub_exp_code = pattern
                         if time.time()-self.times[x] >= delay:
                             srcname = self.find_files(local, ext)
                             if srcname is None or srcname == self.latest[x]:
                                 continue
                             if 'png' in ext:
                                 dstname = self.getftpname(srcname, exp_code, sub_exp_code)
                             else:
                                 dstname = srcname
                             src = os.path.join(local, srcname)
                             if os.path.getmtime(src) < time.time() - 30*60:
                                 continue
                             dst = os.path.join(remote, dstname)
                             if self.ftp is None:
                                 continue
                             self.upload(src, dst)
                             self.times[x] = time.time()
                             self.latest[x] = srcname
                 def run(self):
                     while True:
                         if not self.isConfig:
                             self.setup()
                         if self.ftp is not None:
                             self.check()
                             self.send_files()
                         time.sleep(2)
                 def close():
                     if self.ftp is not None:
                         if self.ftp is not None:
                             self.ftp.close()
                     self.terminate()

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages