schain Commit - r1362:036afbeb089e · Jicamarca Repository

Merge branch 'master' of http://jro-dev.igp.gob.pe/rhodecode/schain

Danny Scipión -

r1362:036afbeb089e

parent child

Context file:

r1362:036afbeb089e

Collapse all files

schainpy/model/graphics/jroplot_base.py +22 -19

             # Copyright (c) 2012-2020 Jicamarca Radio Observatory
             # All rights reserved.
             #
             # Distributed under the terms of the BSD 3-clause license.
             """Base class to create plot operations
             """
             import os
             import sys
             import zmq
             import time
             import numpy
             import datetime
             from collections import deque
             from functools import wraps
             from threading import Thread
             import matplotlib
             if 'BACKEND' in os.environ:
                 matplotlib.use(os.environ['BACKEND'])
             elif 'linux' in sys.platform:
                 matplotlib.use("TkAgg")
             elif 'darwin' in sys.platform:
                 matplotlib.use('MacOSX')
             else:
                 from schainpy.utils import log
                 log.warning('Using default Backend="Agg"', 'INFO')
                 matplotlib.use('Agg')
             import matplotlib.pyplot as plt
             from matplotlib.patches import Polygon
             from mpl_toolkits.axes_grid1 import make_axes_locatable
             from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
             from schainpy.model.data.jrodata import PlotterData
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.utils import log
             jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
             blu_values = matplotlib.pyplot.get_cmap(
                 'seismic_r', 20)(numpy.arange(20))[10:15]
             ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
                 'jro', numpy.vstack((blu_values, jet_values)))
             matplotlib.pyplot.register_cmap(cmap=ncmap)
             CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis',
                                                'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
             EARTH_RADIUS = 6.3710e3
             def ll2xy(lat1, lon1, lat2, lon2):
                 p = 0.017453292519943295
                 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
                     numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
                 r = 12742 * numpy.arcsin(numpy.sqrt(a))
                 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
                                       * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
                 theta = -theta + numpy.pi/2
                 return r*numpy.cos(theta), r*numpy.sin(theta)
             def km2deg(km):
                 '''
                 Convert distance in km to degrees
                 '''
                 return numpy.rad2deg(km/EARTH_RADIUS)
             def figpause(interval):
                 backend = plt.rcParams['backend']
                 if backend in matplotlib.rcsetup.interactive_bk:
                     figManager = matplotlib._pylab_helpers.Gcf.get_active()
                     if figManager is not None:
                         canvas = figManager.canvas
                         if canvas.figure.stale:
                             canvas.draw()
                         try:
                             canvas.start_event_loop(interval)
                         except:
                             pass
                         return
             def popup(message):
                 '''
                 '''
                 fig = plt.figure(figsize=(12, 8), facecolor='r')
                 text = '\n'.join([s.strip() for s in message.split(':')])
                 fig.text(0.01, 0.5, text, ha='left', va='center',
                          size='20', weight='heavy', color='w')
                 fig.show()
                 figpause(1000)
             class Throttle(object):
                 '''
                 Decorator that prevents a function from being called more than once every
                 time period.
                 To create a function that cannot be called more than once a minute, but
                 will sleep until it can be called:
                 @Throttle(minutes=1)
                 def foo():
                   pass
                 for i in range(10):
                   foo()
                   print "This function has run %s times." % i
                 '''
                 def __init__(self, seconds=0, minutes=0, hours=0):
                     self.throttle_period = datetime.timedelta(
                         seconds=seconds, minutes=minutes, hours=hours
                     )
                     self.time_of_last_call = datetime.datetime.min
                 def __call__(self, fn):
                     @wraps(fn)
                     def wrapper(*args, **kwargs):
                         coerce = kwargs.pop('coerce', None)
                         if coerce:
                             self.time_of_last_call = datetime.datetime.now()
                             return fn(*args, **kwargs)
                         else:
                             now = datetime.datetime.now()
                             time_since_last_call = now - self.time_of_last_call
                             time_left = self.throttle_period - time_since_last_call
                             if time_left > datetime.timedelta(seconds=0):
                                 return
                         self.time_of_last_call = datetime.datetime.now()
                         return fn(*args, **kwargs)
                     return wrapper
             def apply_throttle(value):
                 @Throttle(seconds=value)
                 def fnThrottled(fn):
                     fn()
                 return fnThrottled
             @MPDecorator
             class Plot(Operation):
                 """Base class for Schain plotting operations
                 This class should never be use directtly you must subclass a new operation,
                 children classes must be defined as follow:
                 ExamplePlot(Plot):
                     CODE = 'code'
                     colormap = 'jet'
                     plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
                     def setup(self):
                         pass
                     def plot(self):
                         pass
                 """
                 CODE = 'Figure'
                 colormap = 'jet'
                 bgcolor = 'white'
                 buffering = True
                 __missing = 1E30
                 __attrs__ = ['show', 'save', 'ymin', 'ymax', 'zmin', 'zmax', 'title',
                              'showprofile']
                 def __init__(self):
                     Operation.__init__(self)
                     self.isConfig = False
                     self.isPlotConfig = False
                     self.save_time = 0
                     self.sender_time = 0
                     self.data = None
                     self.firsttime = True
                     self.sender_queue = deque(maxlen=10)
                     self.plots_adjust = {'left': 0.125, 'right': 0.9, 'bottom': 0.15, 'top': 0.9, 'wspace': 0.2, 'hspace': 0.2}
                 def __fmtTime(self, x, pos):
                     '''
                     '''
                     return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
                 def __setup(self, **kwargs):
                     '''
                     Initialize variables
                     '''
                     self.figures = []
                     self.axes = []
                     self.cb_axes = []
                     self.localtime = kwargs.pop('localtime', True)
                     self.show = kwargs.get('show', True)
                     self.save = kwargs.get('save', False)
                     self.save_period = kwargs.get('save_period', 0)
                     self.colormap = kwargs.get('colormap', self.colormap)
                     self.colormap_coh = kwargs.get('colormap_coh', 'jet')
                     self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
                     self.colormaps = kwargs.get('colormaps', None)
                     self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
                     self.showprofile = kwargs.get('showprofile', False)
                     self.title = kwargs.get('wintitle', self.CODE.upper())
                     self.cb_label = kwargs.get('cb_label', None)
                     self.cb_labels = kwargs.get('cb_labels', None)
                     self.labels = kwargs.get('labels', None)
                     self.xaxis = kwargs.get('xaxis', 'frequency')
                     self.zmin = kwargs.get('zmin', None)
                     self.zmax = kwargs.get('zmax', None)
                     self.zlimits = kwargs.get('zlimits', None)
                     self.xmin = kwargs.get('xmin', None)
                     self.xmax = kwargs.get('xmax', None)
                     self.xrange = kwargs.get('xrange', 12)
                     self.xscale = kwargs.get('xscale', None)
                     self.ymin = kwargs.get('ymin', None)
                     self.ymax = kwargs.get('ymax', None)
                     self.yscale = kwargs.get('yscale', None)
                     self.xlabel = kwargs.get('xlabel', None)
                     self.attr_time = kwargs.get('attr_time', 'utctime')
                     self.attr_data = kwargs.get('attr_data', 'data_param')
                     self.decimation = kwargs.get('decimation', None)
-                    self.showSNR = kwargs.get('showSNR', False)
                     self.oneFigure = kwargs.get('oneFigure', True)
                     self.width = kwargs.get('width', None)
                     self.height = kwargs.get('height', None)
                     self.colorbar = kwargs.get('colorbar', True)
                     self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
                     self.channels = kwargs.get('channels', None)
                     self.titles = kwargs.get('titles', [])
                     self.polar = False
                     self.type = kwargs.get('type', 'iq')
                     self.grid = kwargs.get('grid', False)
                     self.pause = kwargs.get('pause', False)
                     self.save_code = kwargs.get('save_code', self.CODE)
                     self.throttle = kwargs.get('throttle', 0)
                     self.exp_code = kwargs.get('exp_code', None)
                     self.server = kwargs.get('server', False)
                     self.sender_period = kwargs.get('sender_period', 60)
                     self.tag = kwargs.get('tag', '')
                     self.height_index = kwargs.get('height_index', None)
                     self.__throttle_plot = apply_throttle(self.throttle)
                     code = self.attr_data if self.attr_data else self.CODE
                     self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
                     if self.server:
                         if not self.server.startswith('tcp://'):
                             self.server = 'tcp://{}'.format(self.server)
                         log.success(
                             'Sending to server: {}'.format(self.server),
                             self.name
                         )
+                    if isinstance(self.attr_data, str):
+                        self.attr_data = [self.attr_data]
                 def __setup_plot(self):
                     '''
                     Common setup for all figures, here figures and axes are created
                     '''
                     self.setup()
                     self.time_label = 'LT' if self.localtime else 'UTC'
                     if self.width is None:
                         self.width = 8
                     self.figures = []
                     self.axes = []
                     self.cb_axes = []
                     self.pf_axes = []
                     self.cmaps = []
                     size = '15%' if self.ncols == 1 else '30%'
                     pad = '4%' if self.ncols == 1 else '8%'
                     if self.oneFigure:
                         if self.height is None:
                             self.height = 1.4 * self.nrows + 1
                         fig = plt.figure(figsize=(self.width, self.height),
                                          edgecolor='k',
                                          facecolor='w')
                         self.figures.append(fig)
                         for n in range(self.nplots):
                             ax = fig.add_subplot(self.nrows, self.ncols,
                                                  n + 1, polar=self.polar)
                             ax.tick_params(labelsize=8)
                             ax.firsttime = True
                             ax.index = 0
                             ax.press = None
                             self.axes.append(ax)
                             if self.showprofile:
                                 cax = self.__add_axes(ax, size=size, pad=pad)
                                 cax.tick_params(labelsize=8)
                                 self.pf_axes.append(cax)
                     else:
                         if self.height is None:
                             self.height = 3
                         for n in range(self.nplots):
                             fig = plt.figure(figsize=(self.width, self.height),
                                              edgecolor='k',
                                              facecolor='w')
                             ax = fig.add_subplot(1, 1, 1, polar=self.polar)
                             ax.tick_params(labelsize=8)
                             ax.firsttime = True
                             ax.index = 0
                             ax.press = None
                             self.figures.append(fig)
                             self.axes.append(ax)
                             if self.showprofile:
                                 cax = self.__add_axes(ax, size=size, pad=pad)
                                 cax.tick_params(labelsize=8)
                                 self.pf_axes.append(cax)
                     for n in range(self.nrows):
+                        print(self.nrows)
                         if self.colormaps is not None:
                             cmap = plt.get_cmap(self.colormaps[n])
                         else:
                             cmap = plt.get_cmap(self.colormap)
                         cmap.set_bad(self.bgcolor, 1.)
                         self.cmaps.append(cmap)
                 def __add_axes(self, ax, size='30%', pad='8%'):
                     '''
                     Add new axes to the given figure
                     '''
                     divider = make_axes_locatable(ax)
                     nax = divider.new_horizontal(size=size, pad=pad)
                     ax.figure.add_axes(nax)
                     return nax
                 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
                     '''
                     Create a masked array for missing data
                     '''
                     if x_buffer.shape[0] < 2:
                         return x_buffer, y_buffer, z_buffer
                     deltas = x_buffer[1:] - x_buffer[0:-1]
                     x_median = numpy.median(deltas)
                     index = numpy.where(deltas > 5 * x_median)
                     if len(index[0]) != 0:
                         z_buffer[::, index[0], ::] = self.__missing
                         z_buffer = numpy.ma.masked_inside(z_buffer,
 .99 * self.__missing,
 .01 * self.__missing)
                     return x_buffer, y_buffer, z_buffer
                 def decimate(self):
                     # dx = int(len(self.x)/self.__MAXNUMX) + 1
                     dy = int(len(self.y) / self.decimation) + 1
                     # x = self.x[::dx]
                     x = self.x
                     y = self.y[::dy]
                     z = self.z[::, ::, ::dy]
                     return x, y, z
                 def format(self):
                     '''
                     Set min and max values, labels, ticks and titles
                     '''
                     for n, ax in enumerate(self.axes):
                         if ax.firsttime:
                             if self.xaxis != 'time':
                                 xmin = self.xmin
                                 xmax = self.xmax
                             else:
                                 xmin = self.tmin
                                 xmax = self.tmin + self.xrange*60*60
                                 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
                                 ax.xaxis.set_major_locator(LinearLocator(9))
                             ymin = self.ymin if self.ymin is not None else numpy.nanmin(self.y[numpy.isfinite(self.y)])
                             ymax = self.ymax if self.ymax is not None else numpy.nanmax(self.y[numpy.isfinite(self.y)])
                             ax.set_facecolor(self.bgcolor)
                             if self.xscale:
                                 ax.xaxis.set_major_formatter(FuncFormatter(
                                     lambda x, pos: '{0:g}'.format(x*self.xscale)))
                             if self.yscale:
                                 ax.yaxis.set_major_formatter(FuncFormatter(
                                     lambda x, pos: '{0:g}'.format(x*self.yscale)))
                             if self.xlabel is not None:
                                 ax.set_xlabel(self.xlabel)
                             if self.ylabel is not None:
                                 ax.set_ylabel(self.ylabel)
                             if self.showprofile:
                                 self.pf_axes[n].set_ylim(ymin, ymax)
                                 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
                                 self.pf_axes[n].set_xlabel('dB')
                                 self.pf_axes[n].grid(b=True, axis='x')
                                 [tick.set_visible(False)
                                  for tick in self.pf_axes[n].get_yticklabels()]
                             if self.colorbar:
                                 ax.cbar = plt.colorbar(
                                     ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
                                 ax.cbar.ax.tick_params(labelsize=8)
                                 ax.cbar.ax.press = None
                                 if self.cb_label:
                                     ax.cbar.set_label(self.cb_label, size=8)
                                 elif self.cb_labels:
                                     ax.cbar.set_label(self.cb_labels[n], size=8)
                             else:
                                 ax.cbar = None
                             ax.set_xlim(xmin, xmax)
                             ax.set_ylim(ymin, ymax)
                             ax.firsttime = False
                             if self.grid:
                                 ax.grid(True)
                         if not self.polar:
                             ax.set_title('{} {} {}'.format(
                                 self.titles[n],
                                 self.getDateTime(self.data.max_time).strftime(
                                     '%Y-%m-%d %H:%M:%S'),
                                 self.time_label),
                                 size=8)
                         else:
                             ax.set_title('{}'.format(self.titles[n]), size=8)
                             ax.set_ylim(0, 90)
                             ax.set_yticks(numpy.arange(0, 90, 20))
                             ax.yaxis.labelpad = 40
                     if self.firsttime:
                         for n, fig in enumerate(self.figures):
                             fig.subplots_adjust(**self.plots_adjust)
                         self.firsttime = False
                 def clear_figures(self):
                     '''
                     Reset axes for redraw plots
                     '''
                     for ax in self.axes+self.pf_axes+self.cb_axes:
                         ax.clear()
                         ax.firsttime = True
                         if hasattr(ax, 'cbar') and ax.cbar:
                             ax.cbar.remove()
                 def __plot(self):
                     '''
                     Main function to plot, format and save figures
                     '''
                     self.plot()
                     self.format()
                     for n, fig in enumerate(self.figures):
                         if self.nrows == 0 or self.nplots == 0:
                             log.warning('No data', self.name)
                             fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
                             fig.canvas.manager.set_window_title(self.CODE)
                             continue
                         fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
                                                                              self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
                         fig.canvas.draw()
                         if self.show:
                             fig.show()
                             figpause(0.01)
                         if self.save:
                             self.save_figure(n)
                     if self.server:
                         self.send_to_server()
                 def __update(self, dataOut, timestamp):
                     '''
                     '''
                     metadata = {
                         'yrange': dataOut.heightList,
                         'interval': dataOut.timeInterval,
                         'channels': dataOut.channelList
                     }
                     data, meta = self.update(dataOut)
                     metadata.update(meta)
                     self.data.update(data, timestamp, metadata)
                 def save_figure(self, n):
                     '''
                     '''
                     if (self.data.max_time - self.save_time) <= self.save_period:
                         return
                     self.save_time = self.data.max_time
                     fig = self.figures[n]
-                    figname = os.path.join(
-                        self.save,
-                        self.save_code,
-                        '{}_{}.png'.format(
-                            self.save_code,
-                            self.getDateTime(self.data.max_time).strftime(
-                                '%Y%m%d_%H%M%S'
-                                ),
-                    log.log('Saving figure: {}'.format(figname), self.name)
-                    if not os.path.isdir(os.path.dirname(figname)):
-                        os.makedirs(os.path.dirname(figname))
-                    fig.savefig(figname)
                     if self.throttle == 0:
                         figname = os.path.join(
                             self.save,
-                            '{}_{}.png'.format(
+                            self.save_code,
+                            '{}_{}.png'.format(
                                 self.save_code,
-                                self.getDateTime(self.data.min_time).strftime(
+                                self.getDateTime(self.data.max_time).strftime(
-                                    '%Y%m%d'
+                                    '%Y%m%d_%H%M%S'
                                     ),
                                 )
                             )
+                        log.log('Saving figure: {}'.format(figname), self.name)
+                        if not os.path.isdir(os.path.dirname(figname)):
+                            os.makedirs(os.path.dirname(figname))
                         fig.savefig(figname)
+                    figname = os.path.join(
+                        self.save,
+                        '{}_{}.png'.format(
+                            self.save_code,
+                            self.getDateTime(self.data.min_time).strftime(
+                                '%Y%m%d'
+                                ),
+                            )
+                        )
+                    fig.savefig(figname)
                 def send_to_server(self):
                     '''
                     '''
                     if self.exp_code == None:
                         log.warning('Missing `exp_code` skipping sending to server...')
                     last_time = self.data.max_time
                     interval = last_time - self.sender_time
                     if interval < self.sender_period:
                         return
                     self.sender_time = last_time
                     attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
                     for attr in attrs:
                         value = getattr(self, attr)
                         if value:
                             if isinstance(value, (numpy.float32, numpy.float64)):
                                 value = round(float(value), 2)
                             self.data.meta[attr] = value
                     if self.colormap == 'jet':
                         self.data.meta['colormap'] = 'Jet'
                     elif 'RdBu' in self.colormap:
                         self.data.meta['colormap'] = 'RdBu'
                     else:
                         self.data.meta['colormap'] = 'Viridis'
                     self.data.meta['interval'] = int(interval)
                     self.sender_queue.append(last_time)
                     while True:
                         try:
                             tm = self.sender_queue.popleft()
                         except IndexError:
                             break
                         msg = self.data.jsonify(tm, self.save_code, self.plot_type)
                         self.socket.send_string(msg)
                         socks = dict(self.poll.poll(2000))
                         if socks.get(self.socket) == zmq.POLLIN:
                             reply = self.socket.recv_string()
                             if reply == 'ok':
                                 log.log("Response from server ok", self.name)
                                 time.sleep(0.1)
                                 continue
                             else:
                                 log.warning(
                                     "Malformed reply from server: {}".format(reply), self.name)
                         else:
                             log.warning(
                                 "No response from server, retrying...", self.name)
                         self.sender_queue.appendleft(tm)
                         self.socket.setsockopt(zmq.LINGER, 0)
                         self.socket.close()
                         self.poll.unregister(self.socket)
                         self.socket = self.context.socket(zmq.REQ)
                         self.socket.connect(self.server)
                         self.poll.register(self.socket, zmq.POLLIN)
                         break
                 def setup(self):
                     '''
                     This method should be implemented in the child class, the following
                     attributes should be set:
                     self.nrows: number of rows
                     self.ncols: number of cols
                     self.nplots: number of plots (channels or pairs)
                     self.ylabel: label for Y axes
                     self.titles: list of axes title
                     '''
                     raise NotImplementedError
                 def plot(self):
                     '''
                     Must be defined in the child class, the actual plotting method
                     '''
                     raise NotImplementedError
                 def update(self, dataOut):
                     '''
                     Must be defined in the child class, update self.data with new data
                     '''
                     data = {
                         self.CODE: getattr(dataOut, 'data_{}'.format(self.CODE))
                     }
                     meta = {}
                     return data, meta
                 def run(self, dataOut, **kwargs):
                     '''
                     Main plotting routine
                     '''
                     if self.isConfig is False:
                         self.__setup(**kwargs)
                         if self.localtime:
                             self.getDateTime = datetime.datetime.fromtimestamp
                         else:
                             self.getDateTime = datetime.datetime.utcfromtimestamp
                         self.data.setup()
                         self.isConfig = True
                         if self.server:
                             self.context = zmq.Context()
                             self.socket = self.context.socket(zmq.REQ)
                             self.socket.connect(self.server)
                             self.poll = zmq.Poller()
                             self.poll.register(self.socket, zmq.POLLIN)
                     tm = getattr(dataOut, self.attr_time)
                     if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
                         self.save_time = tm
                         self.__plot()
                         self.tmin += self.xrange*60*60
                         self.data.setup()
                         self.clear_figures()
                     self.__update(dataOut, tm)
                     if self.isPlotConfig is False:
                         self.__setup_plot()
                         self.isPlotConfig = True
                         if self.xaxis == 'time':
                             dt = self.getDateTime(tm)
                             if self.xmin is None:
                                 self.tmin = tm
                                 self.xmin = dt.hour
                             minutes = (self.xmin-int(self.xmin)) * 60
                             seconds = (minutes - int(minutes)) * 60
                             self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
                                     datetime.datetime(1970, 1, 1)).total_seconds()
                             if self.localtime:
                                 self.tmin += time.timezone
                             if self.xmin is not None and self.xmax is not None:
                                 self.xrange = self.xmax - self.xmin
                     if self.throttle == 0:
                         self.__plot()
                     else:
                         self.__throttle_plot(self.__plot)#, coerce=coerce)
                 def close(self):
                     if self.data and not self.data.flagNoData:
                         self.save_time = self.data.max_time
                         self.__plot()
                     if self.data and not self.data.flagNoData and self.pause:
                         figpause(10)

schainpy/model/graphics/jroplot_parameters.py +4 -5

             import os
             import datetime
             import numpy
             from schainpy.model.graphics.jroplot_base import Plot, plt
             from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot, SpectraCutPlot
             from schainpy.utils import log
             EARTH_RADIUS = 6.3710e3
             def ll2xy(lat1, lon1, lat2, lon2):
                 p = 0.017453292519943295
                 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
                     numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
                 r = 12742 * numpy.arcsin(numpy.sqrt(a))
                 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
                                       * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
                 theta = -theta + numpy.pi/2
                 return r*numpy.cos(theta), r*numpy.sin(theta)
             def km2deg(km):
                 '''
                 Convert distance in km to degrees
                 '''
                 return numpy.rad2deg(km/EARTH_RADIUS)
             class SpectralMomentsPlot(SpectraPlot):
                 '''
                 Plot for Spectral Moments
                 '''
                 CODE = 'spc_moments'
                 # colormap = 'jet'
                 # plot_type = 'pcolor'
             class DobleGaussianPlot(SpectraPlot):
                 '''
                 Plot for Double Gaussian Plot
                 '''
                 CODE = 'gaussian_fit'
                 # colormap = 'jet'
                 # plot_type = 'pcolor'
             class DoubleGaussianSpectraCutPlot(SpectraCutPlot):
                 '''
                 Plot SpectraCut with Double Gaussian Fit
                 '''
                 CODE = 'cut_gaussian_fit'
             class SnrPlot(RTIPlot):
                 '''
                 Plot for SNR Data
                 '''
                 CODE = 'snr'
                 colormap = 'jet'
                 def update(self, dataOut):
                     data = {
                         'snr': 10*numpy.log10(dataOut.data_snr)
                     }
                     return data, {}
             class DopplerPlot(RTIPlot):
                 '''
                 Plot for DOPPLER Data (1st moment)
                 '''
                 CODE = 'dop'
                 colormap = 'jet'
                 def update(self, dataOut):
                     data = {
                         'dop': 10*numpy.log10(dataOut.data_dop)
                     }
                     return data, {}
             class PowerPlot(RTIPlot):
                 '''
                 Plot for Power Data (0 moment)
                 '''
                 CODE = 'pow'
                 colormap = 'jet'
                 def update(self, dataOut):
                     data = {
                         'pow': 10*numpy.log10(dataOut.data_pow/dataOut.normFactor)
                     }
                     return data, {}
             class SpectralWidthPlot(RTIPlot):
                 '''
                 Plot for Spectral Width Data (2nd moment)
                 '''
                 CODE = 'width'
                 colormap = 'jet'
                 def update(self, dataOut):
                     data = {
                         'width': dataOut.data_width
                     }
                     return data, {}
             class SkyMapPlot(Plot):
                 '''
                 Plot for meteors detection data
                 '''
                 CODE = 'param'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = 1
                     self.width = 7.2
                     self.height = 7.2
                     self.nplots = 1
                     self.xlabel = 'Zonal Zenith Angle (deg)'
                     self.ylabel = 'Meridional Zenith Angle (deg)'
                     self.polar = True
                     self.ymin = -180
                     self.ymax = 180
                     self.colorbar = False
                 def plot(self):
                     arrayParameters = numpy.concatenate(self.data['param'])
                     error = arrayParameters[:, -1]
                     indValid = numpy.where(error == 0)[0]
                     finalMeteor = arrayParameters[indValid, :]
                     finalAzimuth = finalMeteor[:, 3]
                     finalZenith = finalMeteor[:, 4]
                     x = finalAzimuth * numpy.pi / 180
                     y = finalZenith
                     ax = self.axes[0]
                     if ax.firsttime:
                         ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
                     else:
                         ax.plot.set_data(x, y)
                     dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
                     dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
                     title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
                                                                                                  dt2,
                                                                                                  len(x))
                     self.titles[0] = title
             class GenericRTIPlot(Plot):
                 '''
                 Plot for data_xxxx object
                 '''
                 CODE = 'param'
                 colormap = 'viridis'
                 plot_type = 'pcolorbuffer'
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
-                    self.nrows = self.data.shape(self.attr_data)[0]
+                    self.nrows = self.data.shape('param')[0]
                     self.nplots = self.nrows
                     self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
                     if not self.xlabel:
                         self.xlabel = 'Time'
                     self.ylabel = 'Range [km]'
                     if not self.titles:
-                        self.titles = self.data.parameters \
+                        self.titles = ['Param {}'.format(x) for x in range(self.nrows)]
-                            if self.data.parameters else ['Param {}'.format(x) for x in range(self.nrows)]
                 def update(self, dataOut):
                     data = {
-                        self.attr_data : getattr(dataOut, self.attr_data)
+                        'param' : numpy.concatenate([getattr(dataOut, attr) for attr in self.attr_data], axis=0)
                     }
                     meta = {}
                     return data, meta
                 def plot(self):
                     # self.data.normalize_heights()
                     self.x = self.data.times
                     self.y = self.data.yrange
-                    self.z = self.data[self.attr_data]
+                    self.z = self.data['param']
                     self.z = numpy.ma.masked_invalid(self.z)
                     if self.decimation is None:
                         x, y, z = self.fill_gaps(self.x, self.y, self.z)
                     else:
                         x, y, z = self.fill_gaps(*self.decimate())
                     for n, ax in enumerate(self.axes):
                         self.zmax = self.zmax if self.zmax is not None else numpy.max(
                             self.z[n])
                         self.zmin = self.zmin if self.zmin is not None else numpy.min(
                             self.z[n])
                         if ax.firsttime:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=self.cmaps[n]
                                                    )
                         else:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.collections.remove(ax.collections[0])
                             ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=self.cmaps[n]
                                                    )
             class PolarMapPlot(Plot):
                 '''
                 Plot for weather radar
                 '''
                 CODE = 'param'
                 colormap = 'seismic'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = 1
                     self.width = 9
                     self.height = 8
                     self.mode = self.data.meta['mode']
                     if self.channels is not None:
                         self.nplots = len(self.channels)
                         self.nrows = len(self.channels)
                     else:
                         self.nplots = self.data.shape(self.CODE)[0]
                         self.nrows = self.nplots
                         self.channels = list(range(self.nplots))
                     if self.mode == 'E':
                         self.xlabel = 'Longitude'
                         self.ylabel = 'Latitude'
                     else:
                         self.xlabel = 'Range (km)'
                         self.ylabel = 'Height (km)'
                     self.bgcolor = 'white'
                     self.cb_labels = self.data.meta['units']
                     self.lat = self.data.meta['latitude']
                     self.lon = self.data.meta['longitude']
                     self.xmin, self.xmax = float(
                         km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
                     self.ymin, self.ymax = float(
                         km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
                     #  self.polar = True
                 def plot(self):
                     for n, ax in enumerate(self.axes):
                         data = self.data['param'][self.channels[n]]
                         zeniths = numpy.linspace(
 , self.data.meta['max_range'], data.shape[1])
                         if self.mode == 'E':
                             azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
                             r, theta = numpy.meshgrid(zeniths, azimuths)
                             x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
                                 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
                             x = km2deg(x) + self.lon
                             y = km2deg(y) + self.lat
                         else:
                             azimuths = numpy.radians(self.data.yrange)
                             r, theta = numpy.meshgrid(zeniths, azimuths)
                             x, y = r*numpy.cos(theta), r*numpy.sin(theta)
                         self.y = zeniths
                         if ax.firsttime:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.plt = ax.pcolormesh(  # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
                                 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
                                 vmin=self.zmin,
                                 vmax=self.zmax,
                                 cmap=self.cmaps[n])
                         else:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.collections.remove(ax.collections[0])
                             ax.plt = ax.pcolormesh(  # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
                                 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
                                 vmin=self.zmin,
                                 vmax=self.zmax,
                                 cmap=self.cmaps[n])
                         if self.mode == 'A':
                             continue
                         # plot district names
                         f = open('/data/workspace/schain_scripts/distrito.csv')
                         for line in f:
                             label, lon, lat = [s.strip() for s in line.split(',') if s]
                             lat = float(lat)
                             lon = float(lon)
                             # ax.plot(lon, lat, '.b', ms=2)
                             ax.text(lon, lat, label.decode('utf8'), ha='center',
                                     va='bottom', size='8', color='black')
                         # plot limites
                         limites = []
                         tmp = []
                         for line in open('/data/workspace/schain_scripts/lima.csv'):
                             if '#' in line:
                                 if tmp:
                                     limites.append(tmp)
                                 tmp = []
                                 continue
                             values = line.strip().split(',')
                             tmp.append((float(values[0]), float(values[1])))
                         for points in limites:
                             ax.add_patch(
                                 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
                         # plot Cuencas
                         for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
                             f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
                             values = [line.strip().split(',') for line in f]
                             points = [(float(s[0]), float(s[1])) for s in values]
                             ax.add_patch(Polygon(points, ec='b', fc='none'))
                         # plot grid
                         for r in (15, 30, 45, 60):
                             ax.add_artist(plt.Circle((self.lon, self.lat),
                                                      km2deg(r), color='0.6', fill=False, lw=0.2))
                             ax.text(
                                 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
                                 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
                                 '{}km'.format(r),
                                 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
                     if self.mode == 'E':
                         title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
                         label = 'E{:02d}'.format(int(self.data.meta['elevation']))
                     else:
                         title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
                         label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
                     self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
                     self.titles = ['{} {}'.format(
                         self.data.parameters[x], title) for x in self.channels]

schainpy/model/io/bltrIO_spectra.py +2 -11

             import os
             import sys
             import glob
-            import fnmatch
-            import datetime
-            import time
-            import re
-            import h5py
             import numpy
-            import pylab as plb
-            from scipy.optimize import curve_fit
-            from scipy import asarray as ar, exp
             SPEED_OF_LIGHT = 299792458
             SPEED_OF_LIGHT = 3e8
             from .utils import folder_in_range
             import schainpy.admin
             from schainpy.model.data.jrodata import Spectra
-            from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
+            from schainpy.model.proc.jroproc_base import ProcessingUnit
             from schainpy.utils import log
-            from schainpy.model.io.jroIO_base import JRODataReader
             def pol2cart(rho, phi):
                 x = rho * numpy.cos(phi)
                 y = rho * numpy.sin(phi)
                 return(x, y)
             FILE_STRUCTURE = numpy.dtype([  # HEADER 48bytes
                 ('FileMgcNumber', '<u4'),  # 0x23020100
                 ('nFDTdataRecors', '<u4'),
                 ('OffsetStartHeader', '<u4'),
                 ('RadarUnitId', '<u4'),
                 ('SiteName', 'S32'),  # Null terminated
             ])
             class FileHeaderBLTR():
                 def __init__(self, fo):
                     self.fo = fo
                     self.size = 48
                     self.read()
                 def read(self):
                     header = numpy.fromfile(self.fo, FILE_STRUCTURE, 1)
                     self.FileMgcNumber = hex(header['FileMgcNumber'][0])
                     self.nFDTdataRecors = int(header['nFDTdataRecors'][0])
                     self.RadarUnitId = int(header['RadarUnitId'][0])
                     self.OffsetStartHeader = int(header['OffsetStartHeader'][0])
                     self.SiteName = header['SiteName'][0]
                 def write(self, fp):
                     headerTuple = (self.FileMgcNumber,
                                    self.nFDTdataRecors,
                                    self.RadarUnitId,
                                    self.SiteName,
                                    self.size)
                     header = numpy.array(headerTuple, FILE_STRUCTURE)
                     header.tofile(fp)
                     ''' ndarray.tofile(fid, sep, format)    Write array to a file as text or binary (default).
                     fid : file or str
                     An open file object, or a string containing a filename.
                     sep : str
                     Separator between array items for text output. If "" (empty), a binary file is written,
                     equivalent to file.write(a.tobytes()).
                     format : str
                     Format string for text file output. Each entry in the array is formatted to text by
                     first converting it to the closest Python type, and then using "format" % item.
                     '''
                     return 1
             RECORD_STRUCTURE = numpy.dtype([  # RECORD HEADER 180+20N bytes
                 ('RecMgcNumber', '<u4'),  # 0x23030001
                 ('RecCounter', '<u4'),  # Record counter(0,1, ...)
                 # Offset to start of next record form start of this record
                                         ('Off2StartNxtRec', '<u4'),
                                         # Offset to start of data from start of this record
                                         ('Off2StartData', '<u4'),
                                         # Epoch time stamp of start of acquisition (seconds)
                                         ('nUtime', '<i4'),
                                         # Millisecond component of time stamp (0,...,999)
                                         ('nMilisec', '<u4'),
                                         # Experiment tag name (null terminated)
                                         ('ExpTagName', 'S32'),
                                         # Experiment comment (null terminated)
                                         ('ExpComment', 'S32'),
                                         # Site latitude (from GPS) in degrees (positive implies North)
                                         ('SiteLatDegrees', '<f4'),
                                         # Site longitude (from GPS) in degrees (positive implies East)
                                         ('SiteLongDegrees', '<f4'),
                                         # RTC GPS engine status (0=SEEK, 1=LOCK, 2=NOT FITTED, 3=UNAVAILABLE)
                                         ('RTCgpsStatus', '<u4'),
                                         ('TransmitFrec', '<u4'),  # Transmit frequency (Hz)
                                         ('ReceiveFrec', '<u4'),  # Receive frequency
                                         # First local oscillator frequency (Hz)
                                         ('FirstOsciFrec', '<u4'),
                                         # (0="O", 1="E", 2="linear 1", 3="linear2")
                                         ('Polarisation', '<u4'),
                                         # Receiver filter settings (0,1,2,3)
                                         ('ReceiverFiltSett', '<u4'),
                                         # Number of modes in use (1 or 2)
                                         ('nModesInUse', '<u4'),
                                         # Dual Mode index number for these data (0 or 1)
                                         ('DualModeIndex', '<u4'),
                                         # Dual Mode range correction for these data (m)
                                         ('DualModeRange', '<u4'),
                                         # Number of digital channels acquired (2*N)
                                         ('nDigChannels', '<u4'),
                                         # Sampling resolution (meters)
                                         ('SampResolution', '<u4'),
                                         # Number of range gates sampled
                                         ('nHeights', '<u4'),
                                         # Start range of sampling (meters)
                                         ('StartRangeSamp', '<u4'),
                                         ('PRFhz', '<u4'),  # PRF (Hz)
                                         ('nCohInt', '<u4'),  # Integrations
                                         # Number of data points transformed
                                         ('nProfiles', '<u4'),
                                         # Number of receive beams stored in file (1 or N)
                                         ('nChannels', '<u4'),
                                         ('nIncohInt', '<u4'),  # Number of spectral averages
                                         # FFT windowing index (0 = no window)
                                         ('FFTwindowingInd', '<u4'),
                                         # Beam steer angle (azimuth) in degrees (clockwise from true North)
                                         ('BeamAngleAzim', '<f4'),
                                         # Beam steer angle (zenith) in degrees (0=> vertical)
                                         ('BeamAngleZen', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaCoord0', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaAngl0', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaCoord1', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaAngl1', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaCoord2', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaAngl2', '<f4'),
                                         # Receiver phase calibration (degrees) - N values
                                         ('RecPhaseCalibr0', '<f4'),
                                         # Receiver phase calibration (degrees) - N values
                                         ('RecPhaseCalibr1', '<f4'),
                                         # Receiver phase calibration (degrees) - N values
                                         ('RecPhaseCalibr2', '<f4'),
                                         # Receiver amplitude calibration (ratio relative to receiver one) - N values
                                         ('RecAmpCalibr0', '<f4'),
                                         # Receiver amplitude calibration (ratio relative to receiver one) - N values
                                         ('RecAmpCalibr1', '<f4'),
                                         # Receiver amplitude calibration (ratio relative to receiver one) - N values
                                         ('RecAmpCalibr2', '<f4'),
                                         # Receiver gains in dB - N values
                                         ('ReceiverGaindB0', '<i4'),
                                         # Receiver gains in dB - N values
                                         ('ReceiverGaindB1', '<i4'),
                                         # Receiver gains in dB - N values
                                         ('ReceiverGaindB2', '<i4'),
             ])
             class RecordHeaderBLTR():
                 def __init__(self, fo):
                     self.fo = fo
                     self.OffsetStartHeader = 48
                     self.Off2StartNxtRec = 811248
                 def read(self, block):
                     OffRHeader = self.OffsetStartHeader + block * self.Off2StartNxtRec
                     self.fo.seek(OffRHeader, os.SEEK_SET)
                     header = numpy.fromfile(self.fo, RECORD_STRUCTURE, 1)
                     self.RecMgcNumber = hex(header['RecMgcNumber'][0])  # 0x23030001
                     self.RecCounter = int(header['RecCounter'][0])
                     self.Off2StartNxtRec = int(header['Off2StartNxtRec'][0])
                     self.Off2StartData = int(header['Off2StartData'][0])
                     self.nUtime = header['nUtime'][0]
                     self.nMilisec = header['nMilisec'][0]
                     self.ExpTagName = '' # str(header['ExpTagName'][0])
                     self.ExpComment = '' # str(header['ExpComment'][0])
                     self.SiteLatDegrees = header['SiteLatDegrees'][0]
                     self.SiteLongDegrees = header['SiteLongDegrees'][0]
                     self.RTCgpsStatus = header['RTCgpsStatus'][0]
                     self.TransmitFrec = header['TransmitFrec'][0]
                     self.ReceiveFrec = header['ReceiveFrec'][0]
                     self.FirstOsciFrec = header['FirstOsciFrec'][0]
                     self.Polarisation = header['Polarisation'][0]
                     self.ReceiverFiltSett = header['ReceiverFiltSett'][0]
                     self.nModesInUse = header['nModesInUse'][0]
                     self.DualModeIndex = header['DualModeIndex'][0]
                     self.DualModeRange = header['DualModeRange'][0]
                     self.nDigChannels = header['nDigChannels'][0]
                     self.SampResolution = header['SampResolution'][0]
                     self.nHeights = header['nHeights'][0]
                     self.StartRangeSamp = header['StartRangeSamp'][0]
                     self.PRFhz = header['PRFhz'][0]
                     self.nCohInt = header['nCohInt'][0]
                     self.nProfiles = header['nProfiles'][0]
                     self.nChannels = header['nChannels'][0]
                     self.nIncohInt = header['nIncohInt'][0]
                     self.FFTwindowingInd = header['FFTwindowingInd'][0]
                     self.BeamAngleAzim = header['BeamAngleAzim'][0]
                     self.BeamAngleZen = header['BeamAngleZen'][0]
                     self.AntennaCoord0 = header['AntennaCoord0'][0]
                     self.AntennaAngl0 = header['AntennaAngl0'][0]
                     self.AntennaCoord1 = header['AntennaCoord1'][0]
                     self.AntennaAngl1 = header['AntennaAngl1'][0]
                     self.AntennaCoord2 = header['AntennaCoord2'][0]
                     self.AntennaAngl2 = header['AntennaAngl2'][0]
                     self.RecPhaseCalibr0 = header['RecPhaseCalibr0'][0]
                     self.RecPhaseCalibr1 = header['RecPhaseCalibr1'][0]
                     self.RecPhaseCalibr2 = header['RecPhaseCalibr2'][0]
                     self.RecAmpCalibr0 = header['RecAmpCalibr0'][0]
                     self.RecAmpCalibr1 = header['RecAmpCalibr1'][0]
                     self.RecAmpCalibr2 = header['RecAmpCalibr2'][0]
                     self.ReceiverGaindB0 = header['ReceiverGaindB0'][0]
                     self.ReceiverGaindB1 = header['ReceiverGaindB1'][0]
                     self.ReceiverGaindB2 = header['ReceiverGaindB2'][0]
                     self.ipp = 0.5 * (SPEED_OF_LIGHT / self.PRFhz)
                     self.RHsize = 180 + 20 * self.nChannels
                     self.Datasize = self.nProfiles * self.nChannels * self.nHeights * 2 * 4
                     endFp = self.OffsetStartHeader + self.RecCounter * self.Off2StartNxtRec
                     if OffRHeader > endFp:
                         sys.stderr.write(
                             "Warning %s: Size value read from System Header is lower than it has to be\n" % fp)
                         return 0
                     if OffRHeader < endFp:
                         sys.stderr.write(
                             "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp)
                         return 0
                     return 1
             class BLTRSpectraReader (ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.ext = ".fdt"
                     self.optchar = "P"
                     self.fpFile = None
                     self.fp = None
                     self.BlockCounter = 0
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.fileSelector = 0
                     self.Off2StartNxtRec = 0
                     self.RecCounter = 0
                     self.flagNoMoreFiles = 0
                     self.data_spc = None
                     self.data_cspc = None
                     self.path = None
                     self.OffsetStartHeader = 0
                     self.Off2StartData = 0
                     self.ipp = 0
                     self.nFDTdataRecors = 0
                     self.blocksize = 0
                     self.dataOut = Spectra()
                     self.dataOut.flagNoData = False
                 def search_files(self):
                     '''
                     Function that indicates the number of .fdt files that exist in the folder to be read.
                     It also creates an organized list with the names of the files to read.
                     '''
                     files = glob.glob(os.path.join(self.path, '*{}'.format(self.ext)))
                     files = sorted(files)
                     for f in files:
                         filename = f.split('/')[-1]
                         if folder_in_range(filename.split('.')[1], self.startDate, self.endDate, '%Y%m%d'):
                             self.filenameList.append(f)
                 def run(self, **kwargs):
                     '''
                     This method will be the one that will initiate the data entry, will be called constantly.
                     You should first verify that your Setup () is set up and then continue to acquire
                     the data to be processed with getData ().
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
                 def setup(self,
                           path=None,
                           startDate=None,
                           endDate=None,
                           startTime=None,
                           endTime=None,
                           walk=True,
                           code=None,
                           online=False,
                           mode=None,
                           **kwargs):
                     self.isConfig = True
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.walk = walk
                     self.mode = int(mode)
                     self.search_files()
                     if self.filenameList:
                         self.readFile()
                 def getData(self):
                     '''
                     Before starting this function, you should check that there is still an unread file,
                     If there are still blocks to read or if the data block is empty.
                     You should call the file "read".
                     '''
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         raise schainpy.admin.SchainError('No more files')
                     self.readBlock()
                 def readFile(self):
                     '''
                     You must indicate if you are reading in Online or Offline mode and load the
                     The parameters for this file reading mode.
                     Then you must do 2 actions:
 . Get the BLTR FileHeader.
 . Start reading the first block.
                     '''
                     if self.fileSelector < len(self.filenameList):
                         log.success('Opening file: {}'.format(self.filenameList[self.fileSelector]), self.name)
                         self.fp = open(self.filenameList[self.fileSelector])
                         self.fheader = FileHeaderBLTR(self.fp)
                         self.rheader = RecordHeaderBLTR(self.fp)
                         self.nFDTdataRecors = self.fheader.nFDTdataRecors
                         self.fileSelector += 1
                         self.BlockCounter = 0
                         return 1
                     else:
                         self.flagNoMoreFiles = True
                         self.dataOut.flagNoData = True
                         return 0
                 def readBlock(self):
                     '''
                     It should be checked if the block has data, if it is not passed to the next file.
                     Then the following is done:
 . Read the RecordHeader
 . Fill the buffer with the current block number.
                     '''
                     if self.BlockCounter == self.nFDTdataRecors:
                         if not self.readFile():
                             return
                     if self.mode == 1:
                         self.rheader.read(self.BlockCounter+1)
                     elif self.mode == 0:
                         self.rheader.read(self.BlockCounter)
                     self.RecCounter = self.rheader.RecCounter
                     self.OffsetStartHeader = self.rheader.OffsetStartHeader
                     self.Off2StartNxtRec = self.rheader.Off2StartNxtRec
                     self.Off2StartData = self.rheader.Off2StartData
                     self.nProfiles = self.rheader.nProfiles
                     self.nChannels = self.rheader.nChannels
                     self.nHeights = self.rheader.nHeights
                     self.frequency = self.rheader.TransmitFrec
                     self.DualModeIndex = self.rheader.DualModeIndex
                     self.pairsList = [(0, 1), (0, 2), (1, 2)]
                     self.dataOut.pairsList = self.pairsList
                     self.nRdPairs = len(self.dataOut.pairsList)
                     self.dataOut.nRdPairs = self.nRdPairs
                     self.dataOut.heightList = (self.rheader.StartRangeSamp + numpy.arange(self.nHeights) * self.rheader.SampResolution) / 1000.
                     self.dataOut.channelList = range(self.nChannels)
                     self.dataOut.nProfiles=self.rheader.nProfiles
                     self.dataOut.nIncohInt=self.rheader.nIncohInt
                     self.dataOut.nCohInt=self.rheader.nCohInt
                     self.dataOut.ippSeconds= 1/float(self.rheader.PRFhz)
                     self.dataOut.PRF=self.rheader.PRFhz
                     self.dataOut.nFFTPoints=self.rheader.nProfiles
                     self.dataOut.utctime = self.rheader.nUtime + self.rheader.nMilisec/1000.
                     self.dataOut.timeZone = 0
                     self.dataOut.useLocalTime = False
                     self.dataOut.nmodes = 2
                     log.log('Reading block {} - {}'.format(self.BlockCounter, self.dataOut.datatime), self.name)
                     OffDATA = self.OffsetStartHeader + self.RecCounter * \
                         self.Off2StartNxtRec + self.Off2StartData
                     self.fp.seek(OffDATA, os.SEEK_SET)
                     self.data_fft = numpy.fromfile(self.fp, [('complex','<c8')], self.nProfiles*self.nChannels*self.nHeights )
                     self.data_fft = self.data_fft.astype(numpy.dtype('complex'))
                     self.data_block = numpy.reshape(self.data_fft,(self.nHeights, self.nChannels, self.nProfiles))
                     self.data_block = numpy.transpose(self.data_block, (1,2,0))
                     copy = self.data_block.copy()
                     spc = copy * numpy.conjugate(copy)
                     self.data_spc = numpy.absolute(spc)  # valor absoluto o magnitud
-                    self.dataOut.data_spc = self.data_spc
                     cspc = self.data_block.copy()
                     self.data_cspc = self.data_block.copy()
                     for i in range(self.nRdPairs):
                         chan_index0 = self.dataOut.pairsList[i][0]
                         chan_index1 = self.dataOut.pairsList[i][1]
                         self.data_cspc[i, :, :] = cspc[chan_index0, :, :] * numpy.conjugate(cspc[chan_index1, :, :])
                     '''Getting Eij and Nij'''
                     (AntennaX0, AntennaY0) = pol2cart(
                         self.rheader.AntennaCoord0, self.rheader.AntennaAngl0 * numpy.pi / 180)
                     (AntennaX1, AntennaY1) = pol2cart(
                         self.rheader.AntennaCoord1, self.rheader.AntennaAngl1 * numpy.pi / 180)
                     (AntennaX2, AntennaY2) = pol2cart(
                         self.rheader.AntennaCoord2, self.rheader.AntennaAngl2 * numpy.pi / 180)
                     E01 = AntennaX0 - AntennaX1
                     N01 = AntennaY0 - AntennaY1
                     E02 = AntennaX0 - AntennaX2
                     N02 = AntennaY0 - AntennaY2
                     E12 = AntennaX1 - AntennaX2
                     N12 = AntennaY1 - AntennaY2
                     self.ChanDist = numpy.array(
                         [[E01, N01], [E02, N02], [E12, N12]])
                     self.dataOut.ChanDist = self.ChanDist
                     self.BlockCounter += 2
                     self.dataOut.data_spc = self.data_spc
                     self.dataOut.data_cspc =self.data_cspc

schainpy/model/io/jroIO_param.py +7 -7

             import os
             import time
             import datetime
             import numpy
             import h5py
             import schainpy.admin
             from schainpy.model.data.jrodata import *
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.io.jroIO_base import *
             from schainpy.utils import log
             class HDFReader(Reader, ProcessingUnit):
                 """Processing unit to read HDF5 format files
                 This unit reads HDF5 files created with `HDFWriter` operation contains
                 by default two groups Data and Metadata all variables would be saved as `dataOut`
                 attributes.
                 It is possible to read any HDF5 file by given the structure in the `description`
                 parameter, also you can add extra values to metadata with the parameter `extras`.
                 Parameters:
                 -----------
                 path : str
                     Path where files are located.
                 startDate : date
                     Start date of the files
                 endDate : list
                     End date of the files
                 startTime : time
                     Start time of the files
                 endTime : time
                     End time of the files
                 description : dict, optional
                     Dictionary with the description of the HDF5 file
                 extras : dict, optional
                     Dictionary with extra metadata to be be added to `dataOut`
                 Examples
                 --------
                 desc = {
                     'Data': {
                         'data_output': ['u', 'v', 'w'],
                         'utctime': 'timestamps',
                     }  ,
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 desc = {
                     'Data': {
                         'data_output': 'winds',
                         'utctime': 'timestamps'
                     },
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 extras = {
                     'timeZone': 300
                 }
                 reader = project.addReadUnit(
                     name='HDFReader',
                     path='/path/to/files',
                     startDate='2019/01/01',
                     endDate='2019/01/31',
                     startTime='00:00:00',
                     endTime='23:59:59',
                     # description=json.dumps(desc),
                     # extras=json.dumps(extras),
                     )
                 """
                 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                     self.ext = ".hdf5"
                     self.optchar = "D"
                     self.meta = {}
                     self.data = {}
                     self.open_file = h5py.File
                     self.open_mode = 'r'
                     self.description = {}
                     self.extras = {}
                     self.filefmt = "*%Y%j***"
                     self.folderfmt = "*%Y%j"
                 def setup(self, **kwargs):
                     self.set_kwargs(**kwargs)
                     if not self.ext.startswith('.'):
                         self.ext = '.{}'.format(self.ext)
                     if self.online:
                         log.log("Searching files in online mode...", self.name)
                         for nTries in range(self.nTries):
                             fullpath = self.searchFilesOnLine(self.path, self.startDate,
                                 self.endDate, self.expLabel, self.ext, self.walk,
                                 self.filefmt, self.folderfmt)
                             try:
                                 fullpath = next(fullpath)
                             except:
                                 fullpath = None
                             if fullpath:
                                 break
                             log.warning(
                                 'Waiting {} sec for a valid file in {}: try {} ...'.format(
                                     self.delay, self.path, nTries + 1),
                                 self.name)
                             time.sleep(self.delay)
                         if not(fullpath):
                             raise schainpy.admin.SchainError(
                                 'There isn\'t any valid file in {}'.format(self.path))
                         pathname, filename = os.path.split(fullpath)
                         self.year = int(filename[1:5])
                         self.doy = int(filename[5:8])
                         self.set = int(filename[8:11]) - 1
                     else:
                         log.log("Searching files in {}".format(self.path), self.name)
                         self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
                             self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
                     self.setNextFile()
                     return
                 def readFirstHeader(self):
                     '''Read metadata and data'''
                     self.__readMetadata()
                     self.__readData()
                     self.__setBlockList()
                     if 'type' in self.meta:
                         self.dataOut = eval(self.meta['type'])()
                     for attr in self.meta:
                         setattr(self.dataOut, attr, self.meta[attr])
                     self.blockIndex = 0
                     return
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     startTime = self.startTime
                     endTime = self.endTime
                     thisUtcTime = self.data['utctime']
                     self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
                     thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     '''
                     meta = {}
                     if self.description:
                         for key, value in self.description['Metadata'].items():
-                            meta[key] = self.fp[value].value
+                            meta[key] = self.fp[value][()]
                     else:
                         grp = self.fp['Metadata']
                         for name in grp:
-                            meta[name] = grp[name].value
+                            meta[name] = grp[name][()]
                     if self.extras:
                         for key, value in self.extras.items():
                             meta[key] = value
                     self.meta = meta
                     return
                 def __readData(self):
                     data = {}
                     if self.description:
                         for key, value in self.description['Data'].items():
                             if isinstance(value, str):
                                 if isinstance(self.fp[value], h5py.Dataset):
-                                    data[key] = self.fp[value].value
+                                    data[key] = self.fp[value][()]
                                 elif isinstance(self.fp[value], h5py.Group):
                                     array = []
                                     for ch in self.fp[value]:
-                                        array.append(self.fp[value][ch].value)
+                                        array.append(self.fp[value][ch][()])
                                     data[key] = numpy.array(array)
                             elif isinstance(value, list):
                                 array = []
                                 for ch in value:
-                                    array.append(self.fp[ch].value)
+                                    array.append(self.fp[ch][()])
                                 data[key] = numpy.array(array)
                     else:
                         grp = self.fp['Data']
                         for name in grp:
                             if isinstance(grp[name], h5py.Dataset):
-                                array = grp[name].value
+                                array = grp[name][()]
                             elif isinstance(grp[name], h5py.Group):
                                 array = []
                                 for ch in grp[name]:
-                                    array.append(grp[name][ch].value)
+                                    array.append(grp[name][ch][()])
                                 array = numpy.array(array)
                             else:
                                 log.warning('Unknown type: {}'.format(name))
                             if name in self.description:
                                 key = self.description[name]
                             else:
                                 key = name
                             data[key] = array
                     self.data = data
                     return
                 def getData(self):
                     for attr in self.data:
                         if self.data[attr].ndim == 1:
                             setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
                         else:
                             setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     log.log("Block No. {}/{} -> {}".format(
                         self.blockIndex,
                         self.blocksPerFile,
                         self.dataOut.datatime.ctime()), self.name)
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     if self.blockIndex == self.blocksPerFile:
                         self.setNextFile()
                     self.getData()
                     return
             @MPDecorator
             class HDFWriter(Operation):
                 """Operation to write HDF5 files.
                 The HDF5 file contains by default two groups Data and Metadata where
                 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
                 parameters, data attributes are normaly time dependent where the metadata
                 are not.
                 It is possible to customize the structure of the HDF5 file with the
                 optional description parameter see the examples.
                 Parameters:
                 -----------
                 path : str
                     Path where files will be saved.
                 blocksPerFile : int
                     Number of blocks per file
                 metadataList : list
                     List of the dataOut attributes that will be saved as metadata
                 dataList : int
                     List of the dataOut attributes that will be saved as data
                 setType : bool
                     If True the name of the files corresponds to the timestamp of the data
                 description : dict, optional
                     Dictionary with the desired description of the HDF5 file
                 Examples
                 --------
                 desc = {
                     'data_output': {'winds': ['z', 'w', 'v']},
                     'utctime': 'timestamps',
                     'heightList': 'heights'
                 }
                 desc = {
                     'data_output': ['z', 'w', 'v'],
                     'utctime': 'timestamps',
                     'heightList': 'heights'
                 }
                 desc = {
                     'Data': {
                         'data_output': 'winds',
                         'utctime': 'timestamps'
                     },
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 writer = proc_unit.addOperation(name='HDFWriter')
                 writer.addParameter(name='path', value='/path/to/file')
                 writer.addParameter(name='blocksPerFile', value='32')
                 writer.addParameter(name='metadataList', value='heightList,timeZone')
                 writer.addParameter(name='dataList',value='data_output,utctime')
                 # writer.addParameter(name='description',value=json.dumps(desc))
                 """
                 ext = ".hdf5"
                 optchar = "D"
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 currentDay = None
                 lastTime = None
                 def __init__(self):
                     Operation.__init__(self)
                     return
                 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = [s.strip() for s in dataList]
                     self.setType = setType
                     self.description = description
                     if self.metadataList is None:
                         self.metadataList = self.dataOut.metadata_list
                     tableList = []
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         if hasattr(self.dataOut, self.dataList[i]):
                             dataAux = getattr(self.dataOut, self.dataList[i])
                             dsDict['variable'] = self.dataList[i]
                         else:
                             log.warning('Attribute {} not found in dataOut', self.name)
                             continue
                         if dataAux is None:
                             continue
                         elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                             dsDict['nDim'] = 0
                         else:
                             dsDict['nDim'] = len(dataAux.shape)
                             dsDict['shape'] = dataAux.shape
                             dsDict['dsNumber'] = dataAux.shape[0]
                             dsDict['dtype'] = dataAux.dtype
                         dsList.append(dsDict)
                     self.dsList = dsList
                     self.currentDay = self.dataOut.datatime.date()
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     timeTuple = time.localtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     if self.lastTime is None:
                         self.lastTime = currentTime
                         self.currentDay = dataDay
                         return False
                     timeDiff = currentTime - self.lastTime
                     #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     if dataDay != self.currentDay:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > 3*60*60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
                         dataList=[], setType=None, description={}):
                     self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList,
                                    setType=setType, description=description)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     if os.path.exists(fullpath):
                         filesList = os.listdir(fullpath)
                         filesList = [k for k in filesList if k.startswith(self.optchar)]
                         if len( filesList ) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     self.filename = os.path.join( path, subfolder, file )
                     #Setting HDF5 File
                     self.fp = h5py.File(self.filename, 'w')
                     #write metadata
                     self.writeMetadata(self.fp)
                     #Write data
                     self.writeData(self.fp)
                 def getLabel(self, name, x=None):
                     if x is None:
                         if 'Data' in self.description:
                             data = self.description['Data']
                             if 'Metadata' in self.description:
                                 data.update(self.description['Metadata'])
                         else:
                             data = self.description
                         if name in data:
                             if isinstance(data[name], str):
                                 return data[name]
                             elif isinstance(data[name], list):
                                 return None
                             elif isinstance(data[name], dict):
                                 for key, value in data[name].items():
                                     return key
                         return name
                     else:
                         if 'Metadata' in self.description:
                             meta = self.description['Metadata']
                         else:
                             meta = self.description
                         if name in meta:
                             if isinstance(meta[name], list):
                                 return meta[name][x]
                             elif isinstance(meta[name], dict):
                                 for key, value in meta[name].items():
                                     return value[x]
                         if 'cspc' in name:
                             return 'pair{:02d}'.format(x)
                         else:
                             return 'channel{:02d}'.format(x)
                 def writeMetadata(self, fp):
                     if self.description:
                         if 'Metadata' in self.description:
                             grp = fp.create_group('Metadata')
                         else:
                             grp = fp
                     else:
                         grp = fp.create_group('Metadata')
                     for i in range(len(self.metadataList)):
                         if not hasattr(self.dataOut, self.metadataList[i]):
                             log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
                             continue
                         value = getattr(self.dataOut, self.metadataList[i])
                         if isinstance(value, bool):
                             if value is True:
                                 value = 1
                             else:
                                 value = 0
                         grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
                     return
                 def writeData(self, fp):
                     if self.description:
                         if 'Data' in self.description:
                             grp = fp.create_group('Data')
                         else:
                             grp = fp
                     else:
                         grp = fp.create_group('Data')
                     dtsets = []
                     data = []
                     for dsInfo in self.dsList:
                         if dsInfo['nDim'] == 0:
                             ds = grp.create_dataset(
                                 self.getLabel(dsInfo['variable']),
                                 (self.blocksPerFile, ),
                                 chunks=True,
                                 dtype=numpy.float64)
                             dtsets.append(ds)
                             data.append((dsInfo['variable'], -1))
                         else:
                             label = self.getLabel(dsInfo['variable'])
                             if label is not None:
                                 sgrp = grp.create_group(label)
                             else:
                                 sgrp = grp
                             for i in range(dsInfo['dsNumber']):
                                 ds = sgrp.create_dataset(
                                     self.getLabel(dsInfo['variable'], i),
                                     (self.blocksPerFile, ) + dsInfo['shape'][1:],
                                     chunks=True,
                                     dtype=dsInfo['dtype'])
                                 dtsets.append(ds)
                                 data.append((dsInfo['variable'], i))
                     fp.flush()
                     log.log('Creating file: {}'.format(fp.filename), self.name)
                     self.ds = dtsets
                     self.data = data
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
                         self.closeFile()
                         self.setNextFile()
                     for i, ds in enumerate(self.ds):
                         attr, ch = self.data[i]
                         if ch == -1:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)
                         else:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
                     self.fp.flush()
                     self.blockIndex += 1
                     log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
                     return
                 def closeFile(self):
                     if self.blockIndex != self.blocksPerFile:
                         for ds in self.ds:
                             ds.resize(self.blockIndex, axis=0)
                     if self.fp:
                         self.fp.flush()
                         self.fp.close()
                 def close(self):
                     self.closeFile()

schainpy/model/proc/bltrproc_parameters.py +3 -16

             '''
             Created on Oct 24, 2016
             @author: roj- LouVD
             '''
             import numpy
-            import copy
             import datetime
             import time
-            from time import gmtime
-            from numpy import transpose
+            from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
-            from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jrodata import Parameters
             class BLTRParametersProc(ProcessingUnit):
                 '''
                 Processing unit for BLTR parameters data (winds)
                 Inputs:
                     self.dataOut.nmodes - Number of operation modes
                     self.dataOut.nchannels - Number of channels
                     self.dataOut.nranges - Number of ranges
                     self.dataOut.data_snr - SNR array
                     self.dataOut.data_output - Zonal, Vertical and Meridional velocity array
                     self.dataOut.height - Height array (km)
                     self.dataOut.time - Time array (seconds)
                     self.dataOut.fileIndex -Index of the file currently read
                     self.dataOut.lat - Latitude coordinate of BLTR location
                     self.dataOut.doy - Experiment doy (number of the day in the current year)
                     self.dataOut.month - Experiment month
                     self.dataOut.day - Experiment day
                     self.dataOut.year - Experiment year
                 '''
                 def __init__(self):
                     '''
                     Inputs: None
                     '''
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                 def setup(self, mode):
                     '''
                     '''
                     self.dataOut.mode = mode
                 def run(self, mode, snr_threshold=None):
                     '''
                     Inputs:
                         mode = High resolution (0) or Low resolution (1) data
                         snr_threshold = snr filter value
                     '''
                     if not self.isConfig:
                         self.setup(mode)
                         self.isConfig = True
                     if self.dataIn.type == 'Parameters':
                         self.dataOut.copy(self.dataIn)
                     self.dataOut.data_param = self.dataOut.data[mode]
                     self.dataOut.heightList = self.dataOut.height[0]
                     self.dataOut.data_snr = self.dataOut.data_snr[mode]
-                    data_param = numpy.zeros([4, len(self.dataOut.heightList)])
                     SNRavg = numpy.average(self.dataOut.data_snr, axis=0)
                     SNRavgdB = 10*numpy.log10(SNRavg)
+                    self.dataOut.data_snr_avg_db = SNRavgdB.reshape(1, *SNRavgdB.shape)
                     # Censoring Data
                     if snr_threshold is not None:
                         for i in range(3):
                             self.dataOut.data_param[i][SNRavgdB <= snr_threshold] = numpy.nan
-                    # Including AvgSNR in data_param
-                    for i in range(data_param.shape[0]):
-                        if i == 3:
-                            data_param[i] = SNRavgdB
-                        else:
-                            data_param[i] = self.dataOut.data_param[i]
-                    self.dataOut.data_param = data_param
             # TODO
             class OutliersFilter(Operation):
                 def __init__(self):
                     '''
                     '''
                     Operation.__init__(self)
                 def run(self, svalue2, method, factor, filter, npoints=9):
                     '''
                     Inputs:
                         svalue    -  string to select array velocity
                         svalue2    -  string to choose axis filtering
                         method    - 0 for SMOOTH or 1 for MEDIAN
                         factor    - number used to set threshold
                         filter    - 1 for data filtering using the standard deviation criteria else 0
                         npoints    - number of points for mask filter
                     '''
                     print('    Outliers Filter {} {} / threshold = {}'.format(svalue, svalue, factor))
                     yaxis = self.dataOut.heightList
                     xaxis = numpy.array([[self.dataOut.utctime]])
                     # Zonal
                     value_temp = self.dataOut.data_output[0]
                     # Zonal
                     value_temp = self.dataOut.data_output[1]
                     # Vertical
                     value_temp = numpy.transpose(self.dataOut.data_output[2])
                     htemp = yaxis
                     std = value_temp
                     for h in range(len(htemp)):
                         nvalues_valid = len(numpy.where(numpy.isfinite(value_temp[h]))[0])
                         minvalid = npoints
                         #only if valid values greater than the minimum required (10%)
                         if nvalues_valid > minvalid:
                             if method == 0:
                                 #SMOOTH
                                 w = value_temp[h] - self.Smooth(input=value_temp[h], width=npoints, edge_truncate=1)
                             if method == 1:
                                 #MEDIAN
                                 w = value_temp[h] - self.Median(input=value_temp[h], width = npoints)
                             dw = numpy.std(w[numpy.where(numpy.isfinite(w))],ddof = 1)
                             threshold = dw*factor
                             value_temp[numpy.where(w > threshold),h] = numpy.nan
                             value_temp[numpy.where(w < -1*threshold),h] = numpy.nan
                     #At the end
                     if svalue2 == 'inHeight':
                         value_temp = numpy.transpose(value_temp)
                     output_array[:,m] = value_temp
                     if svalue == 'zonal':
                         self.dataOut.data_output[0] = output_array
                     elif svalue == 'meridional':
                         self.dataOut.data_output[1] = output_array
                     elif svalue == 'vertical':
                         self.dataOut.data_output[2] = output_array
                     return self.dataOut.data_output
                 def Median(self,input,width):
                     '''
                     Inputs:
                         input - Velocity array
                         width - Number of points for mask filter
                     '''
                     if numpy.mod(width,2) == 1:
                         pc = int((width - 1) / 2)
                     cont = 0
                     output = []
                     for i in range(len(input)):
                         if i >= pc and i < len(input) - pc:
                             new2 = input[i-pc:i+pc+1]
                             temp = numpy.where(numpy.isfinite(new2))
                             new = new2[temp]
                             value = numpy.median(new)
                             output.append(value)
                     output = numpy.array(output)
                     output = numpy.hstack((input[0:pc],output))
                     output = numpy.hstack((output,input[-pc:len(input)]))
                     return output
                 def Smooth(self,input,width,edge_truncate = None):
                     '''
                     Inputs:
                         input - Velocity array
                         width - Number of points for mask filter
                         edge_truncate - 1 for truncate the convolution product else
                     '''
                     if numpy.mod(width,2) == 0:
                         real_width = width + 1
                         nzeros = width / 2
                     else:
                         real_width = width
                         nzeros = (width - 1) / 2
                     half_width = int(real_width)/2
                     length = len(input)
                     gate = numpy.ones(real_width,dtype='float')
                     norm_of_gate = numpy.sum(gate)
                     nan_process = 0
                     nan_id = numpy.where(numpy.isnan(input))
                     if len(nan_id[0]) > 0:
                         nan_process = 1
                         pb = numpy.zeros(len(input))
                         pb[nan_id] = 1.
                         input[nan_id] = 0.
                     if edge_truncate == True:
                         output = numpy.convolve(input/norm_of_gate,gate,mode='same')
                     elif edge_truncate == False or edge_truncate == None:
                         output = numpy.convolve(input/norm_of_gate,gate,mode='valid')
                         output = numpy.hstack((input[0:half_width],output))
                         output = numpy.hstack((output,input[len(input)-half_width:len(input)]))
                     if nan_process:
                         pb = numpy.convolve(pb/norm_of_gate,gate,mode='valid')
                         pb = numpy.hstack((numpy.zeros(half_width),pb))
                         pb = numpy.hstack((pb,numpy.zeros(half_width)))
                         output[numpy.where(pb > 0.9999)] = numpy.nan
                         input[nan_id] = numpy.nan
                     return output
                 def Average(self,aver=0,nhaver=1):
                     '''
                     Inputs:
                         aver - Indicates the time period over which is averaged or consensus data
                         nhaver - Indicates the decimation factor in heights
                     '''
                     nhpoints = 48
                     lat_piura = -5.17
                     lat_huancayo = -12.04
                     lat_porcuya = -5.8
                     if '%2.2f'%self.dataOut.lat == '%2.2f'%lat_piura:
                         hcm = 3.
                         if self.dataOut.year == 2003 :
                             if self.dataOut.doy >= 25 and self.dataOut.doy < 64:
                                 nhpoints = 12
                     elif '%2.2f'%self.dataOut.lat == '%2.2f'%lat_huancayo:
                         hcm = 3.
                         if self.dataOut.year == 2003 :
                             if self.dataOut.doy >= 25 and self.dataOut.doy < 64:
                                 nhpoints = 12
                     elif '%2.2f'%self.dataOut.lat == '%2.2f'%lat_porcuya:
                         hcm = 5.#2
                     pdata = 0.2
                     taver = [1,2,3,4,6,8,12,24]
                     t0 = 0
                     tf = 24
                     ntime =(tf-t0)/taver[aver]
                     ti = numpy.arange(ntime)
                     tf = numpy.arange(ntime) + taver[aver]
                     old_height = self.dataOut.heightList
                     if nhaver > 1:
                         num_hei = len(self.dataOut.heightList)/nhaver/self.dataOut.nmodes
                         deltha = 0.05*nhaver
                         minhvalid = pdata*nhaver
                         for im in range(self.dataOut.nmodes):
                             new_height = numpy.arange(num_hei)*deltha + self.dataOut.height[im,0] + deltha/2.
                     data_fHeigths_List = []
                     data_fZonal_List = []
                     data_fMeridional_List = []
                     data_fVertical_List = []
                     startDTList = []
                     for i in range(ntime):
                         height = old_height
                         start = datetime.datetime(self.dataOut.year,self.dataOut.month,self.dataOut.day) + datetime.timedelta(hours = int(ti[i])) - datetime.timedelta(hours = 5)
                         stop = datetime.datetime(self.dataOut.year,self.dataOut.month,self.dataOut.day) + datetime.timedelta(hours = int(tf[i])) - datetime.timedelta(hours = 5)
                         limit_sec1 = time.mktime(start.timetuple())
                         limit_sec2 = time.mktime(stop.timetuple())
                         t1 = numpy.where(self.f_timesec >= limit_sec1)
                         t2 = numpy.where(self.f_timesec < limit_sec2)
                         time_select = []
                         for val_sec in t1[0]:
                             if val_sec in t2[0]:
                                 time_select.append(val_sec)
                         time_select = numpy.array(time_select,dtype = 'int')
                         minvalid = numpy.ceil(pdata*nhpoints)
                         zon_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan
                         mer_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan
                         ver_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan
                         if nhaver > 1:
                             new_zon_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan
                             new_mer_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan
                             new_ver_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan
                         if len(time_select) > minvalid:
                             time_average = self.f_timesec[time_select]
                             for im in range(self.dataOut.nmodes):
                                 for ih in range(self.dataOut.nranges):
                                     if numpy.sum(numpy.isfinite(self.f_zon[time_select,ih,im])) >= minvalid:
                                         zon_aver[ih,im] = numpy.nansum(self.f_zon[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_zon[time_select,ih,im]))
                                     if numpy.sum(numpy.isfinite(self.f_mer[time_select,ih,im])) >= minvalid:
                                         mer_aver[ih,im] = numpy.nansum(self.f_mer[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_mer[time_select,ih,im]))
                                     if numpy.sum(numpy.isfinite(self.f_ver[time_select,ih,im])) >= minvalid:
                                         ver_aver[ih,im] = numpy.nansum(self.f_ver[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_ver[time_select,ih,im]))
                                 if nhaver > 1:
                                     for ih in range(num_hei):
                                         hvalid = numpy.arange(nhaver) + nhaver*ih
                                         if numpy.sum(numpy.isfinite(zon_aver[hvalid,im])) >= minvalid:
                                             new_zon_aver[ih,im] = numpy.nansum(zon_aver[hvalid,im]) /  numpy.sum(numpy.isfinite(zon_aver[hvalid,im]))
                                         if numpy.sum(numpy.isfinite(mer_aver[hvalid,im])) >= minvalid:
                                             new_mer_aver[ih,im] = numpy.nansum(mer_aver[hvalid,im]) /  numpy.sum(numpy.isfinite(mer_aver[hvalid,im]))
                                         if numpy.sum(numpy.isfinite(ver_aver[hvalid,im])) >= minvalid:
                                             new_ver_aver[ih,im] = numpy.nansum(ver_aver[hvalid,im]) /  numpy.sum(numpy.isfinite(ver_aver[hvalid,im]))
                             if nhaver > 1:
                                 zon_aver = new_zon_aver
                                 mer_aver = new_mer_aver
                                 ver_aver = new_ver_aver
                                 height = new_height
                             tstart = time_average[0]
                             tend = time_average[-1]
                             startTime = time.gmtime(tstart)
                             year = startTime.tm_year
                             month = startTime.tm_mon
                             day = startTime.tm_mday
                             hour = startTime.tm_hour
                             minute = startTime.tm_min
                             second = startTime.tm_sec
                             startDTList.append(datetime.datetime(year,month,day,hour,minute,second))
                             o_height = numpy.array([])
                             o_zon_aver = numpy.array([])
                             o_mer_aver = numpy.array([])
                             o_ver_aver = numpy.array([])
                             if self.dataOut.nmodes > 1:
                                 for im in range(self.dataOut.nmodes):
                                     if im == 0:
                                         h_select = numpy.where(numpy.bitwise_and(height[0,:] >=0,height[0,:] <= hcm,numpy.isfinite(height[0,:])))
                                     else:
                                         h_select = numpy.where(numpy.bitwise_and(height[1,:] > hcm,height[1,:] < 20,numpy.isfinite(height[1,:])))
                                     ht = h_select[0]
                                     o_height = numpy.hstack((o_height,height[im,ht]))
                                     o_zon_aver = numpy.hstack((o_zon_aver,zon_aver[ht,im]))
                                     o_mer_aver = numpy.hstack((o_mer_aver,mer_aver[ht,im]))
                                     o_ver_aver = numpy.hstack((o_ver_aver,ver_aver[ht,im]))
                                 data_fHeigths_List.append(o_height)
                                 data_fZonal_List.append(o_zon_aver)
                                 data_fMeridional_List.append(o_mer_aver)
                                 data_fVertical_List.append(o_ver_aver)
                             else:
                                 h_select = numpy.where(numpy.bitwise_and(height[0,:] <= hcm,numpy.isfinite(height[0,:])))
                                 ht = h_select[0]
                                 o_height = numpy.hstack((o_height,height[im,ht]))
                                 o_zon_aver = numpy.hstack((o_zon_aver,zon_aver[ht,im]))
                                 o_mer_aver = numpy.hstack((o_mer_aver,mer_aver[ht,im]))
                                 o_ver_aver = numpy.hstack((o_ver_aver,ver_aver[ht,im]))
                                 data_fHeigths_List.append(o_height)
                                 data_fZonal_List.append(o_zon_aver)
                                 data_fMeridional_List.append(o_mer_aver)
                                 data_fVertical_List.append(o_ver_aver)
                     return startDTList, data_fHeigths_List, data_fZonal_List, data_fMeridional_List, data_fVertical_List

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages