schain Commit - r1251:6d7191c1eaa6 · Jicamarca Repository

Cleaning IO modules (base, voltaje and spectra)

jespinoza -

r1251:6d7191c1eaa6

parent child

Context file:

r1251:6d7191c1eaa6

Collapse all files

schainpy/model/__init__.py 0 -6

-            #from schainpy.model.data.jrodata import *
-            # from schainpy.model.io.jrodataIO import *
-            # from schainpy.model.proc.jroprocessing import *
-            # from schainpy.model.graphics.jroplot import *
-            # from schainpy.model.utils.jroutils import *
-            # from schainpy.serializer import *
             from .graphics import *
             from .data import *
             from .io import *
             from .proc import *
             from .utils import *

schainpy/model/graphics/jroplot_base.py +1 -1

             import os
             import sys
             import zmq
             import time
             import numpy
             import datetime
             from functools import wraps
             from threading import Thread
             import matplotlib
             if 'BACKEND' in os.environ:
                 matplotlib.use(os.environ['BACKEND'])
             elif 'linux' in sys.platform:
                 matplotlib.use("TkAgg")
             elif 'darwin' in sys.platform:
                 matplotlib.use('WxAgg')
             else:
                 from schainpy.utils import log
                 log.warning('Using default Backend="Agg"', 'INFO')
                 matplotlib.use('Agg')
             import matplotlib.pyplot as plt
             from matplotlib.patches import Polygon
             from mpl_toolkits.axes_grid1 import make_axes_locatable
             from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
             from schainpy.model.data.jrodata import PlotterData
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.utils import log
             jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
             blu_values = matplotlib.pyplot.get_cmap(
                 'seismic_r', 20)(numpy.arange(20))[10:15]
             ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
                 'jro', numpy.vstack((blu_values, jet_values)))
             matplotlib.pyplot.register_cmap(cmap=ncmap)
             CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis',
                                                'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
             EARTH_RADIUS = 6.3710e3
             def ll2xy(lat1, lon1, lat2, lon2):
                 p = 0.017453292519943295
                 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
                     numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
                 r = 12742 * numpy.arcsin(numpy.sqrt(a))
                 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
                                       * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
                 theta = -theta + numpy.pi/2
                 return r*numpy.cos(theta), r*numpy.sin(theta)
             def km2deg(km):
                 '''
                 Convert distance in km to degrees
                 '''
                 return numpy.rad2deg(km/EARTH_RADIUS)
             def figpause(interval):
                 backend = plt.rcParams['backend']
                 if backend in matplotlib.rcsetup.interactive_bk:
                     figManager = matplotlib._pylab_helpers.Gcf.get_active()
                     if figManager is not None:
                         canvas = figManager.canvas
                         if canvas.figure.stale:
                             canvas.draw()
                         try:
                             canvas.start_event_loop(interval)
                         except:
                             pass
                         return
             def popup(message):
                 '''
                 '''
                 fig = plt.figure(figsize=(12, 8), facecolor='r')
                 text = '\n'.join([s.strip() for s in message.split(':')])
                 fig.text(0.01, 0.5, text, ha='left', va='center',
                          size='20', weight='heavy', color='w')
                 fig.show()
                 figpause(1000)
             class Throttle(object):
                 '''
                 Decorator that prevents a function from being called more than once every
                 time period.
                 To create a function that cannot be called more than once a minute, but
                 will sleep until it can be called:
                 @Throttle(minutes=1)
                 def foo():
                   pass
                 for i in range(10):
                   foo()
                   print "This function has run %s times." % i
                 '''
                 def __init__(self, seconds=0, minutes=0, hours=0):
                     self.throttle_period = datetime.timedelta(
                         seconds=seconds, minutes=minutes, hours=hours
                     )
                     self.time_of_last_call = datetime.datetime.min
                 def __call__(self, fn):
                     @wraps(fn)
                     def wrapper(*args, **kwargs):
                         coerce = kwargs.pop('coerce', None)
                         if coerce:
                             self.time_of_last_call = datetime.datetime.now()
                             return fn(*args, **kwargs)
                         else:
                             now = datetime.datetime.now()
                             time_since_last_call = now - self.time_of_last_call
                             time_left = self.throttle_period - time_since_last_call
                             if time_left > datetime.timedelta(seconds=0):
                                 return
                         self.time_of_last_call = datetime.datetime.now()
                         return fn(*args, **kwargs)
                     return wrapper
             def apply_throttle(value):
                 @Throttle(seconds=value)
                 def fnThrottled(fn):
                     fn()
                 return fnThrottled
             @MPDecorator
             class Plot(Operation):
                 '''
                 Base class for Schain plotting operations
                 '''
                 CODE = 'Figure'
                 colormap = 'jet'
                 bgcolor = 'white'
                 __missing = 1E30
                 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
                              'zlimits', 'xlabel', 'ylabel', 'xaxis', 'cb_label', 'title',
                              'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
                              'showprofile', 'decimation', 'pause']
                 def __init__(self):
                     Operation.__init__(self)
                     self.isConfig = False
                     self.isPlotConfig = False
                     self.save_counter = 1
                     self.sender_counter = 1
                     self.data = None
                 def __fmtTime(self, x, pos):
                     '''
                     '''
                     return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
                 def __setup(self, **kwargs):
                     '''
                     Initialize variables
                     '''
                     self.figures = []
                     self.axes = []
                     self.cb_axes = []
                     self.localtime = kwargs.pop('localtime', True)
                     self.show = kwargs.get('show', True)
                     self.save = kwargs.get('save', False)
-                    self.save_period = kwargs.get('save_period', 2)
+                    self.save_period = kwargs.get('save_period', 1)
                     self.ftp = kwargs.get('ftp', False)
                     self.colormap = kwargs.get('colormap', self.colormap)
                     self.colormap_coh = kwargs.get('colormap_coh', 'jet')
                     self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
                     self.colormaps = kwargs.get('colormaps', None)
                     self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
                     self.showprofile = kwargs.get('showprofile', False)
                     self.title = kwargs.get('wintitle', self.CODE.upper())
                     self.cb_label = kwargs.get('cb_label', None)
                     self.cb_labels = kwargs.get('cb_labels', None)
                     self.labels = kwargs.get('labels', None)
                     self.xaxis = kwargs.get('xaxis', 'frequency')
                     self.zmin = kwargs.get('zmin', None)
                     self.zmax = kwargs.get('zmax', None)
                     self.zlimits = kwargs.get('zlimits', None)
                     self.xmin = kwargs.get('xmin', None)
                     self.xmax = kwargs.get('xmax', None)
                     self.xrange = kwargs.get('xrange', 24)
                     self.xscale = kwargs.get('xscale', None)
                     self.ymin = kwargs.get('ymin', None)
                     self.ymax = kwargs.get('ymax', None)
                     self.yscale = kwargs.get('yscale', None)
                     self.xlabel = kwargs.get('xlabel', None)
                     self.decimation = kwargs.get('decimation', None)
                     self.showSNR = kwargs.get('showSNR', False)
                     self.oneFigure = kwargs.get('oneFigure', True)
                     self.width = kwargs.get('width', None)
                     self.height = kwargs.get('height', None)
                     self.colorbar = kwargs.get('colorbar', True)
                     self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
                     self.channels = kwargs.get('channels', None)
                     self.titles = kwargs.get('titles', [])
                     self.polar = False
                     self.type = kwargs.get('type', 'iq')
                     self.grid = kwargs.get('grid', False)
                     self.pause = kwargs.get('pause', False)
                     self.save_labels = kwargs.get('save_labels', None)
                     self.realtime = kwargs.get('realtime', True)
                     self.buffering = kwargs.get('buffering', True)
                     self.throttle = kwargs.get('throttle', 2)
                     self.exp_code = kwargs.get('exp_code', None)
                     self.plot_server = kwargs.get('plot_server', False)
                     self.sender_period = kwargs.get('sender_period', 1)
                     self.__throttle_plot = apply_throttle(self.throttle)
                     self.data = PlotterData(
                         self.CODE, self.throttle, self.exp_code, self.buffering, snr=self.showSNR)
                     if self.plot_server:
                         if not self.plot_server.startswith('tcp://'):
                             self.plot_server = 'tcp://{}'.format(self.plot_server)
                         log.success(
                             'Sending to server: {}'.format(self.plot_server),
                             self.name
                         )
                     if 'plot_name' in kwargs:
                         self.plot_name = kwargs['plot_name']
                 def __setup_plot(self):
                     '''
                     Common setup for all figures, here figures and axes are created
                     '''
                     self.setup()
                     self.time_label = 'LT' if self.localtime else 'UTC'
                     if self.width is None:
                         self.width = 8
                     self.figures = []
                     self.axes = []
                     self.cb_axes = []
                     self.pf_axes = []
                     self.cmaps = []
                     size = '15%' if self.ncols == 1 else '30%'
                     pad = '4%' if self.ncols == 1 else '8%'
                     if self.oneFigure:
                         if self.height is None:
                             self.height = 1.4 * self.nrows + 1
                         fig = plt.figure(figsize=(self.width, self.height),
                                          edgecolor='k',
                                          facecolor='w')
                         self.figures.append(fig)
                         for n in range(self.nplots):
                             ax = fig.add_subplot(self.nrows, self.ncols,
                                                  n + 1, polar=self.polar)
                             ax.tick_params(labelsize=8)
                             ax.firsttime = True
                             ax.index = 0
                             ax.press = None
                             self.axes.append(ax)
                             if self.showprofile:
                                 cax = self.__add_axes(ax, size=size, pad=pad)
                                 cax.tick_params(labelsize=8)
                                 self.pf_axes.append(cax)
                     else:
                         if self.height is None:
                             self.height = 3
                         for n in range(self.nplots):
                             fig = plt.figure(figsize=(self.width, self.height),
                                              edgecolor='k',
                                              facecolor='w')
                             ax = fig.add_subplot(1, 1, 1, polar=self.polar)
                             ax.tick_params(labelsize=8)
                             ax.firsttime = True
                             ax.index = 0
                             ax.press = None
                             self.figures.append(fig)
                             self.axes.append(ax)
                             if self.showprofile:
                                 cax = self.__add_axes(ax, size=size, pad=pad)
                                 cax.tick_params(labelsize=8)
                                 self.pf_axes.append(cax)
                     for n in range(self.nrows):
                         if self.colormaps is not None:
                             cmap = plt.get_cmap(self.colormaps[n])
                         else:
                             cmap = plt.get_cmap(self.colormap)
                         cmap.set_bad(self.bgcolor, 1.)
                         self.cmaps.append(cmap)
                     for fig in self.figures:
                         fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
                         fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
                         fig.canvas.mpl_connect('button_press_event', self.onBtnPress)
                         fig.canvas.mpl_connect('motion_notify_event', self.onMotion)
                         fig.canvas.mpl_connect('button_release_event', self.onBtnRelease)
                 def OnKeyPress(self, event):
                     '''
                     Event for pressing keys (up, down) change colormap
                     '''
                     ax = event.inaxes
                     if ax in self.axes:
                         if event.key == 'down':
                             ax.index += 1
                         elif event.key == 'up':
                             ax.index -= 1
                         if ax.index < 0:
                             ax.index = len(CMAPS) - 1
                         elif ax.index == len(CMAPS):
                             ax.index = 0
                         cmap = CMAPS[ax.index]
                         ax.cbar.set_cmap(cmap)
                         ax.cbar.draw_all()
                         ax.plt.set_cmap(cmap)
                         ax.cbar.patch.figure.canvas.draw()
                         self.colormap = cmap.name
                 def OnBtnScroll(self, event):
                     '''
                     Event for scrolling, scale figure
                     '''
                     cb_ax = event.inaxes
                     if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
                         ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
                         pt = ax.cbar.ax.bbox.get_points()[:, 1]
                         nrm = ax.cbar.norm
                         vmin, vmax, p0, p1, pS = (
                             nrm.vmin, nrm.vmax, pt[0], pt[1], event.y)
                         scale = 2 if event.step == 1 else 0.5
                         point = vmin + (vmax - vmin) / (p1 - p0) * (pS - p0)
                         ax.cbar.norm.vmin = point - scale * (point - vmin)
                         ax.cbar.norm.vmax = point - scale * (point - vmax)
                         ax.plt.set_norm(ax.cbar.norm)
                         ax.cbar.draw_all()
                         ax.cbar.patch.figure.canvas.draw()
                 def onBtnPress(self, event):
                     '''
                     Event for mouse button press
                     '''
                     cb_ax = event.inaxes
                     if cb_ax is None:
                         return
                     if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
                         cb_ax.press = event.x, event.y
                     else:
                         cb_ax.press = None
                 def onMotion(self, event):
                     '''
                     Event for move inside colorbar
                     '''
                     cb_ax = event.inaxes
                     if cb_ax is None:
                         return
                     if cb_ax not in [ax.cbar.ax for ax in self.axes if ax.cbar]:
                         return
                     if cb_ax.press is None:
                         return
                     ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
                     xprev, yprev = cb_ax.press
                     dx = event.x - xprev
                     dy = event.y - yprev
                     cb_ax.press = event.x, event.y
                     scale = ax.cbar.norm.vmax - ax.cbar.norm.vmin
                     perc = 0.03
                     if event.button == 1:
                         ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
                         ax.cbar.norm.vmax -= (perc * scale) * numpy.sign(dy)
                     elif event.button == 3:
                         ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
                         ax.cbar.norm.vmax += (perc * scale) * numpy.sign(dy)
                     ax.cbar.draw_all()
                     ax.plt.set_norm(ax.cbar.norm)
                     ax.cbar.patch.figure.canvas.draw()
                 def onBtnRelease(self, event):
                     '''
                     Event for mouse button release
                     '''
                     cb_ax = event.inaxes
                     if cb_ax is not None:
                         cb_ax.press = None
                 def __add_axes(self, ax, size='30%', pad='8%'):
                     '''
                     Add new axes to the given figure
                     '''
                     divider = make_axes_locatable(ax)
                     nax = divider.new_horizontal(size=size, pad=pad)
                     ax.figure.add_axes(nax)
                     return nax
                 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
                     '''
                     Create a masked array for missing data
                     '''
                     if x_buffer.shape[0] < 2:
                         return x_buffer, y_buffer, z_buffer
                     deltas = x_buffer[1:] - x_buffer[0:-1]
                     x_median = numpy.median(deltas)
                     index = numpy.where(deltas > 5 * x_median)
                     if len(index[0]) != 0:
                         z_buffer[::, index[0], ::] = self.__missing
                         z_buffer = numpy.ma.masked_inside(z_buffer,
 .99 * self.__missing,
 .01 * self.__missing)
                     return x_buffer, y_buffer, z_buffer
                 def decimate(self):
                     # dx = int(len(self.x)/self.__MAXNUMX) + 1
                     dy = int(len(self.y) / self.decimation) + 1
                     # x = self.x[::dx]
                     x = self.x
                     y = self.y[::dy]
                     z = self.z[::, ::, ::dy]
                     return x, y, z
                 def format(self):
                     '''
                     Set min and max values, labels, ticks and titles
                     '''
                     if self.xmin is None:
                         xmin = self.data.min_time
                     else:
                         if self.xaxis is 'time':
                             dt = self.getDateTime(self.data.min_time)
                             xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
                                     datetime.datetime(1970, 1, 1)).total_seconds()
                             if self.data.localtime:
                                 xmin += time.timezone
                         else:
                             xmin = self.xmin
                     if self.xmax is None:
                         xmax = xmin + self.xrange * 60 * 60
                     else:
                         if self.xaxis is 'time':
                             dt = self.getDateTime(self.data.max_time)
                             xmax = (dt.replace(hour=int(self.xmax), minute=59, second=59) -
                                     datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds()
                             if self.data.localtime:
                                 xmax += time.timezone
                         else:
                             xmax = self.xmax
                     ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
                     ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
                     #Y = numpy.array([1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000])
                     #i = 1 if numpy.where(
                     #    abs(ymax-ymin) <= Y)[0][0] < 0 else numpy.where(abs(ymax-ymin) <= Y)[0][0]
                     #ystep = Y[i] / 10.
                     dig = int(numpy.log10(ymax))
                     if dig == 0:
                         digD = len(str(ymax)) - 2
                         ydec = ymax*(10**digD)
                         dig = int(numpy.log10(ydec))
                         ystep = ((ydec + (10**(dig)))//10**(dig))*(10**(dig))
                         ystep = ystep/5
                         ystep = ystep/(10**digD)
                     else:
                         ystep = ((ymax + (10**(dig)))//10**(dig))*(10**(dig))
                         ystep = ystep/5
                     if self.xaxis is not 'time':
                         dig = int(numpy.log10(xmax))
                         if dig <= 0:
                             digD = len(str(xmax)) - 2
                             xdec = xmax*(10**digD)
                             dig = int(numpy.log10(xdec))
                             xstep = ((xdec + (10**(dig)))//10**(dig))*(10**(dig))
                             xstep = xstep*0.5
                             xstep = xstep/(10**digD)
                         else:
                             xstep = ((xmax + (10**(dig)))//10**(dig))*(10**(dig))
                             xstep = xstep/5
                     for n, ax in enumerate(self.axes):
                         if ax.firsttime:
                             ax.set_facecolor(self.bgcolor)
                             ax.yaxis.set_major_locator(MultipleLocator(ystep))
                             if self.xscale:
                                 ax.xaxis.set_major_formatter(FuncFormatter(
                                     lambda x, pos: '{0:g}'.format(x*self.xscale)))
                             if self.xscale:
                                 ax.yaxis.set_major_formatter(FuncFormatter(
                                     lambda x, pos: '{0:g}'.format(x*self.yscale)))
                             if self.xaxis is 'time':
                                 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
                                 ax.xaxis.set_major_locator(LinearLocator(9))
                             else:
                                 ax.xaxis.set_major_locator(MultipleLocator(xstep))
                             if self.xlabel is not None:
                                 ax.set_xlabel(self.xlabel)
                             ax.set_ylabel(self.ylabel)
                             ax.firsttime = False
                             if self.showprofile:
                                 self.pf_axes[n].set_ylim(ymin, ymax)
                                 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
                                 self.pf_axes[n].set_xlabel('dB')
                                 self.pf_axes[n].grid(b=True, axis='x')
                                 [tick.set_visible(False)
                                  for tick in self.pf_axes[n].get_yticklabels()]
                             if self.colorbar:
                                 ax.cbar = plt.colorbar(
                                     ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
                                 ax.cbar.ax.tick_params(labelsize=8)
                                 ax.cbar.ax.press = None
                                 if self.cb_label:
                                     ax.cbar.set_label(self.cb_label, size=8)
                                 elif self.cb_labels:
                                     ax.cbar.set_label(self.cb_labels[n], size=8)
                             else:
                                 ax.cbar = None
                         if self.grid:
                             ax.grid(True)
                         if not self.polar:
                             ax.set_xlim(xmin, xmax)
                             ax.set_ylim(ymin, ymax)
                             ax.set_title('{} {} {}'.format(
                                 self.titles[n],
                                 self.getDateTime(self.data.max_time).strftime(
                                     '%Y-%m-%d %H:%M:%S'),
                                 self.time_label),
                                 size=8)
                         else:
                             ax.set_title('{}'.format(self.titles[n]), size=8)
                             ax.set_ylim(0, 90)
                             ax.set_yticks(numpy.arange(0, 90, 20))
                             ax.yaxis.labelpad = 40
                 def clear_figures(self):
                     '''
                     Reset axes for redraw plots
                     '''
                     for ax in self.axes:
                         ax.clear()
                         ax.firsttime = True
                         if ax.cbar:
                             ax.cbar.remove()
                 def __plot(self):
                     '''
                     Main function to plot, format and save figures
                     '''
                     try:
                         self.plot()
                         self.format()
                     except Exception as e:
                        log.warning('{} Plot could not be updated... check data'.format(
                            self.CODE), self.name)
                        log.error(str(e), '')
                        return
                     for n, fig in enumerate(self.figures):
                         if self.nrows == 0 or self.nplots == 0:
                             log.warning('No data', self.name)
                             fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
                             fig.canvas.manager.set_window_title(self.CODE)
                             continue
                         fig.tight_layout()
                         fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
                                                                              self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
                         fig.canvas.draw()
                         if self.show:
                             fig.show()
                             figpause(0.1)
                         if self.save:
                             self.save_figure(n)
                     if self.plot_server:
                         self.send_to_server()
                         # t = Thread(target=self.send_to_server)
                         # t.start()
                 def save_figure(self, n):
                     '''
                     '''
                     if self.save_counter < self.save_period:
                         self.save_counter += 1
                         return
                     self.save_counter = 1
                     fig = self.figures[n]
                     if self.save_labels:
                         labels = self.save_labels
                     else:
                         labels = list(range(self.nrows))
                     if self.oneFigure:
                         label = ''
                     else:
                         label = '-{}'.format(labels[n])
                     figname = os.path.join(
                         self.save,
                         self.CODE,
                         '{}{}_{}.png'.format(
                             self.CODE,
                             label,
                             self.getDateTime(self.data.max_time).strftime(
                                 '%Y%m%d_%H%M%S'
                                 ),
                             )
                         )
                     log.log('Saving figure: {}'.format(figname), self.name)
                     if not os.path.isdir(os.path.dirname(figname)):
                         os.makedirs(os.path.dirname(figname))
                     fig.savefig(figname)
                     if self.realtime:
                         figname = os.path.join(
                             self.save,
                             '{}{}_{}.png'.format(
                                 self.CODE,
                                 label,
                                 self.getDateTime(self.data.min_time).strftime(
                                     '%Y%m%d'
                                     ),
                                 )
                             )
                         fig.savefig(figname)
                 def send_to_server(self):
                     '''
                     '''
                     if self.sender_counter < self.sender_period:
                         self.sender_counter += 1
                         return
                     self.sender_counter = 1
                     self.data.meta['titles'] = self.titles
                     retries = 2
                     while True:
                         self.socket.send_string(self.data.jsonify(self.plot_name, self.plot_type))
                         socks = dict(self.poll.poll(5000))
                         if socks.get(self.socket) == zmq.POLLIN:
                             reply = self.socket.recv_string()
                             if reply == 'ok':
                                 log.log("Response from server ok", self.name)
                                 break
                             else:
                                 log.warning(
                                     "Malformed reply from server: {}".format(reply), self.name)
                         else:
                             log.warning(
                                 "No response from server, retrying...", self.name)
                         self.socket.setsockopt(zmq.LINGER, 0)
                         self.socket.close()
                         self.poll.unregister(self.socket)
                         retries -= 1
                         if retries == 0:
                             log.error(
                                 "Server seems to be offline, abandoning", self.name)
                             self.socket = self.context.socket(zmq.REQ)
                             self.socket.connect(self.plot_server)
                             self.poll.register(self.socket, zmq.POLLIN)
                             time.sleep(1)
                             break
                         self.socket = self.context.socket(zmq.REQ)
                         self.socket.connect(self.plot_server)
                         self.poll.register(self.socket, zmq.POLLIN)
                         time.sleep(0.5)
                 def setup(self):
                     '''
                     This method should be implemented in the child class, the following
                     attributes should be set:
                     self.nrows: number of rows
                     self.ncols: number of cols
                     self.nplots: number of plots (channels or pairs)
                     self.ylabel: label for Y axes
                     self.titles: list of axes title
                     '''
                     raise NotImplementedError
                 def plot(self):
                     '''
                     Must be defined in the child class
                     '''
                     raise NotImplementedError
                 def run(self, dataOut, **kwargs):
                     '''
                     Main plotting routine
                     '''
                     if self.isConfig is False:
                         self.__setup(**kwargs)
                         if dataOut.type == 'Parameters':
                             t = dataOut.utctimeInit
                         else:
                             t = dataOut.utctime
                         if dataOut.useLocalTime:
                             self.getDateTime = datetime.datetime.fromtimestamp
                             if not self.localtime:
                                 t += time.timezone
                         else:
                             self.getDateTime = datetime.datetime.utcfromtimestamp
                             if self.localtime:
                                 t -= time.timezone
                         if 'buffer' in self.plot_type:
                             if self.xmin is None:
                                 self.tmin = t
                             else:
                                 self.tmin = (
                                     self.getDateTime(t).replace(
                                         hour=self.xmin,
                                         minute=0,
                                         second=0) - self.getDateTime(0)).total_seconds()
                         self.data.setup()
                         self.isConfig = True
                         if self.plot_server:
                             self.context = zmq.Context()
                             self.socket = self.context.socket(zmq.REQ)
                             self.socket.connect(self.plot_server)
                             self.poll = zmq.Poller()
                             self.poll.register(self.socket, zmq.POLLIN)
                     if dataOut.type == 'Parameters':
                         tm = dataOut.utctimeInit
                     else:
                         tm = dataOut.utctime
                     if not dataOut.useLocalTime and self.localtime:
                         tm -= time.timezone
                     if dataOut.useLocalTime and not self.localtime:
                         tm += time.timezone
                     if self.xaxis is 'time' and self.data and (tm - self.tmin) >= self.xrange*60*60:
                         self.save_counter = self.save_period
                         self.__plot()
                         self.xmin += self.xrange
                         if self.xmin >= 24:
                             self.xmin -= 24
                         self.tmin += self.xrange*60*60
                         self.data.setup()
                         self.clear_figures()
                     self.data.update(dataOut, tm)
                     if self.isPlotConfig is False:
                         self.__setup_plot()
                         self.isPlotConfig = True
                     if self.realtime:
                         self.__plot()
                     else:
                         self.__throttle_plot(self.__plot)#, coerce=coerce)
                 def close(self):
                     if self.data:
                         self.save_counter = self.save_period
                         self.__plot()
                     if self.data and self.pause:
                         figpause(10)

schainpy/model/io/jroIO_base.py +352 -618

-            '''
+            """
             Created on Jul 2, 2014
             @author: roj-idl71
-            '''
+            """
             import os
             import sys
             import glob
             import time
             import numpy
             import fnmatch
             import inspect
             import time
             import datetime
-            import traceback
             import zmq
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
             from schainpy.utils import log
             import schainpy.admin
             LOCALTIME = True
+            DT_DIRECTIVES = {
+                '%Y': 4,
+                '%y': 2,
+                '%m': 2,
+                '%d': 2,
+                '%j': 3,
+                '%H': 2,
+                '%M': 2,
+                '%S': 2,
+                '%f': 6
+            }
             def isNumber(cad):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float(cad)
                     return True
                 except:
                     return False
             def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename, 'rb')
                 except IOError:
                     print("The file %s can't be opened" % (filename))
                     return 0
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print("Skipping the file %s because it has not a valid header" % (filename))
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isTimeInRange(thisTime, startTime, endTime):
                 if endTime >= startTime:
                     if (thisTime < startTime) or (thisTime > endTime):
                         return 0
                     return 1
                 else:
                     if (thisTime < startTime) and (thisTime > endTime):
                         return 0
                     return 1
             def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename, 'rb')
                 except IOError:
                     print("The file %s can't be opened" % (filename))
                     return None
                 firstBasicHeaderObj = BasicHeader(LOCALTIME)
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
                 processingHeaderObj = ProcessingHeader()
                 lastBasicHeaderObj = BasicHeader(LOCALTIME)
                 sts = firstBasicHeaderObj.read(fp)
                 if not(sts):
                     print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
                     return None
                 if not systemHeaderObj.read(fp):
                     return None
                 if not radarControllerHeaderObj.read(fp):
                     return None
                 if not processingHeaderObj.read(fp):
                     return None
                 filesize = os.path.getsize(filename)
                 offset = processingHeaderObj.blockSize + 24  # header size
                 if filesize <= offset:
                     print("[Reading] %s: This file has not enough data" % filename)
                     return None
                 fp.seek(-offset, 2)
                 sts = lastBasicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = lastBasicHeaderObj.datatime
                 thisTime_last_block = thisDatetime.time()
                 thisDatetime = firstBasicHeaderObj.datatime
                 thisDate = thisDatetime.date()
                 thisTime_first_block = thisDatetime.time()
                 # General case
                 #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                 #-----------o----------------------------o-----------
                 #       startTime                     endTime
                 if endTime >= startTime:
                     if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
                         return None
                     return thisDatetime
                 # If endTime < startTime then endTime belongs to the next day
                 #<<<<<<<<<<<o                            o>>>>>>>>>>>
                 #-----------o----------------------------o-----------
                 #        endTime                    startTime
                 if (thisDate == startDate) and (thisTime_last_block < startTime):
                     return None
                 if (thisDate == endDate) and (thisTime_first_block > endTime):
                     return None
                 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
                     return None
                 return thisDatetime
             def isFolderInDateRange(folder, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     folder            :    nombre completo del directorio.
                                            Su formato deberia ser "/path_root/?YYYYDDD"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el directorio no tiene el formato adecuado
                 """
                 basename = os.path.basename(folder)
                 if not isRadarFolder(basename):
                     print("The folder %s has not the rigth format" % folder)
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFolder(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def isFileInDateRange(filename, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                                            Su formato deberia ser "?YYYYDDDsss"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                                                 sss     :    set
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no tiene el formato adecuado
                 """
                 basename = os.path.basename(filename)
                 if not isRadarFile(basename):
                     print("The filename %s has not the rigth format" % filename)
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFile(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def getFileFromSet(path, ext, set):
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     try:
                         year = int(thisFile[1:5])
                         doy = int(thisFile[5:8])
                     except:
                         continue
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 myfile = fnmatch.filter(
                     validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
                 if len(myfile) != 0:
                     return myfile[0]
                 else:
                     filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
                     print('the filename %s does not exist' % filename)
                     print('...going to the last file: ')
                 if validFilelist:
                     validFilelist = sorted(validFilelist, key=str.lower)
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     year = thisFile[1:5]
                     if not isNumber(year):
                         continue
                     doy = thisFile[5:8]
                     if not isNumber(doy):
                         continue
                     year = int(year)
                     doy = int(doy)
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 if validFilelist:
                     validFilelist = sorted(validFilelist, key=str.lower)
                     return validFilelist[-1]
                 return None
-            def checkForRealPath(path, foldercounter, year, doy, set, ext):
-                """
-                Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
-                Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
-                el path exacto de un determinado file.
-                Example    :
-                    nombre correcto del file es  .../.../D2009307/P2009307367.ext
-                    Entonces la funcion prueba con las siguientes combinaciones
-                        .../.../y2009307367.ext
-                        .../.../Y2009307367.ext
-                        .../.../x2009307/y2009307367.ext
-                        .../.../x2009307/Y2009307367.ext
-                        .../.../X2009307/y2009307367.ext
-                        .../.../X2009307/Y2009307367.ext
-                    siendo para este caso, la ultima combinacion de letras, identica al file buscado
-                Return:
-                    Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
-                    caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
-                    para el filename
-                """
-                fullfilename = None
-                find_flag = False
-                filename = None
-                prefixDirList = [None, 'd', 'D']
-                if ext.lower() == ".r":  # voltage
-                    prefixFileList = ['d', 'D']
-                elif ext.lower() == ".pdata":  # spectra
-                    prefixFileList = ['p', 'P']
-                else:
-                    return None, filename
-                # barrido por las combinaciones posibles
-                for prefixDir in prefixDirList:
-                    thispath = path
-                    if prefixDir != None:
-                        # formo el nombre del directorio xYYYYDDD (x=d o x=D)
-                        if foldercounter == 0:
-                            thispath = os.path.join(path, "%s%04d%03d" %
-                                                    (prefixDir, year, doy))
-                        else:
-                            thispath = os.path.join(path, "%s%04d%03d_%02d" % (
-                                prefixDir, year, doy, foldercounter))
-                    for prefixFile in prefixFileList:  # barrido por las dos combinaciones posibles de "D"
-                        # formo el nombre del file xYYYYDDDSSS.ext
-                        filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
-                        fullfilename = os.path.join(
-                            thispath, filename)  # formo el path completo
-                        if os.path.exists(fullfilename):  # verifico que exista
-                            find_flag = True
-                            break
-                    if find_flag:
-                        break
-                if not(find_flag):
-                    return None, filename
-                return fullfilename, filename
             def isRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
             def isRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return 0
                 return 1
             def getDateFromRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
                 return thisDate
             def getDateFromRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
                 return thisDate
+            def parse_format(s, fmt):
-            class JRODataIO:
+                for i in range(fmt.count('%')):
+                    x = fmt.index('%')
+                    d = DT_DIRECTIVES[fmt[x:x+2]]
+                    fmt = fmt.replace(fmt[x:x+2], s[x:x+d])
+                return fmt
-                c = 3E8
+            class Reader(object):
+                c = 3E8
                 isConfig = False
-                basicHeaderObj = None
-                systemHeaderObj = None
-                radarControllerHeaderObj = None
-                processingHeaderObj = None
                 dtype = None
                 pathList = []
                 filenameList = []
+                datetimeList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagDiscontinuousBlock = 0
                 flagIsNewBlock = 0
+                flagNoMoreFiles = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
-            #     ippSeconds = None
                 fileSizeByHeader = None
+                fileIndex = -1
-                fileIndex = None
                 profileIndex = None
+                blockIndex = 0
-                blockIndex = None
+                nTotalBlocks = 0
-                nTotalBlocks = None
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
-                blocksize = None
                 getByBlock = False
+                path = None
-                def __init__(self):
+                startDate = None
+                endDate = None
-                    raise NotImplementedError
+                startTime = datetime.time(0, 0, 0)
+                endTime = datetime.time(23, 59, 59)
+                set = None
+                expLabel = ""
+                online = False
+                delay = 60
+                nTries = 3  # quantity tries
+                nFiles = 3  # number of files for searching
+                walk = True
+                getblock = False
+                nTxs = 1
+                realtime = False
+                blocksize = 0
+                blocktime = None
+                warnings = True
+                verbose = True
+                server = None
+                format = None
+                oneDDict = None
+                twoDDict = None
+                independentParam = None
+                filefmt = None
+                folderfmt = None
                 def run(self):
                     raise NotImplementedError
-                def getDtypeWidth(self):
-                    dtype_index = get_dtype_index(self.dtype)
-                    dtype_width = get_dtype_width(dtype_index)
-                    return dtype_width
                 def getAllowedArgs(self):
                     if hasattr(self, '__attrs__'):
                         return self.__attrs__
                     else:
                         return inspect.getargspec(self.run).args
+                def set_kwargs(self, **kwargs):
-            class JRODataReader(JRODataIO):
+                    for key, value in kwargs.items():
+                        setattr(self, key, value)
-                online = 0
+                def find_folders(self, path, startDate, endDate, folderfmt, last=False):
-                realtime = 0
+                    folders = [x for f in path.split(',')
+                               for x in os.listdir(f) if os.path.isdir(os.path.join(f, x))]
+                    folders.sort()
-                nReadBlocks = 0
+                    if last:
+                        folders = [folders[-1]]
-                delay = 10  # number of seconds waiting a new file
-                nTries = 3  # quantity tries
-                nFiles = 3  # number of files for searching
-                path = None
+                    for folder in folders:
+                        try:
-                foldercounter = 0
+                            dt = datetime.datetime.strptime(parse_format(folder, folderfmt), folderfmt).date()
+                            if dt >= startDate and dt <= endDate:
-                flagNoMoreFiles = 0
+                                yield os.path.join(path, folder)
+                            else:
-                datetimeList = []
+                                log.log('Skiping folder {}'.format(folder), self.name)
+                        except Exception as e:
-                __isFirstTimeOnline = 1
+                            log.log('Skiping folder {}'.format(folder), self.name)
+                            continue
-                __printInfo = True
+                    return
-                profileIndex = None
-                nTxs = 1
-                txIndex = None
-                # Added--------------------
-                selBlocksize = None
-                selBlocktime = None
-                def __init__(self):
+                def find_files(self, folders, ext, filefmt, startDate=None, endDate=None,
-                    """
+                               expLabel='', last=False):
-                    This class is used to find data files
-                    Example:
+                    for path in folders:
-                        reader = JRODataReader()
+                        files = glob.glob1(path, '*{}'.format(ext))
-                        fileList = reader.findDataFiles()
+                        files.sort()
+                        if last:
-                    """
+                            if files:
+                                fo = files[-1]
+                                try:
+                                    dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
+                                    yield os.path.join(path, expLabel, fo)
+                                except Exception as e:
                                     pass
+                                return None
-                def createObjByDefault(self):
-                    """
-                    """
-                    raise NotImplementedError
-                def getBlockDimension(self):
-                    raise NotImplementedError
-                def searchFilesOffLine(self,
-                                       path,
-                                       startDate=None,
-                                       endDate=None,
-                                       startTime=datetime.time(0, 0, 0),
-                                       endTime=datetime.time(23, 59, 59),
-                                       set=None,
-                                       expLabel='',
-                                       ext='.r',
-                                       cursor=None,
-                                       skip=None,
-                                       walk=True):
-                    self.filenameList = []
-                    self.datetimeList = []
-                    pathList = []
-                    dateList, pathList = self.findDatafiles(
-                        path, startDate, endDate, expLabel, ext, walk, include_path=True)
-                    if dateList == []:
-                        return [], []
-                    if len(dateList) > 1:
-                        print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
                             else:
-                        print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
+                                return None
-                    filenameList = []
-                    datetimeList = []
-                    for thisPath in pathList:
-                        fileList = glob.glob1(thisPath, "*%s" % ext)
-                        fileList.sort()
-                        for file in fileList:
-                            filename = os.path.join(thisPath, file)
-                            if not isFileInDateRange(filename, startDate, endDate):
-                                continue
-                            thisDatetime = isFileInTimeRange(
-                                filename, startDate, endDate, startTime, endTime)
-                            if not(thisDatetime):
+                        for fo in files:
+                            try:
+                                dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
+                                if dt >= startDate and dt <= endDate:
+                                    yield os.path.join(path, expLabel, fo)
+                                else:
+                                    log.log('Skiping file {}'.format(fo), self.name)
+                            except Exception as e:
+                                log.log('Skiping file {}'.format(fo), self.name)
                                 continue
-                            filenameList.append(filename)
+                def searchFilesOffLine(self, path, startDate, endDate,
-                            datetimeList.append(thisDatetime)
+                                       expLabel, ext, walk,
+                                       filefmt, folderfmt):
-                    if cursor is not None and skip is not None:
+                    """Search files in offline mode for the given arguments
-                        filenameList = filenameList[cursor * skip:cursor * skip + skip]
-                        datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
-                    if not(filenameList):
+                    Return:
-                        print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
+                        Generator of files
-                        return [], []
+                    """
-                    print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
-                    # for i in range(len(filenameList)):
-                    #     print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
-                    self.filenameList = filenameList
+                    if walk:
-                    self.datetimeList = datetimeList
+                        folders = self.find_folders(
+                            path, startDate, endDate, folderfmt)
+                    else:
+                        folders = path.split(',')
-                    return pathList, filenameList
+                    return self.find_files(
+                        folders, ext, filefmt, startDate, endDate, expLabel)
-                def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
+                def searchFilesOnLine(self, path, startDate, endDate,
-                    """
+                                      expLabel, ext, walk,
-                    Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
+                                      filefmt, folderfmt):
-                    devuelve el archivo encontrado ademas de otros datos.
+                    """Search for the last file of the last folder
-                    Input:
+                    Arguments:
                         path        :    carpeta donde estan contenidos los files que contiene data
                         expLabel    :    Nombre del subexperimento (subfolder)
                         ext         :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
-                        directory   :    eL directorio donde esta el file encontrado
+                        generator with the full path of last filename
-                        filename    :    el ultimo file de una determinada carpeta
-                        year        :    el anho
-                        doy         :    el numero de dia del anho
-                        set         :    el set del archivo
                     """
-                    if not os.path.isdir(path):
-                        return None, None, None, None, None, None
-                    dirList = []
-                    if not walk:
+                    if walk:
-                        fullpath = path
+                        folders = self.find_folders(
-                        foldercounter = 0
+                            path, startDate, endDate, folderfmt, last=True)
                     else:
-                        # Filtra solo los directorios
+                        folders = path.split(',')
-                        for thisPath in os.listdir(path):
-                            if not os.path.isdir(os.path.join(path, thisPath)):
-                                continue
-                            if not isRadarFolder(thisPath):
-                                continue
-                            dirList.append(thisPath)
+                    return self.find_files(
+                        folders, ext, filefmt, startDate, endDate, expLabel, last=True)
-                        if not(dirList):
-                            return None, None, None, None, None, None
-                        dirList = sorted(dirList, key=str.lower)
-                        doypath = dirList[-1]
+                def setNextFile(self):
-                        foldercounter = int(doypath.split('_')[1]) if len(
+                    """Set the next file to be readed open it and parse de file header"""
-                            doypath.split('_')) > 1 else 0
-                        fullpath = os.path.join(path, doypath, expLabel)
-                    print("[Reading] %s folder was found: " % (fullpath))
+                    if self.fp != None:
+                        self.fp.close()
-                    if set == None:
+                    if self.online:
-                        filename = getlastFileFromPath(fullpath, ext)
+                        newFile = self.setNextFileOnline()
                     else:
-                        filename = getFileFromSet(fullpath, ext, set)
+                        newFile = self.setNextFileOffline()
-                    if not(filename):
-                        return None, None, None, None, None, None
-                    print("[Reading] %s file was found" % (filename))
+                    if not(newFile):
+                        if self.online:
+                            raise schainpy.admin.SchainError('Time to wait for new files reach')
+                        else:
+                            if self.fileIndex == -1:
+                                raise schainpy.admin.SchainWarning('No files found in the given path')
+                            else:
+                                raise schainpy.admin.SchainWarning('No more files to read')
-                    if not(self.__verifyFile(os.path.join(fullpath, filename))):
+                    if not(self.verifyFile(self.filename)):
-                        return None, None, None, None, None, None
+                        self.setNextFile()
-                    year = int(filename[1:5])
+                    log.log('Opening file: %s' % self.filename, self.name)
-                    doy = int(filename[5:8])
-                    set = int(filename[8:11])
-                    return fullpath, foldercounter, filename, year, doy, set
+                    self.readFirstHeader()
+                    self.nReadBlocks = 0
-                def __setNextFileOffline(self):
+                def setNextFileOnline(self):
+                    """Check for the next file to be readed in online mode.
-                    idFile = self.fileIndex
+                    Set:
+                        self.filename
+                        self.fp
+                        self.filesize
-                    while (True):
+                    Return:
-                        idFile += 1
+                        boolean
-                        if not(idFile < len(self.filenameList)):
-                            self.flagNoMoreFiles = 1
-                            return 0
-                        filename = self.filenameList[idFile]
+                    """
+                    nextFile = True
+                    nextDay = False
-                        if not(self.__verifyFile(filename)):
+                    for nFiles in range(self.nFiles+1):
+                        for nTries in range(self.nTries):
+                            fullfilename, filename = self.checkForRealPath(nextFile, nextDay)
+                            if fullfilename is not None:
+                                break
+                            log.warning(
+                                "Waiting %0.2f sec for the next file: \"%s\" , try %02d ..." % (self.delay, filename, nTries + 1),
+                                self.name)
+                            time.sleep(self.delay)
+                            nextFile = False
                             continue
-                        fileSize = os.path.getsize(filename)
+                        if fullfilename:
-                        fp = open(filename, 'rb')
                             break
+                        self.nTries = 1
+                        nextFile = True
+                        if nFiles == (self.nFiles - 1):
+                            log.log('Trying with next day...', self.name)
+                            nextDay = True
+                    if fullfilename:
+                        self.fileSize = os.path.getsize(fullfilename)
+                        self.filename = fullfilename
                         self.flagIsNewFile = 1
-                    self.fileIndex = idFile
+                        if self.fp != None:
-                    self.filename = filename
+                            self.fp.close()
-                    self.fileSize = fileSize
+                        self.fp = open(fullfilename, 'rb')
-                    self.fp = fp
+                        self.flagNoMoreFiles = 0
+                        self.fileIndex += 1
+                        return 1
+                    else:
+                        return 0
+                def setNextFileOffline(self):
+                    """Open the next file to be readed in offline mode"""
+                    try:
+                        filename = next(self.filenameList)
+                        self.fileIndex +=1
+                    except StopIteration:
+                        self.flagNoMoreFiles = 1
+                        return 0
-            #         print "[Reading] Setting the file: %s"%self.filename
+                    self.filename = filename
+                    self.fileSize = os.path.getsize(filename)
+                    self.fp = open(filename, 'rb')
+                    self.flagIsNewFile = 1
                     return 1
-                def __setNextFileOnline(self):
+                def verifyFile(self, filename):
-                    """
+                    """Check for a valid file
-                    Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
-                    no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
-                    siguientes.
-                    Affected:
+                    Arguments:
-                        self.flagIsNewFile
+                        filename -- full path filename
-                        self.filename
-                        self.fileSize
-                        self.fp
-                        self.set
-                        self.flagNoMoreFiles
                     Return:
-: si luego de una busqueda del siguiente file valido este no pudo ser encontrado
+                        boolean
-: si el file fue abierto con exito y esta listo a ser leido
-                    Excepciones:
-                        Si un determinado file no puede ser abierto
                     """
-                    nFiles = 0
-                    fileOk_flag = False
-                    firstTime_flag = True
-                    self.set += 1
+                    return True
-                    if self.set > 999:
+                def checkForRealPath(self, nextFile, nextDay):
-                        self.set = 0
+                    """Check if the next file to be readed exists"""
-                        self.foldercounter += 1
-                    # busca el 1er file disponible
+                    raise NotImplementedError
-                    fullfilename, filename = checkForRealPath(
-                        self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
-                    if fullfilename:
-                        if self.__verifyFile(fullfilename, False):
-                            fileOk_flag = True
-                    # si no encuentra un file entonces espera y vuelve a buscar
+                def readFirstHeader(self):
-                    if not(fileOk_flag):
+                    """Parse the file header"""
-                        # busco en los siguientes self.nFiles+1 files posibles
-                        for nFiles in range(self.nFiles + 1):
-                            if firstTime_flag:  # si es la 1era vez entonces hace el for self.nTries veces
+                    pass
-                                tries = self.nTries
-                            else:
-                                tries = 1  # si no es la 1era vez entonces solo lo hace una vez
-                            for nTries in range(tries):
+            class JRODataReader(Reader):
-                                if firstTime_flag:
-                                    log.warning(
-                                        "Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1),
-                                        self.name)
-                                    time.sleep(self.delay)
-                                else:
-                                    log.warning(
-                                        "Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext),
-                                        self.name)
-                                fullfilename, filename = checkForRealPath(
+                utc = 0
-                                    self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
+                nReadBlocks = 0
-                                if fullfilename:
+                foldercounter = 0
-                                    if self.__verifyFile(fullfilename):
+                firstHeaderSize = 0
-                                        fileOk_flag = True
+                basicHeaderSize = 24
-                                        break
+                __isFirstTimeOnline = 1
+                __printInfo = True
+                filefmt = "*%Y%j***"
+                folderfmt = "*%Y%j"
-                            if fileOk_flag:
+                def getDtypeWidth(self):
-                                break
-                            firstTime_flag = False
+                    dtype_index = get_dtype_index(self.dtype)
+                    dtype_width = get_dtype_width(dtype_index)
-                            log.warning(
+                    return dtype_width
-                                'Skipping the file {} due to this file doesn\'t exist'.format(filename),
-                                self.name)
-                            self.set += 1
-                            # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
+                def checkForRealPath(self, nextFile, nextDay):
-                            if nFiles == (self.nFiles - 1):
+                    """Check if the next file to be readed exists.
-                                self.set = 0
-                                self.doy += 1
-                                self.foldercounter = 0
-                    if fileOk_flag:
+                    Example    :
-                        self.fileSize = os.path.getsize(fullfilename)
+                        nombre correcto del file es  .../.../D2009307/P2009307367.ext
-                        self.filename = fullfilename
-                        self.flagIsNewFile = 1
-                        if self.fp != None:
-                            self.fp.close()
-                        self.fp = open(fullfilename, 'rb')
-                        self.flagNoMoreFiles = 0
-                    else:
-                        raise schainpy.admin.SchainError('Time for waiting new files reach')
-                        self.fileSize = 0
-                        self.filename = None
-                        self.flagIsNewFile = 0
-                        self.fp = None
-                        self.flagNoMoreFiles = 1
-                    return fileOk_flag
+                        Entonces la funcion prueba con las siguientes combinaciones
+                            .../.../y2009307367.ext
+                            .../.../Y2009307367.ext
+                            .../.../x2009307/y2009307367.ext
+                            .../.../x2009307/Y2009307367.ext
+                            .../.../X2009307/y2009307367.ext
+                            .../.../X2009307/Y2009307367.ext
+                        siendo para este caso, la ultima combinacion de letras, identica al file buscado
-                def setNextFile(self):
+                    Return:
-                    if self.fp != None:
+                        str -- fullpath of the file
-                        self.fp.close()
+                    """
-                    if self.online:
-                        newFile = self.__setNextFileOnline()
-                    else:
-                        newFile = self.__setNextFileOffline()
-                    if not(newFile):
+                    if nextFile:
-                        raise schainpy.admin.SchainWarning('No more files to read')
+                        self.set += 1
+                    if nextDay:
+                        self.set = 0
+                        self.doy += 1
+                    foldercounter = 0
+                    prefixDirList = [None, 'd', 'D']
+                    if self.ext.lower() == ".r":  # voltage
+                        prefixFileList = ['d', 'D']
+                    elif self.ext.lower() == ".pdata":  # spectra
+                        prefixFileList = ['p', 'P']
+                    # barrido por las combinaciones posibles
+                    for prefixDir in prefixDirList:
+                        thispath = self.path
+                        if prefixDir != None:
+                            # formo el nombre del directorio xYYYYDDD (x=d o x=D)
+                            if foldercounter == 0:
+                                thispath = os.path.join(self.path, "%s%04d%03d" %
+                                                        (prefixDir, self.year, self.doy))
+                            else:
+                                thispath = os.path.join(self.path, "%s%04d%03d_%02d" % (
+                                    prefixDir, self.year, self.doy, foldercounter))
+                        for prefixFile in prefixFileList:  # barrido por las dos combinaciones posibles de "D"
+                            # formo el nombre del file xYYYYDDDSSS.ext
+                            filename = "%s%04d%03d%03d%s" % (prefixFile, self.year, self.doy, self.set, self.ext)
+                            fullfilename = os.path.join(
+                                thispath, filename)
-                    if self.verbose:
+                            if os.path.exists(fullfilename):
-                        print('[Reading] Setting the file: %s' % self.filename)
+                                return fullfilename, filename
-                    self.__readFirstHeader()
+                    return None, filename
-                    self.nReadBlocks = 0
-                    return 1
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range(self.nTries):
                         self.fp.close()
                         self.fp = open(self.filename, 'rb')
                         self.fp.seek(currentPointer)
                         self.fileSize = os.path.getsize(self.filename)
                         currentSize = self.fileSize - currentPointer
                         if (currentSize >= neededSize):
                             self.basicHeaderObj.read(self.fp)
                             return 1
                         if self.fileSize == self.fileSizeByHeader:
                             #                self.flagEoF = True
                             return 0
                         print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
                         time.sleep(self.delay)
                     return 0
                 def waitDataBlock(self, pointer_location, blocksize=None):
                     currentPointer = pointer_location
                     if blocksize is None:
                         neededSize = self.processingHeaderObj.blockSize  # + self.basicHeaderSize
                     else:
                         neededSize = blocksize
                     for nTries in range(self.nTries):
                         self.fp.close()
                         self.fp = open(self.filename, 'rb')
                         self.fp.seek(currentPointer)
                         self.fileSize = os.path.getsize(self.filename)
                         currentSize = self.fileSize - currentPointer
                         if (currentSize >= neededSize):
                             return 1
                         log.warning(
                             "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
                             self.name
                             )
                         time.sleep(self.delay)
                     return 0
-                def __jumpToLastBlock(self):
-                    if not(self.__isFirstTimeOnline):
-                        return
-                    csize = self.fileSize - self.fp.tell()
-                    blocksize = self.processingHeaderObj.blockSize
-                    # salta el primer bloque de datos
-                    if csize > self.processingHeaderObj.blockSize:
-                        self.fp.seek(self.fp.tell() + blocksize)
-                    else:
-                        return
-                    csize = self.fileSize - self.fp.tell()
-                    neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
-                    while True:
-                        if self.fp.tell() < self.fileSize:
-                            self.fp.seek(self.fp.tell() + neededsize)
-                        else:
-                            self.fp.seek(self.fp.tell() - neededsize)
-                            break
-            #        csize = self.fileSize - self.fp.tell()
-            #        neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
-            #        factor = int(csize/neededsize)
-            #        if factor > 0:
-            #            self.fp.seek(self.fp.tell() + factor*neededsize)
-                    self.flagIsNewFile = 0
-                    self.__isFirstTimeOnline = 0
                 def __setNewBlock(self):
-                    # if self.server is None:
                     if self.fp == None:
                         return 0
-            #         if self.online:
-            #             self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if self.realtime:
                         self.flagDiscontinuousBlock = 1
                         if not(self.setNextFile()):
                             return 0
                         else:
                             return 1
-                    # if self.server is None:
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.basicHeaderObj.read(self.fp)
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
-                    # else:
-                    #     self.basicHeaderObj.read(self.zHeader)
-                    #     self.lastUTTime = self.basicHeaderObj.utc
-                    #     return 1
                     if self.__waitNewBlock():
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
-                    # if self.server is None:
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime
                     self.lastUTTime = self.basicHeaderObj.utc
                     self.flagDiscontinuousBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagDiscontinuousBlock = 1
                     return 1
                 def readNextBlock(self):
-                    # Skip block out of startTime and endTime
                     while True:
-                        if not(self.__setNewBlock()):
+                        self.__setNewBlock()
-                            raise schainpy.admin.SchainWarning('No more files to read')
                         if not(self.readBlock()):
                             return 0
                         self.getBasicHeader()
                         if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
                             print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
                                                                                   self.processingHeaderObj.dataBlocksPerFile,
                                                                                   self.dataOut.datatime.ctime()))
                             continue
                         break
                     if self.verbose:
                         print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
                                                                    self.processingHeaderObj.dataBlocksPerFile,
                                                                    self.dataOut.datatime.ctime()))
                     return 1
-                def __readFirstHeader(self):
+                def readFirstHeader(self):
                     self.basicHeaderObj.read(self.fp)
                     self.systemHeaderObj.read(self.fp)
                     self.radarControllerHeaderObj.read(self.fp)
                     self.processingHeaderObj.read(self.fp)
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
                                                PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
                     else:
                         raise ValueError('Data type was not defined')
                     self.dtype = datatype_str
                     #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
                         self.firstHeaderSize + self.basicHeaderSize * \
                         (self.processingHeaderObj.dataBlocksPerFile - 1)
                     #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
                     #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
-                def __verifyFile(self, filename, msgFlag=True):
+                def verifyFile(self, filename, msgFlag=True):
                     msg = None
                     try:
                         fp = open(filename, 'rb')
                     except IOError:
                         if msgFlag:
                             print("[Reading] File %s can't be opened" % (filename))
                         return False
                     currentPosition = fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                     if neededSize == 0:
                         basicHeaderObj = BasicHeader(LOCALTIME)
                         systemHeaderObj = SystemHeader()
                         radarControllerHeaderObj = RadarControllerHeader()
                         processingHeaderObj = ProcessingHeader()
                         if not(basicHeaderObj.read(fp)):
                             fp.close()
                             return False
                         if not(systemHeaderObj.read(fp)):
                             fp.close()
                             return False
                         if not(radarControllerHeaderObj.read(fp)):
                             fp.close()
                             return False
                         if not(processingHeaderObj.read(fp)):
                             fp.close()
                             return False
                         neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
                     else:
                         msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
                     fp.close()
                     fileSize = os.path.getsize(filename)
                     currentSize = fileSize - currentPosition
                     if currentSize < neededSize:
                         if msgFlag and (msg != None):
                             print(msg)
                         return False
                     return True
                 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
                     path_empty = True
                     dateList = []
                     pathList = []
                     multi_path = path.split(',')
                     if not walk:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             fileList = glob.glob1(single_path, "*" + ext)
                             if not fileList:
                                 continue
                             path_empty = False
                             fileList.sort()
                             for thisFile in fileList:
                                 if not os.path.isfile(os.path.join(single_path, thisFile)):
                                     continue
                                 if not isRadarFile(thisFile):
                                     continue
                                 if not isFileInDateRange(thisFile, startDate, endDate):
                                     continue
                                 thisDate = getDateFromRadarFile(thisFile)
                                 if thisDate in dateList or single_path in pathList:
                                     continue
                                 dateList.append(thisDate)
                                 pathList.append(single_path)
                     else:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             dirList = []
                             for thisPath in os.listdir(single_path):
                                 if not os.path.isdir(os.path.join(single_path, thisPath)):
                                     continue
                                 if not isRadarFolder(thisPath):
                                     continue
                                 if not isFolderInDateRange(thisPath, startDate, endDate):
                                     continue
                                 dirList.append(thisPath)
                             if not dirList:
                                 continue
                             dirList.sort()
                             for thisDir in dirList:
                                 datapath = os.path.join(single_path, thisDir, expLabel)
                                 fileList = glob.glob1(datapath, "*" + ext)
                                 if not fileList:
                                     continue
                                 path_empty = False
                                 thisDate = getDateFromRadarFolder(thisDir)
                                 pathList.append(datapath)
                                 dateList.append(thisDate)
                     dateList.sort()
                     if walk:
                         pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
                     else:
                         pattern_path = multi_path[0]
                     if path_empty:
                         raise schainpy.admin.SchainError("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
                     else:
                         if not dateList:
                             raise schainpy.admin.SchainError("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
                     if include_path:
                         return dateList, pathList
                     return dateList
-                def setup(self,
+                def setup(self, **kwargs):
-                          path=None,
-                          startDate=None,
+                    self.set_kwargs(**kwargs)
-                          endDate=None,
+                    if not self.ext.startswith('.'):
-                          startTime=datetime.time(0, 0, 0),
+                        self.ext = '.{}'.format(self.ext)
-                          endTime=datetime.time(23, 59, 59),
-                          set=None,
+                    if self.server is not None:
-                          expLabel="",
+                        if 'tcp://' in self.server:
-                          ext=None,
-                          online=False,
-                          delay=60,
-                          walk=True,
-                          getblock=False,
-                          nTxs=1,
-                          realtime=False,
-                          blocksize=None,
-                          blocktime=None,
-                          skip=None,
-                          cursor=None,
-                          warnings=True,
-                          verbose=True,
-                          server=None,
-                          format=None,
-                          oneDDict=None,
-                          twoDDict=None,
-                          independentParam=None):
-                    self.online = online
-                    self.realtime = realtime
-                    self.delay = delay
-                    self.getByBlock = getblock
-                    self.nTxs = nTxs
-                    self.startTime = startTime
-                    self.endTime = endTime
-                    self.endDate = endDate
-                    self.startDate = startDate
-                    if server is not None:
-                        if 'tcp://' in server:
                             address = server
                         else:
-                            address = 'ipc:///tmp/%s' % server
+                            address = 'ipc:///tmp/%s' % self.server
                         self.server = address
                         self.context = zmq.Context()
                         self.receiver = self.context.socket(zmq.PULL)
                         self.receiver.connect(self.server)
                         time.sleep(0.5)
                         print('[Starting] ReceiverData from {}'.format(self.server))
                     else:
                         self.server = None
-                        if path == None:
+                        if self.path == None:
                             raise ValueError("[Reading] The path is not valid")
-                        if ext == None:
+                        if self.online:
-                            ext = self.ext
+                            log.log("[Reading] Searching files in online mode...", self.name)
-                        if online:
-                            print("[Reading] Searching files in online mode...")
                             for nTries in range(self.nTries):
-                                fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
+                                fullpath = self.searchFilesOnLine(self.path, self.startDate,
-                                    path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
+                                    self.endDate, self.expLabel, self.ext, self.walk,
+                                    self.filefmt, self.folderfmt)
+                                try:
+                                    fullpath = next(fullpath)
+                                except:
+                                    fullpath = None
                                 if fullpath:
                                     break
-                                print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
+                                log.warning(
+                                    'Waiting {} sec for a valid file in {}: try {} ...'.format(
+                                        self.delay, self.path, nTries + 1),
+                                    self.name)
                                 time.sleep(self.delay)
                             if not(fullpath):
-                                raise schainpy.admin.SchainError('There isn\'t any valid file in {}'.format(path))
+                                raise schainpy.admin.SchainError(
-                                return
+                                    'There isn\'t any valid file in {}'.format(self.path))
-                            self.year = year
+                            pathname, filename = os.path.split(fullpath)
-                            self.doy = doy
+                            self.year = int(filename[1:5])
-                            self.set = set - 1
+                            self.doy = int(filename[5:8])
-                            self.path = path
+                            self.set = int(filename[8:11]) - 1
-                            self.foldercounter = foldercounter
-                            last_set = None
-                        else:
-                            print("[Reading] Searching files in offline mode ...")
-                            pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
-                                                                             startTime=startTime, endTime=endTime,
-                                                                             set=set, expLabel=expLabel, ext=ext,
-                                                                             walk=walk, cursor=cursor,
-                                                                             skip=skip)
-                            if not(pathList):
-                                self.fileIndex = -1
-                                self.pathList = []
-                                self.filenameList = []
-                                return
-                            self.fileIndex = -1
-                            self.pathList = pathList
-                            self.filenameList = filenameList
-                            file_name = os.path.basename(filenameList[-1])
-                            basename, ext = os.path.splitext(file_name)
-                            last_set = int(basename[-3:])
-                        ext = ext.lower()
-                        self.ext = ext
-                        # Added-----------------
-                        self.selBlocksize = blocksize
-                        self.selBlocktime = blocktime
-                        # Verbose-----------
-                        self.verbose = verbose
-                        self.warnings = warnings
-                        if not(self.setNextFile()):
-                            if (startDate != None) and (endDate != None):
-                                print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
-                            elif startDate != None:
-                                print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
                         else:
-                                print("[Reading] No files")
+                            log.log("Searching files in {}".format(self.path), self.name)
+                            self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
+                                self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
-                                self.fileIndex = -1
+                        self.setNextFile()
-                                self.pathList = []
-                                self.filenameList = []
-                                return
-                        if last_set != None:
-                            self.dataOut.last_block = last_set * \
-                                self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
                     return
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
 . + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                     self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
              #         self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
                 def getFirstHeader(self):
                     raise NotImplementedError
                 def getData(self):
                     raise NotImplementedError
                 def hasNotDataInBuffer(self):
                     raise NotImplementedError
                 def readBlock(self):
                     raise NotImplementedError
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
                 def printTotalBlocks(self):
                     print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
                 def printNumberOfBlock(self):
                     'SPAM!'
              #         if self.flagIsNewBlock:
              #             print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
              #                                                       self.processingHeaderObj.dataBlocksPerFile,
              #                                                       self.dataOut.datatime.ctime())
                 def printInfo(self):
                     if self.__printInfo == False:
                         return
                     self.basicHeaderObj.printInfo()
                     self.systemHeaderObj.printInfo()
                     self.radarControllerHeaderObj.printInfo()
                     self.processingHeaderObj.printInfo()
                     self.__printInfo = False
-                def run(self,
+                def run(self, **kwargs):
-                        path=None,
+                    """
-                        startDate=None,
-                        endDate=None,
+                    Arguments:
-                        startTime=datetime.time(0, 0, 0),
+                        path        :
-                        endTime=datetime.time(23, 59, 59),
+                        startDate   :
-                        set=None,
+                        endDate     :
-                        expLabel="",
+                        startTime   :
-                        ext=None,
+                        endTime     :
-                        online=False,
+                        set         :
-                        delay=60,
+                        expLabel    :
-                        walk=True,
+                        ext         :
-                        getblock=False,
+                        online      :
-                        nTxs=1,
+                        delay       :
-                        realtime=False,
+                        walk        :
-                        blocksize=None,
+                        getblock    :
-                        blocktime=None,
+                        nTxs        :
-                        skip=None,
+                        realtime    :
-                        cursor=None,
+                        blocksize   :
-                        warnings=True,
+                        blocktime   :
-                        server=None,
+                        skip        :
-                        verbose=True,
+                        cursor      :
-                        format=None,
+                        warnings    :
-                        oneDDict=None,
+                        server      :
-                        twoDDict=None,
+                        verbose     :
-                        independentParam=None, **kwargs):
+                        format      :
+                        oneDDict    :
+                        twoDDict    :
+                        independentParam    :
+                    """
                     if not(self.isConfig):
-                        self.setup(path=path,
+                        self.setup(**kwargs)
-                                   startDate=startDate,
-                                   endDate=endDate,
-                                   startTime=startTime,
-                                   endTime=endTime,
-                                   set=set,
-                                   expLabel=expLabel,
-                                   ext=ext,
-                                   online=online,
-                                   delay=delay,
-                                   walk=walk,
-                                   getblock=getblock,
-                                   nTxs=nTxs,
-                                   realtime=realtime,
-                                   blocksize=blocksize,
-                                   blocktime=blocktime,
-                                   skip=skip,
-                                   cursor=cursor,
-                                   warnings=warnings,
-                                   server=server,
-                                   verbose=verbose,
-                                   format=format,
-                                   oneDDict=oneDDict,
-                                   twoDDict=twoDDict,
-                                   independentParam=independentParam)
                         self.isConfig = True
-                    if server is None:
+                    if self.server is None:
                         self.getData()
                     else:
                         self.getFromServer()
-            class JRODataWriter(JRODataIO):
+            class JRODataWriter(Reader):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
-                blockIndex = 0
-                path = None
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 fileDate = None
                 def __init__(self, dataOut=None):
                     raise NotImplementedError
                 def hasAllDataInBuffer(self):
                     raise NotImplementedError
                 def setBlockDimension(self):
                     raise NotImplementedError
                 def writeBlock(self):
                     raise NotImplementedError
                 def putData(self):
                     raise NotImplementedError
+                def getDtypeWidth(self):
+                    dtype_index = get_dtype_index(self.dtype)
+                    dtype_width = get_dtype_width(dtype_index)
+                    return dtype_width
                 def getProcessFlags(self):
                     processFlags = 0
                     dtype_index = get_dtype_index(self.dtype)
                     procflag_dtype = get_procflag_dtype(dtype_index)
                     processFlags += procflag_dtype
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code is not None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nCohInt > 1:
                         processFlags += PROCFLAG.COHERENT_INTEGRATION
                     if self.dataOut.type == "Spectra":
                         if self.dataOut.nIncohInt > 1:
                             processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                         if self.dataOut.data_dc is not None:
                             processFlags += PROCFLAG.SAVE_CHANNELS_DC
                         if self.dataOut.flagShiftFFT:
                             processFlags += PROCFLAG.SHIFT_FFT_DATA
                     return processFlags
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize  # bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond = (self.dataOut.utctime - utc) * 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
                     raise NotImplementedError
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + \
                         self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not(self.__setNewBlock()):
                         return 0
                     self.writeBlock()
                     print("[Writing] Block No. %d/%d" % (self.blockIndex,
                                                          self.processingHeaderObj.dataBlocksPerFile))
                     return 1
                 def setNextFile(self):
-                    """
+                    """Determina el siguiente file que sera escrito
-                    Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
+                    if not os.path.exists(path):
+                        os.mkdir(path)
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     setFile = self.setFile
                     if not(os.path.exists(fullpath)):
                         os.mkdir(fullpath)
                         setFile = -1  # inicializo mi contador de seteo
                     else:
                         filesList = os.listdir(fullpath)
                         if len(filesList) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 # inicializo mi contador de seteo al seteo del ultimo file
                                 setFile = int(filen[8:11])
                             else:
                                 setFile = -1
                         else:
                             setFile = -1  # inicializo mi contador de seteo
                     setFile += 1
                     # If this is a new day it resets some values
                     if self.dataOut.datatime.date() > self.fileDate:
                         setFile = 0
                         self.nTotalBlocks = 0
                     filen = '{}{:04d}{:03d}{:03d}{}'.format(
                             self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
                     filename = os.path.join(path, subfolder, filen)
                     fp = open(filename, 'wb')
                     self.blockIndex = 0
-                    # guardando atributos
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.fileDate = self.dataOut.datatime.date()
                     self.setFirstHeader()
                     print('[Writing] Opening file: %s' % self.filename)
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path                :    directory where data will be saved
                         profilesPerBlock    :    number of profiles per block
                         set                 :    initial file set
                         datatype            :    An integer number that defines data type:
 : int8  (1 byte)
 : int16 (2 bytes)
 : int32 (4 bytes)
 : int64 (8 bytes)
 : float32 (4 bytes)
 : double64 (8 bytes)
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     self.ext = ext.lower()
                     self.path = path
                     if set is None:
                         self.setFile = -1
                     else:
                         self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     self.fileDate = self.dataOut.datatime.date()
-                    # By default
                     self.dtype = self.dataOut.dtype
                     if datatype is not None:
                         self.dtype = get_numpy_dtype(datatype)
                     if not(self.setNextFile()):
                         print("[Writing] There isn't a next file")
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
                                    set=set, ext=ext, datatype=datatype, **kwargs)
                         self.isConfig = True
                     self.dataOut = dataOut
                     self.putData()
                     return self.dataOut

schainpy/model/io/jroIO_spectra.py +14 -162

             '''
             Created on Jul 2, 2014
             @author: roj-idl71
             '''
             import numpy
             from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jrodata import Spectra
             from schainpy.utils import log
             @MPDecorator
             class SpectraReader(JRODataReader, ProcessingUnit):
                 """
                 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
                 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
                                     paresCanalesIguales    * alturas * perfiles  (Self Spectra)
                                     paresCanalesDiferentes * alturas * perfiles  (Cross Spectra)
                                     canales * alturas                            (DC Channels)
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = SpectraReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         readerObj.getData()
                         print readerObj.data_spc
                         print readerObj.data_cspc
                         print readerObj.data_dc
                         if readerObj.flagNoMoreFiles:
                             break
                 """
-                pts2read_SelfSpectra = 0
-                pts2read_CrossSpectra = 0
-                pts2read_DCchannels = 0
-                ext = ".pdata"
-                optchar = "P"
-                dataOut = None
-                nRdChannels = None
-                nRdPairs = None
-                rdPairList = []
                 def __init__(self):#, **kwargs):
                     """
                     Inicializador de la clase SpectraReader para la lectura de datos de espectros.
                     Inputs:
                         dataOut    :    Objeto de la clase Spectra. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Affected:
                         self.dataOut
                     Return      : None
                     """
-                    #Eliminar de la base la herencia
+                    ProcessingUnit.__init__(self)
-                    ProcessingUnit.__init__(self)#, **kwargs)
                     self.pts2read_SelfSpectra = 0
                     self.pts2read_CrossSpectra = 0
                     self.pts2read_DCchannels = 0
-                    self.datablock = None
-                    self.utc = None
                     self.ext = ".pdata"
                     self.optchar = "P"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
-                    self.online = 0
-                    self.fp = None
-                    self.idFile = None
-                    self.dtype = None
-                    self.fileSizeByHeader = None
-                    self.filenameList = []
-                    self.filename = None
-                    self.fileSize = None
-                    self.firstHeaderSize = 0
-                    self.basicHeaderSize = 24
-                    self.pathList = []
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
+                    self.dataOut = Spectra()
-                    self.flagNoMoreFiles = 0
+                    self.profileIndex = 1
+                    self.nRdChannels = None
-                    self.set = 0
+                    self.nRdPairs = None
+                    self.rdPairList = []
-                    self.path = None
-                    self.delay  = 60   #seconds
-                    self.nTries = 3  #quantity tries
-                    self.nFiles = 3   #number of files for searching
-                    self.nReadBlocks = 0
-                    self.flagIsNewFile = 1
-                    self.__isFirstTimeOnline = 1
-                    self.flagDiscontinuousBlock = 0
-                    self.flagIsNewBlock = 0
-                    self.nTotalBlocks = 0
-                    self.blocksize = 0
-                    self.dataOut = self.createObjByDefault()
-                    self.profileIndex = 1 #Always
                 def createObjByDefault(self):
                     dataObj = Spectra()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     return 1
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.nRdChannels
                         self.nRdPairs
                         self.pts2read_SelfSpectra
                         self.pts2read_CrossSpectra
                         self.pts2read_DCchannels
                         self.blocksize
                         self.dataOut.nChannels
                         self.dataOut.nPairs
                     Return:
                         None
                     """
                     self.nRdChannels = 0
                     self.nRdPairs = 0
                     self.rdPairList = []
                     for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
                         if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
                             self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
                         else:
                             self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
                             self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
                     pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
                     self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
                     self.blocksize = self.pts2read_SelfSpectra
                     if self.processingHeaderObj.flag_cspc:
                         self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
                         self.blocksize += self.pts2read_CrossSpectra
                     if self.processingHeaderObj.flag_dc:
                         self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
                         self.blocksize += self.pts2read_DCchannels
-                    # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
                 def readBlock(self):
                     """
                     Lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Return: None
                     Variables afectadas:
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
-                    blockOk_flag = False
                     fpointer = self.fp.tell()
                     spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
                     spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
                         cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_dc:
                         dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
                         dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
                     if not self.processingHeaderObj.shif_fft:
                         #desplaza a la derecha en el eje 2 determinadas posiciones
                         shift = int(self.processingHeaderObj.profilesPerBlock/2)
                         spc = numpy.roll( spc, shift , axis=2 )
                         if self.processingHeaderObj.flag_cspc:
                             #desplaza a la derecha en el eje 2 determinadas posiciones
                             cspc = numpy.roll( cspc, shift, axis=2 )
                     #Dimensions : nChannels, nProfiles, nSamples
                     spc = numpy.transpose( spc, (0,2,1) )
                     self.data_spc = spc
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.transpose( cspc, (0,2,1) )
                         self.data_cspc = cspc['real'] + cspc['imag']*1j
                     else:
                         self.data_cspc = None
                     if self.processingHeaderObj.flag_dc:
                         self.data_dc = dc['real'] + dc['imag']*1j
                     else:
                         self.data_dc = None
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.getBasicHeader()
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
-            #         self.dataOut.ippSeconds = self.ippSeconds
-            #         self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.processingHeaderObj.profilesPerBlock
                     self.dataOut.dtype = self.dtype
-            #         self.dataOut.nPairs = self.nPairs
                     self.dataOut.pairsList = self.rdPairList
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
                     self.dataOut.flagShiftFFT = True    #Data is always shifted
                     self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
                 def getData(self):
                     """
                     First method to execute before "RUN" is called.
                     Copia el buffer de lectura a la clase "Spectra",
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Return:
 :    Si no hay mas archivos disponibles
 :    Si hizo una buena copia del buffer
                     Affected:
                         self.dataOut
                         self.flagDiscontinuousBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         return 0
                     self.flagDiscontinuousBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             self.dataOut.flagNoData = True
                             return 0
                     #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
                     if self.data_spc is None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.getBasicHeader()
                     self.getFirstHeader()
                     self.dataOut.data_spc = self.data_spc
                     self.dataOut.data_cspc = self.data_cspc
                     self.dataOut.data_dc = self.data_dc
                     self.dataOut.flagNoData = False
                     self.dataOut.realtime = self.online
                     return self.dataOut.data_spc
             @MPDecorator
             class SpectraWriter(JRODataWriter, Operation):
                 """
                 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
-                ext = ".pdata"
-                optchar = "P"
-                shape_spc_Buffer = None
-                shape_cspc_Buffer = None
-                shape_dc_Buffer = None
-                data_spc = None
-                data_cspc = None
-                data_dc = None
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj
                     Return: None
                     """
                     Operation.__init__(self)
-                    self.nTotalBlocks = 0
+                    self.ext = ".pdata"
+                    self.optchar = "P"
+                    self.shape_spc_Buffer = None
+                    self.shape_cspc_Buffer = None
+                    self.shape_dc_Buffer = None
                     self.data_spc = None
                     self.data_cspc = None
                     self.data_dc = None
-                    self.fp = None
-                    self.flagIsNewFile = 1
-                    self.nTotalBlocks = 0
-                    self.flagIsNewBlock = 0
                     self.setFile = None
-                    self.dtype = None
-                    self.path = None
                     self.noMoreFiles = 0
-                    self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     return 1
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shape_spc_Buffer = (self.dataOut.nChannels,
                                              self.processingHeaderObj.nHeights,
                                              self.processingHeaderObj.profilesPerBlock)
                     self.shape_cspc_Buffer = (self.dataOut.nPairs,
                                               self.processingHeaderObj.nHeights,
                                               self.processingHeaderObj.profilesPerBlock)
                     self.shape_dc_Buffer = (self.dataOut.nChannels,
                                             self.processingHeaderObj.nHeights)
                 def writeBlock(self):
                     """processingHeaderObj
                     Escribe el buffer en el file designado
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.nWriteBlocks
                     Return: None
                     """
                     spc = numpy.transpose( self.data_spc, (0,2,1) )
                     if not self.processingHeaderObj.shif_fft:
                         spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                     data = spc.reshape((-1))
                     data = data.astype(self.dtype[0])
                     data.tofile(self.fp)
                     if self.data_cspc is not None:
                         cspc = numpy.transpose( self.data_cspc, (0,2,1) )
                         data = numpy.zeros( numpy.shape(cspc), self.dtype )
                         #print 'data.shape', self.shape_cspc_Buffer
                         if not self.processingHeaderObj.shif_fft:
                             cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                         data['real'] = cspc.real
                         data['imag'] = cspc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     if self.data_dc is not None:
                         dc = self.data_dc
                         data = numpy.zeros( numpy.shape(dc), self.dtype )
                         data['real'] = dc.real
                         data['imag'] = dc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
             #         self.data_spc.fill(0)
             #
             #         if self.data_dc is not None:
             #             self.data_dc.fill(0)
             #
             #         if self.data_cspc is not None:
             #             self.data_cspc.fill(0)
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nWriteBlocks += 1
                     self.blockIndex += 1
             #         print "[Writing] Block = %d04" %self.blockIndex
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagDiscontinuousBlock:
                         self.data_spc.fill(0)
                         if self.dataOut.data_cspc is not None:
                             self.data_cspc.fill(0)
                         if self.dataOut.data_dc is not None:
                             self.data_dc.fill(0)
                         self.setNextFile()
                     if self.flagIsNewFile == 0:
                         self.setBasicHeader()
                     self.data_spc = self.dataOut.data_spc.copy()
                     if self.dataOut.data_cspc is not None:
                         self.data_cspc = self.dataOut.data_cspc.copy()
                     if self.dataOut.data_dc is not None:
                         self.data_dc = self.dataOut.data_dc.copy()
                     # #self.processingHeaderObj.dataBlocksPerFile)
                     if self.hasAllDataInBuffer():
             #            self.setFirstHeader()
                         self.writeNextBlock()
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
                     '''
                     dtype_width = self.getDtypeWidth()
                     pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
                     pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
                     blocksize = (pts2write_SelfSpectra*dtype_width)
                     if self.dataOut.data_cspc is not None:
                         pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
                         blocksize += (pts2write_CrossSpectra*dtype_width*2)
                     if self.dataOut.data_dc is not None:
                         pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
                         blocksize += (pts2write_DCchannels*dtype_width*2)
             #         blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.processingHeaderObj.dtype = 1 # Spectra
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
                     self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
                     self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
                     self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
                     if self.processingHeaderObj.totalSpectra > 0:
                         channelList = []
                         for channel in range(self.dataOut.nChannels):
                             channelList.append(channel)
                             channelList.append(channel)
                         pairsList = []
                         if self.dataOut.nPairs > 0:
                             for pair in self.dataOut.pairsList:
                                 pairsList.append(pair[0])
                                 pairsList.append(pair[1])
                         spectraComb = channelList + pairsList
                         spectraComb = numpy.array(spectraComb, dtype="u1")
                         self.processingHeaderObj.spectraComb = spectraComb
                     if self.dataOut.code is not None:
                         self.processingHeaderObj.code = self.dataOut.code
                         self.processingHeaderObj.nCode = self.dataOut.nCode
                         self.processingHeaderObj.nBaud = self.dataOut.nBaud
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                     self.processingHeaderObj.processFlags = self.getProcessFlags()
                     self.setBasicHeader()

schainpy/model/io/jroIO_voltage.py +6 -80

             '''
             Created on Jul 2, 2014
             @author: roj-idl71
             '''
             import numpy
             from .jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jrodata import Voltage
             import zmq
             import tempfile
             from io import StringIO
             # from _sha import blocksize
             @MPDecorator
             class VoltageReader(JRODataReader, ProcessingUnit):
                 """
                 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
                 perfiles*alturas*canales) son almacenados en la variable "buffer".
                                          perfiles * alturas * canales
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = VoltageReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         #to get one profile
                         profile =  readerObj.getData()
                         #print the profile
                         print profile
                         #If you want to see all datablock
                         print readerObj.datablock
                         if readerObj.flagNoMoreFiles:
                             break
                 """
-                ext = ".r"
+                def __init__(self):
-                optchar = "D"
-                dataOut = None
-                def __init__(self):#, **kwargs):
                     """
                     Inicializador de la clase VoltageReader para la lectura de datos de voltage.
                     Input:
                         dataOut    :    Objeto de la clase Voltage. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Variables afectadas:
                         self.dataOut
                     Return:
                         None
                     """
-                    ProcessingUnit.__init__(self)#, **kwargs)
+                    ProcessingUnit.__init__(self)
-                    self.isConfig = False
-                    self.datablock = None
-                    self.utc = 0
                     self.ext = ".r"
                     self.optchar = "D"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
-                    self.online = 0
-                    self.fp = None
-                    self.idFile = None
-                    self.dtype = None
-                    self.fileSizeByHeader = None
-                    self.filenameList = []
-                    self.filename = None
-                    self.fileSize = None
-                    self.firstHeaderSize = 0
-                    self.basicHeaderSize = 24
-                    self.pathList = []
-                    self.filenameList = []
                     self.lastUTTime = 0
-                    self.maxTimeStep = 30
-                    self.flagNoMoreFiles = 0
-                    self.set = 0
-                    self.path = None
                     self.profileIndex = 2**32 - 1
+                    self.dataOut = Voltage()
-                    self.delay = 3  # seconds
+                    self.selBlocksize = None
+                    self.selBlocktime = None
-                    self.nTries = 3  # quantity tries
-                    self.nFiles = 3  # number of files for searching
-                    self.nReadBlocks = 0
-                    self.flagIsNewFile = 1
-                    self.__isFirstTimeOnline = 1
-            #         self.ippSeconds = 0
-                    self.flagDiscontinuousBlock = 0
-                    self.flagIsNewBlock = 0
-                    self.nTotalBlocks = 0
-                    self.blocksize = 0
-                    self.dataOut = self.createObjByDefault()
-                    self.nTxs = 1
-                    self.txIndex = 0
                 def createObjByDefault(self):
                     dataObj = Voltage()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock * self.nTxs:
                         return 1
                     return 0
                 def getBlockDimension(self):
                     """
                         Obtiene la cantidad de puntos a leer por cada bloque de datos
                         Affected:
                             self.blocksize
                         Return:
                             None
                     """
                     pts2read = self.processingHeaderObj.profilesPerBlock * \
                         self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
                     self.blocksize = pts2read
                 def readBlock(self):
                     """
                         readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
                         (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                         (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                         es seteado a 0
                         Inputs:
                             None
                         Return:
                             None
                         Affected:
                             self.profileIndex
                             self.datablock
                             self.flagIsNewFile
                             self.flagIsNewBlock
                             self.nTotalBlocks
                         Exceptions:
                             Si un bloque leido no es un bloque valido
                     """
                     # if self.server is not None:
                     #     self.zBlock = self.receiver.recv()
                     #     self.zHeader = self.zBlock[:24]
                     #     self.zDataBlock = self.zBlock[24:]
                     #     junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
                     #     self.processingHeaderObj.profilesPerBlock = 240
                     #     self.processingHeaderObj.nHeights = 248
                     #     self.systemHeaderObj.nChannels
                     # else:
                     current_pointer_location = self.fp.tell()
                     junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
                     try:
                         junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
                                              self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
                     except:
                         # print "The read block (%3d) has not enough data" %self.nReadBlocks
                         if self.waitDataBlock(pointer_location=current_pointer_location):
                             junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
                             junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
                                                  self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
                     #             return 0
                     # Dimensions : nChannels, nProfiles, nSamples
                     junk = numpy.transpose(junk, (2, 0, 1))
                     self.datablock = junk['real'] + junk['imag'] * 1j
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.getBasicHeader()
                     self.dataOut.processingHeaderObj = self.processingHeaderObj.copy()
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     if self.nTxs > 1:
                         self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
                     # Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
                     #         self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
                     #
                     #         if self.radarControllerHeaderObj.code is not None:
                     #
                     #             self.dataOut.nCode = self.radarControllerHeaderObj.nCode
                     #
                     #             self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
                     #
                     #             self.dataOut.code = self.radarControllerHeaderObj.code
                     self.dataOut.dtype = self.dtype
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.heightList = numpy.arange(
                         self.processingHeaderObj.nHeights) * self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
                     self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     # asumo q la data no esta decodificada
                     self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode
                     # asumo q la data no esta sin flip
                     self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip
                     self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
                 def reshapeData(self):
                     if self.nTxs < 0:
                         return
                     if self.nTxs == 1:
                         return
                     if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1. / self.nTxs) != 0:
                         raise ValueError("1./nTxs (=%f), should be a multiple of nProfiles (=%d)" % (
 . / self.nTxs, self.processingHeaderObj.profilesPerBlock))
                     if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
                         raise ValueError("nTxs (=%d), should be a multiple of nHeights (=%d)" % (
                             self.nTxs, self.processingHeaderObj.nHeights))
                     self.datablock = self.datablock.reshape(
                         (self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock * self.nTxs, int(self.processingHeaderObj.nHeights / self.nTxs)))
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock * self.nTxs
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights / self.nTxs) * \
                         self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
                     self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
                     return
                 def readFirstHeaderFromServer(self):
                     self.getFirstHeader()
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
                                                PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
                     else:
                         raise ValueError('Data type was not defined')
                     self.dtype = datatype_str
                     #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
                         self.firstHeaderSize + self.basicHeaderSize * \
                         (self.processingHeaderObj.dataBlocksPerFile - 1)
                     #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
                     #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def getFromServer(self):
                     self.flagDiscontinuousBlock = 0
                     self.profileIndex = 0
                     self.flagIsNewBlock = 1
                     self.dataOut.flagNoData = False
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     self.blockPointer = 0
                     block = self.receiver.recv()
                     self.basicHeaderObj.read(block[self.blockPointer:])
                     self.blockPointer += self.basicHeaderObj.length
                     self.systemHeaderObj.read(block[self.blockPointer:])
                     self.blockPointer += self.systemHeaderObj.length
                     self.radarControllerHeaderObj.read(block[self.blockPointer:])
                     self.blockPointer += self.radarControllerHeaderObj.length
                     self.processingHeaderObj.read(block[self.blockPointer:])
                     self.blockPointer += self.processingHeaderObj.length
                     self.readFirstHeaderFromServer()
                     timestamp = self.basicHeaderObj.get_datatime()
                     print('[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp))
                     current_pointer_location = self.blockPointer
                     junk = numpy.fromstring(
                         block[self.blockPointer:], self.dtype, self.blocksize)
                     try:
                         junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
                                              self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
                     except:
                         # print "The read block (%3d) has not enough data" %self.nReadBlocks
                         if self.waitDataBlock(pointer_location=current_pointer_location):
                             junk = numpy.fromstring(
                                 block[self.blockPointer:], self.dtype, self.blocksize)
                             junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
                                                  self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
                     #               return 0
                     # Dimensions : nChannels, nProfiles, nSamples
                     junk = numpy.transpose(junk, (2, 0, 1))
                     self.datablock = junk['real'] + junk['imag'] * 1j
                     self.profileIndex = 0
                     if self.selBlocksize == None:
                         self.selBlocksize = self.dataOut.nProfiles
                     if self.selBlocktime != None:
                         if self.dataOut.nCohInt is not None:
                             nCohInt = self.dataOut.nCohInt
                         else:
                             nCohInt = 1
                         self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
                             nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
                     self.dataOut.data = self.datablock[:,
                                                        self.profileIndex:self.profileIndex + self.selBlocksize, :]
                     datasize = self.dataOut.data.shape[1]
                     if datasize < self.selBlocksize:
                         buffer = numpy.zeros(
                             (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
                         buffer[:, :datasize, :] = self.dataOut.data
                         self.dataOut.data = buffer
                         self.profileIndex = blockIndex
                     self.dataOut.flagDataAsBlock = True
                     self.flagIsNewBlock = 1
                     self.dataOut.realtime = self.online
                     return self.dataOut.data
                 def getData(self):
                     """
                         getData obtiene una unidad de datos del buffer de lectura, un perfil,  y la copia al objeto self.dataOut
                         del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
                         en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
                         "readNextBlock"
                         Ademas incrementa el contador del buffer "self.profileIndex" en 1.
                         Return:
                             Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
                             es igual al total de perfiles leidos desde el archivo.
                             Si self.getByBlock == False:
                                 self.dataOut.data = buffer[:, thisProfile, :]
                                 shape = [nChannels, nHeis]
                             Si self.getByBlock == True:
                                 self.dataOut.data = buffer[:, :, :]
                                 shape = [nChannels, nProfiles, nHeis]
                         Variables afectadas:
                             self.dataOut
                             self.profileIndex
                         Affected:
                             self.dataOut
                             self.profileIndex
                             self.flagDiscontinuousBlock
                             self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         return 0
                     self.flagDiscontinuousBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not(self.readNextBlock()):
                             return 0
                         self.getFirstHeader()
                         self.reshapeData()
                     if self.datablock is None:
                         self.dataOut.flagNoData = True
                         return 0
                     if not self.getByBlock:
                         """
                             Return profile by profile
                             If nTxs > 1 then one profile is divided by nTxs and number of total
                             blocks is increased by nTxs (nProfiles *= nTxs)
                         """
                         self.dataOut.flagDataAsBlock = False
                         self.dataOut.data = self.datablock[:, self.profileIndex, :]
                         self.dataOut.profileIndex = self.profileIndex
                         self.profileIndex += 1
                     else:
                         """
                             Return a block
                         """
                         if self.selBlocksize == None:
                             self.selBlocksize = self.dataOut.nProfiles
                         if self.selBlocktime != None:
                             if self.dataOut.nCohInt is not None:
                                 nCohInt = self.dataOut.nCohInt
                             else:
                                 nCohInt = 1
                             self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
                                 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
                         self.dataOut.data = self.datablock[:,
                                                            self.profileIndex:self.profileIndex + self.selBlocksize, :]
                         self.profileIndex += self.selBlocksize
                         datasize = self.dataOut.data.shape[1]
                         if datasize < self.selBlocksize:
                             buffer = numpy.zeros(
                                 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
                             buffer[:, :datasize, :] = self.dataOut.data
                             while datasize < self.selBlocksize:  # Not enough profiles to fill the block
                                 if not(self.readNextBlock()):
                                     return 0
                                 self.getFirstHeader()
                                 self.reshapeData()
                                 if self.datablock is None:
                                     self.dataOut.flagNoData = True
                                     return 0
                                 # stack data
                                 blockIndex = self.selBlocksize - datasize
                                 datablock1 = self.datablock[:, :blockIndex, :]
                                 buffer[:, datasize:datasize +
                                        datablock1.shape[1], :] = datablock1
                                 datasize += datablock1.shape[1]
                             self.dataOut.data = buffer
                             self.profileIndex = blockIndex
                         self.dataOut.flagDataAsBlock = True
                         self.dataOut.nProfiles = self.dataOut.data.shape[1]
                     self.dataOut.flagNoData = False
                     self.getBasicHeader()
                     self.dataOut.realtime = self.online
                     return self.dataOut.data
             @MPDecorator
             class VoltageWriter(JRODataWriter, Operation):
                 """
                 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".r"
                 optchar = "D"
                 shapeBuffer = None
                 def __init__(self):#, **kwargs):
                     """
                     Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                     Return: None
                     """
                     Operation.__init__(self)#, **kwargs)
                     self.nTotalBlocks = 0
                     self.profileIndex = 0
                     self.isConfig = False
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.blockIndex = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
                         return 1
                     return 0
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
                                         self.processingHeaderObj.nHeights,
                                         self.systemHeaderObj.nChannels)
                     self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
                                                   self.processingHeaderObj.profilesPerBlock,
                                                   self.processingHeaderObj.nHeights),
                                                  dtype=numpy.dtype('complex64'))
                 def writeBlock(self):
                     """
                     Escribe el buffer en el file designado
                     Affected:
                         self.profileIndex
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.blockIndex
                     Return: None
                     """
                     data = numpy.zeros(self.shapeBuffer, self.dtype)
                     junk = numpy.transpose(self.datablock, (1, 2, 0))
                     data['real'] = junk.real
                     data['imag'] = junk.imag
                     data = data.reshape((-1))
                     data.tofile(self.fp)
                     self.datablock.fill(0)
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
             #         print "[Writing] Block = %04d" %self.blockIndex
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.flagIsNewBlock
                         self.profileIndex
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagDiscontinuousBlock:
                         self.datablock.fill(0)
                         self.profileIndex = 0
                         self.setNextFile()
                     if self.profileIndex == 0:
                         self.setBasicHeader()
                     self.datablock[:, self.profileIndex, :] = self.dataOut.data
                     self.profileIndex += 1
                     if self.hasAllDataInBuffer():
                         # if self.flagIsNewFile:
                         self.writeNextBlock()
             #            self.setFirstHeader()
                     return 1
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
                     '''
                     dtype_width = self.getDtypeWidth()
                     blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels *
                                     self.profilesPerBlock * dtype_width * 2)
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.processingHeaderObj.dtype = 0  # Voltage
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     # podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.nWindows = 1
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
                     # Cuando la data de origen es de tipo Voltage
                     self.processingHeaderObj.nIncohInt = 1
                     # Cuando la data de origen es de tipo Voltage
                     self.processingHeaderObj.totalSpectra = 0
                     if self.dataOut.code is not None:
                         self.processingHeaderObj.code = self.dataOut.code
                         self.processingHeaderObj.nCode = self.dataOut.nCode
                         self.processingHeaderObj.nBaud = self.dataOut.nBaud
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - \
                             self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                     self.processingHeaderObj.processFlags = self.getProcessFlags()
                     self.setBasicHeader()
   No newline at end of file

schainpy/model/io/jroIO_example.py removed 0 -175

NO CONTENT: file was removed

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages