From 831d2c3bd88d2944c5c4a3bf90b6e3dd8e3e4c37 2017-10-25 17:27:10 From: José Chávez Date: 2017-10-25 17:27:10 Subject: [PATCH] merge v2.3 --- diff --git a/schainpy/model/graphics/jroplot_data.py b/schainpy/model/graphics/jroplot_data.py index c08503c..71a9ccd 100644 --- a/schainpy/model/graphics/jroplot_data.py +++ b/schainpy/model/graphics/jroplot_data.py @@ -15,11 +15,11 @@ from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator from schainpy.model.proc.jroproc_base import Operation from schainpy.utils import log -jet_values = matplotlib.pyplot.get_cmap("jet", 100)(numpy.arange(100))[10:90] +jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90] blu_values = matplotlib.pyplot.get_cmap( - "seismic_r", 20)(numpy.arange(20))[10:15] + 'seismic_r', 20)(numpy.arange(20))[10:15] ncmap = matplotlib.colors.LinearSegmentedColormap.from_list( - "jro", numpy.vstack((blu_values, jet_values))) + 'jro', numpy.vstack((blu_values, jet_values))) matplotlib.pyplot.register_cmap(cmap=ncmap) CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'RdBu_r', 'seismic')] @@ -49,6 +49,10 @@ class PlotData(Operation, Process): __MAXNUMX = 80 __missing = 1E30 + __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax', + 'zlimits', 'xlabel', 'ylabel', 'cb_label', 'title', 'titles', 'colorbar', + 'bgcolor', 'width', 'height', 'localtime', 'oneFigure', 'showprofile'] + def __init__(self, **kwargs): Operation.__init__(self, plot=True, **kwargs) @@ -91,6 +95,7 @@ class PlotData(Operation, Process): self.colorbar = kwargs.get('colorbar', True) self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1]) self.titles = ['' for __ in range(16)] + self.polar = False def __fmtTime(self, x, pos): ''' @@ -103,6 +108,10 @@ class PlotData(Operation, Process): Common setup for all figures, here figures and axes are created ''' + if self.CODE not in self.data: + raise ValueError(log.error('Missing data for {}'.format(self.CODE), + self.name)) + self.setup() self.time_label = 'LT' if self.localtime else 'UTC' @@ -131,7 +140,8 @@ class PlotData(Operation, Process): facecolor='w') self.figures.append(fig) for n in range(self.nplots): - ax = fig.add_subplot(self.nrows, self.ncols, n + 1) + ax = fig.add_subplot(self.nrows, self.ncols, + n + 1, polar=self.polar) ax.tick_params(labelsize=8) ax.firsttime = True ax.index = 0 @@ -148,7 +158,7 @@ class PlotData(Operation, Process): fig = plt.figure(figsize=(self.width, self.height), edgecolor='k', facecolor='w') - ax = fig.add_subplot(1, 1, 1) + ax = fig.add_subplot(1, 1, 1, polar=self.polar) ax.tick_params(labelsize=8) ax.firsttime = True ax.index = 0 @@ -348,7 +358,7 @@ class PlotData(Operation, Process): else: if self.xaxis is 'time': dt = self.getDateTime(self.max_time) - xmax = (dt.replace(hour=int(self.xmax), minute=0, second=0) - + xmax = (dt.replace(hour=int(self.xmax), minute=59, second=59) - datetime.datetime(1970, 1, 1)).total_seconds() if self.data.localtime: xmax += time.timezone @@ -363,8 +373,6 @@ class PlotData(Operation, Process): 0][0] < 0 else numpy.where(ymax < Y)[0][0] ystep = Y[i - 1] / 5 - ystep = 200 if ymax >= 800 else 100 if ymax >= 400 else 50 if ymax >= 200 else 20 - for n, ax in enumerate(self.axes): if ax.firsttime: ax.set_facecolor(self.bgcolor) @@ -384,7 +392,8 @@ class PlotData(Operation, Process): [tick.set_visible(False) for tick in self.pf_axes[n].get_yticklabels()] if self.colorbar: - ax.cbar = plt.colorbar(ax.plt, ax=ax, pad=0.02, aspect=10) + ax.cbar = plt.colorbar( + ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10) ax.cbar.ax.tick_params(labelsize=8) ax.cbar.ax.press = None if self.cb_label: @@ -394,13 +403,19 @@ class PlotData(Operation, Process): else: ax.cbar = None - ax.set_title('{} - {} {}'.format( - self.titles[n], - self.getDateTime(self.max_time).strftime('%H:%M:%S'), - self.time_label), - size=8) - ax.set_xlim(xmin, xmax) - ax.set_ylim(ymin, ymax) + if not self.polar: + ax.set_xlim(xmin, xmax) + ax.set_ylim(ymin, ymax) + ax.set_title('{} - {} {}'.format( + self.titles[n], + self.getDateTime(self.max_time).strftime('%H:%M:%S'), + self.time_label), + size=8) + else: + ax.set_title('{}'.format(self.titles[n]), size=8) + ax.set_ylim(0, 90) + ax.set_yticks(numpy.arange(0, 90, 20)) + ax.yaxis.labelpad = 40 def __plot(self): ''' @@ -413,6 +428,7 @@ class PlotData(Operation, Process): for n, fig in enumerate(self.figures): if self.nrows == 0 or self.nplots == 0: log.warning('No data', self.name) + fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center') continue fig.tight_layout() @@ -437,7 +453,7 @@ class PlotData(Operation, Process): str(self.contador), ) ) - print 'Saving figure: {}'.format(figname) + log.log('Saving figure: {}'.format(figname), self.name) fig.savefig(figname) def plot(self): @@ -820,7 +836,7 @@ class PlotSkyMapData(PlotData): Plot for meteors detection data ''' - CODE = 'met' + CODE = 'param' def setup(self): @@ -828,25 +844,17 @@ class PlotSkyMapData(PlotData): self.nrows = 1 self.width = 7.2 self.height = 7.2 - + self.nplots = 1 self.xlabel = 'Zonal Zenith Angle (deg)' self.ylabel = 'Meridional Zenith Angle (deg)' - - if self.figure is None: - self.figure = plt.figure(figsize=(self.width, self.height), - edgecolor='k', - facecolor='w') - else: - self.figure.clf() - - self.ax = plt.subplot2grid( - (self.nrows, self.ncols), (0, 0), 1, 1, polar=True) - self.ax.firsttime = True + self.polar = True + self.ymin = -180 + self.ymax = 180 + self.colorbar = False def plot(self): - arrayParameters = numpy.concatenate( - [self.data['param'][t] for t in self.times]) + arrayParameters = numpy.concatenate(self.data['param']) error = arrayParameters[:, -1] indValid = numpy.where(error == 0)[0] finalMeteor = arrayParameters[indValid, :] @@ -856,23 +864,19 @@ class PlotSkyMapData(PlotData): x = finalAzimuth * numpy.pi / 180 y = finalZenith - if self.ax.firsttime: - self.ax.plot = self.ax.plot(x, y, 'bo', markersize=5)[0] - self.ax.set_ylim(0, 90) - self.ax.set_yticks(numpy.arange(0, 90, 20)) - self.ax.set_xlabel(self.xlabel) - self.ax.set_ylabel(self.ylabel) - self.ax.yaxis.labelpad = 40 - self.ax.firsttime = False + ax = self.axes[0] + + if ax.firsttime: + ax.plot = ax.plot(x, y, 'bo', markersize=5)[0] else: - self.ax.plot.set_data(x, y) + ax.plot.set_data(x, y) dt1 = self.getDateTime(self.min_time).strftime('%y/%m/%d %H:%M:%S') dt2 = self.getDateTime(self.max_time).strftime('%y/%m/%d %H:%M:%S') title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1, dt2, len(x)) - self.ax.set_title(title, size=8) + self.titles[0] = title self.saveTime = self.max_time diff --git a/schainpy/model/io/jroIO_base.py b/schainpy/model/io/jroIO_base.py index ebb4ccd..90aab67 100644 --- a/schainpy/model/io/jroIO_base.py +++ b/schainpy/model/io/jroIO_base.py @@ -553,7 +553,10 @@ class JRODataIO: return dtype_width def getAllowedArgs(self): - return inspect.getargspec(self.run).args + if hasattr(self, '__attrs__'): + return self.__attrs__ + else: + return inspect.getargspec(self.run).args class JRODataReader(JRODataIO): diff --git a/schainpy/model/io/jroIO_bltr.py b/schainpy/model/io/jroIO_bltr.py index 4049dd2..6a1c6a5 100644 --- a/schainpy/model/io/jroIO_bltr.py +++ b/schainpy/model/io/jroIO_bltr.py @@ -513,10 +513,8 @@ class BLTRSpectraReader (ProcessingUnit, FileHeaderBLTR, RecordHeaderBLTR, JRODa self.dataOut = Spectra() self.profileIndex = 1 # Always self.dataOut.flagNoData = False - self.dataOut.nRdPairs = 0 - self.dataOut.pairsList = [] - self.dataOut.data_spc = None - self.dataOut.noise = [] + self.dataOut.nRdPairs = 0 + self.dataOut.data_spc = None self.dataOut.velocityX = [] self.dataOut.velocityY = [] self.dataOut.velocityV = [] diff --git a/schainpy/model/io/jroIO_mira35c.py b/schainpy/model/io/jroIO_mira35c.py index 684b106..5ecc67a 100644 --- a/schainpy/model/io/jroIO_mira35c.py +++ b/schainpy/model/io/jroIO_mira35c.py @@ -494,17 +494,14 @@ class MIRA35CReader (ProcessingUnit, FileHeaderMIRA35c, SRVIHeader, RecordHeader self.dataOut = Spectra() self.profileIndex = 1 # Always self.dataOut.flagNoData = False - self.dataOut.nRdPairs = 0 - self.dataOut.pairsList = [] - self.dataOut.data_spc = None - - self.dataOut.normFactor = 1 + self.dataOut.nRdPairs = 0 + self.dataOut.data_spc = None self.nextfileflag = True self.dataOut.RadarConst = 0 self.dataOut.HSDV = [] self.dataOut.NPW = [] self.dataOut.COFA = [] - self.dataOut.noise = 0 + # self.dataOut.noise = 0 def Files2Read(self, fp): ''' @@ -588,6 +585,7 @@ class MIRA35CReader (ProcessingUnit, FileHeaderMIRA35c, SRVIHeader, RecordHeader self.dataOut.noise = self.dataOut.getNoise() # print 'ACAAAAAA', self.dataOut.noise self.dataOut.data_spc = self.dataOut.data_spc + self.dataOut.noise + self.dataOut.normFactor = 1 # print 'self.dataOut.noise',self.dataOut.noise return self.dataOut.data_spc diff --git a/schainpy/model/proc/jroproc_base.py b/schainpy/model/proc/jroproc_base.py index 3b87703..31873d6 100644 --- a/schainpy/model/proc/jroproc_base.py +++ b/schainpy/model/proc/jroproc_base.py @@ -64,10 +64,17 @@ class ProcessingUnit(object): self.args = args self.kwargs = kwargs + + if not hasattr(self, 'name'): + self.name = self.__class__.__name__ + checkKwargs(self.run, kwargs) def getAllowedArgs(self): - return inspect.getargspec(self.run).args + if hasattr(self, '__attrs__'): + return self.__attrs__ + else: + return inspect.getargspec(self.run).args def addOperationKwargs(self, objId, **kwargs): ''' @@ -309,10 +316,15 @@ class Operation(object): self.__buffer = None self.isConfig = False self.kwargs = kwargs + if not hasattr(self, 'name'): + self.name = self.__class__.__name__ checkKwargs(self.run, kwargs) def getAllowedArgs(self): - return inspect.getargspec(self.run).args + if hasattr(self, '__attrs__'): + return self.__attrs__ + else: + return inspect.getargspec(self.run).args def setup(self): diff --git a/schainpy/model/proc/jroproc_parameters.py b/schainpy/model/proc/jroproc_parameters.py index 5e52bfa..113190d 100644 --- a/schainpy/model/proc/jroproc_parameters.py +++ b/schainpy/model/proc/jroproc_parameters.py @@ -1408,18 +1408,18 @@ class SpectralMoments(Operation): def __calculateMoments(self, oldspec, oldfreq, n0, nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None): - if (nicoh == None): nicoh = 1 - if (graph == None): graph = 0 - if (smooth == None): smooth = 0 + if (nicoh is None): nicoh = 1 + if (graph is None): graph = 0 + if (smooth is None): smooth = 0 elif (self.smooth < 3): smooth = 0 - if (type1 == None): type1 = 0 - if (fwindow == None): fwindow = numpy.zeros(oldfreq.size) + 1 - if (snrth == None): snrth = -3 - if (dc == None): dc = 0 - if (aliasing == None): aliasing = 0 - if (oldfd == None): oldfd = 0 - if (wwauto == None): wwauto = 0 + if (type1 is None): type1 = 0 + if (fwindow is None): fwindow = numpy.zeros(oldfreq.size) + 1 + if (snrth is None): snrth = -3 + if (dc is None): dc = 0 + if (aliasing is None): aliasing = 0 + if (oldfd is None): oldfd = 0 + if (wwauto is None): wwauto = 0 if (n0 < 1.e-20): n0 = 1.e-20 @@ -1719,7 +1719,7 @@ class SpectralFitting(Operation): error1 = p0*numpy.nan #Save - if self.dataOut.data_param == None: + if self.dataOut.data_param is None: self.dataOut.data_param = numpy.zeros((nGroups, p0.size, nHeights))*numpy.nan self.dataOut.data_error = numpy.zeros((nGroups, p0.size + 1, nHeights))*numpy.nan @@ -2382,7 +2382,7 @@ class WindProfiler(Operation): self.__isConfig = True - if self.__buffer == None: + if self.__buffer is None: self.__buffer = dataOut.data_param self.__firstdata = copy.copy(dataOut) @@ -2424,7 +2424,7 @@ class WindProfiler(Operation): else: mode = 'SA' #Borrar luego esto - if dataOut.groupList == None: + if dataOut.groupList is None: dataOut.groupList = [(0,1),(0,2),(1,2)] groupList = dataOut.groupList C = 3e8 @@ -2446,7 +2446,7 @@ class WindProfiler(Operation): self.__isConfig = True - if self.__buffer == None: + if self.__buffer is None: self.__buffer = dataOut.data_param self.__firstdata = copy.copy(dataOut) @@ -2808,7 +2808,7 @@ class SMDetection(Operation): #Getting Pairslist - if channelPositions == None: + if channelPositions is None: # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella meteorOps = SMOperations() @@ -2939,7 +2939,7 @@ class SMDetection(Operation): # arrayFinal = arrayParameters.reshape((1,arrayParameters.shape[0],arrayParameters.shape[1])) dataOut.data_param = arrayParameters - if arrayParameters == None: + if arrayParameters is None: dataOut.flagNoData = True else: dataOut.flagNoData = True @@ -3475,7 +3475,7 @@ class CorrectSMPhases(Operation): arrayParameters[:,8:12] = numpy.angle(numpy.exp(1j*(arrayParameters[:,8:12] + phaseOffsets))) meteorOps = SMOperations() - if channelPositions == None: + if channelPositions is None: # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella @@ -3652,7 +3652,7 @@ class SMPhaseCalibration(Operation): self.__isConfig = True - if self.__buffer == None: + if self.__buffer is None: self.__buffer = dataOut.data_param.copy() else: diff --git a/schainpy/model/utils/jroutils_publish.py b/schainpy/model/utils/jroutils_publish.py index 84b067f..56d6163 100644 --- a/schainpy/model/utils/jroutils_publish.py +++ b/schainpy/model/utils/jroutils_publish.py @@ -60,12 +60,17 @@ class throttle(object): def __call__(self, fn): @wraps(fn) def wrapper(*args, **kwargs): - now = datetime.datetime.now() - time_since_last_call = now - self.time_of_last_call - time_left = self.throttle_period - time_since_last_call + coerce = kwargs.pop('coerce', None) + if coerce: + self.time_of_last_call = datetime.datetime.now() + return fn(*args, **kwargs) + else: + now = datetime.datetime.now() + time_since_last_call = now - self.time_of_last_call + time_left = self.throttle_period - time_since_last_call - if time_left > datetime.timedelta(seconds=0): - return + if time_left > datetime.timedelta(seconds=0): + return self.time_of_last_call = datetime.datetime.now() return fn(*args, **kwargs) @@ -104,6 +109,9 @@ class Data(object): ret = numpy.swapaxes(ret, 0, 1) return ret + def __contains__(self, key): + return key in self.data + def setup(self): ''' Configure object @@ -242,6 +250,8 @@ class PublishData(Operation): Operation to send data over zmq. ''' + __attrs__ = ['host', 'port', 'delay', 'zeromq', 'mqtt', 'verbose'] + def __init__(self, **kwargs): """Inicio.""" Operation.__init__(self, **kwargs) @@ -426,6 +436,8 @@ class PublishData(Operation): class ReceiverData(ProcessingUnit): + __attrs__ = ['server'] + def __init__(self, **kwargs): ProcessingUnit.__init__(self, **kwargs) @@ -464,6 +476,7 @@ class ReceiverData(ProcessingUnit): class PlotterReceiver(ProcessingUnit, Process): throttle_value = 5 + __attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle'] def __init__(self, **kwargs): @@ -564,26 +577,30 @@ class PlotterReceiver(ProcessingUnit, Process): while True: dataOut = self.receiver.recv_pyobj() - tm = dataOut.utctime - if dataOut.useLocalTime: - if not self.localtime: - tm += time.timezone - dt = datetime.datetime.fromtimestamp(tm).date() - else: - if self.localtime: - tm -= time.timezone - dt = datetime.datetime.utcfromtimestamp(tm).date() - sended = False - if dt not in self.dates: - if self.data: - self.data.ended = True - self.send(self.data) - sended = True - self.data.setup() - self.dates.append(dt) - - self.data.update(dataOut) + if not dataOut.flagNoData: + if dataOut.type == 'Parameters': + tm = dataOut.utctimeInit + else: + tm = dataOut.utctime + if dataOut.useLocalTime: + if not self.localtime: + tm += time.timezone + dt = datetime.datetime.fromtimestamp(tm).date() + else: + if self.localtime: + tm -= time.timezone + dt = datetime.datetime.utcfromtimestamp(tm).date() + coerce = False + if dt not in self.dates: + if self.data: + self.data.ended = True + self.send(self.data) + coerce = True + self.data.setup() + self.dates.append(dt) + self.data.update(dataOut) + if dataOut.finished is True: self.connections -= 1 if self.connections == 0 and dt in self.dates: @@ -594,9 +611,9 @@ class PlotterReceiver(ProcessingUnit, Process): if self.realtime: self.send(self.data) # self.sender_web.send_string(self.data.jsonify()) - else: - if not sended: - self.sendData(self.send, self.data) + else: + self.sendData(self.send, self.data, coerce=coerce) + coerce = False return diff --git a/schainpy/utils/paramsFinder.py b/schainpy/utils/paramsFinder.py index 3361134..0efaf1c 100644 --- a/schainpy/utils/paramsFinder.py +++ b/schainpy/utils/paramsFinder.py @@ -1,6 +1,5 @@ import schainpy from schainpy.model import Operation, ProcessingUnit -from importlib import import_module from pydoc import locate def clean_modules(module): @@ -11,17 +10,17 @@ def clean_modules(module): def check_module(possible, instance): def check(x): - try: + try: instancia = locate('schainpy.model.{}'.format(x)) return isinstance(instancia(), instance) except Exception as e: - return False + return False clean = clean_modules(possible) return [x for x in clean if check(x)] def getProcs(): - module = dir(import_module('schainpy.model')) + module = dir(schainpy.model) procs = check_module(module, ProcessingUnit) try: procs.remove('ProcessingUnit') @@ -30,7 +29,7 @@ def getProcs(): return procs def getOperations(): - module = dir(import_module('schainpy.model')) + module = dir(schainpy.model) noProcs = [x for x in module if not x.endswith('Proc')] operations = check_module(noProcs, Operation) try: @@ -53,7 +52,7 @@ def getArgs(op): return args def getAll(): - allModules = dir(import_module('schainpy.model')) + allModules = dir(schainpy.model) modules = check_module(allModules, Operation) modules.extend(check_module(allModules, ProcessingUnit)) return modules