schain-jc Commit - r1125:ba0b6ae662be

merge 2.3

José Chávez -

r1125:ba0b6ae662be

parent child

Context file:

r1125:ba0b6ae662be

Collapse all files

Dockerfile created 644 +26 0

@@ -0,0 +1,26
	1	FROM python:2.7-slim
	2
	3	RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommends \
	4	git \
	5	gcc \
	6	libpng-dev \
	7	libfreetype6-dev \
	8	libopenblas-dev \
	9	liblapack-dev \
	10	libatlas-base-dev \
	11	libssl-dev \
	12	libhdf5-dev \
	13	&& git clone --branch v2.3 --depth 1 \
	14	http://jro-dev.igp.gob.pe/rhodecode/schain \
	15	&& pip install numpy \
	16	&& cd schain \
	17	&& pip install . \
	18	&& rm -rf * \
	19	&& apt-get purge -y --auto-remove git gcc \
	20	&& rm -rf /var/lib/apt/lists/*
	21
	22	ENV BACKEND="Agg"
	23
	24	VOLUME /data
	25
	26	ENTRYPOINT ["schain"]

README.md +17 -15

             ## Installation
-            Install system dependencies, clone the latest version from [git](http://jro-dev.igp.gob.pe/rhodecode/schain/) and install it as a normal python package.
+            Install system dependencies, clone the latest version from [here](http://jro-dev.igp.gob.pe/rhodecode/schain/) and install it as a normal python package.
             ### Linux based system
             ```
             $ sudo pip install ./
             ```
-            **It is recommended to install schain in a virtual environment**
-            ```
-            $ sudo pip install virtualenv
-            $ virtualenv /path/to/virtual --system-site-packages
-            $ source /path/to/virtual/bin/activate
-            (virtual) $ cd schain
-            (virtual) $ pip install ./
-            ```
             ### MAC Os
             ```
+            $ brew install python
             $ brew install cartr/qt4/pyqt
             $ git clone http://jro-dev.igp.gob.pe/rhodecode/schain/
             $ cd schain
             $ pip install ./
             ```
-            if ```pip install ./``` does not work, install a proper python enviroment, and repeat the steps.
+            **It is recommended to install schain in a virtual environment**
             ```
-            $ brew install python
+            $ virtualenv /path/to/virtual
+            $ source /path/to/virtual/bin/activate
+            (virtual) $ cd schain
+            (virtual) $ pip install ./
+            (virtual) $ bash link_PyQt4.sh
             ```
-            ### GUI Installation
+            ### Docker
+            Download Dockerfile from the repository, and create a docker image
             ```
-            $ sudo apt-get install python-pip python-dev gfortran libpng-dev freetype* libblas-dev liblapack-dev libatlas-base-dev python-qt4 python-tk libssl-dev libhdf5-dev
+            $ docker build -t schain .
-            $ (virtual) bash link_PyQt4.sh
             ```
+            You can run a container using an xml file or a schain script also you need to mount a volume for the data input and for the output files/plots
+            ```
+            $ docker run -it --rm --volume /path/to/host/data:/data schain xml /data/test.xml
+            $ docker run -it --rm --volume /path/to/host/data:/data --entrypoint=/bin/python schain /data/test.py
+            ```
             ## First Script

link_PyQt4.sh +5 -7

             #!/bin/bash
             # This hook is run after a new virtualenv is activated.
-            # ~/.virtualenvs/postmkvirtualenv
-            libs=( PyQt4 sip.so )
             python_version=python$(python -c "import sys; print (str(sys.version_info[0])+'.'+str(sys.version_info[1]))")
             var=( $(which -a $python_version) )
             get_python_lib_cmd="from distutils.sysconfig import get_python_lib; print (get_python_lib())"
             lib_virtualenv_path=$(python -c "$get_python_lib_cmd")
             lib_system_path=$(${var[-1]} -c "$get_python_lib_cmd")
+            sip_path=$(ls $lib_system_path/sip*.so)
-            for lib in ${libs[@]}
+            echo "Linking Qt4..."
-            do
+            ln -s $lib_system_path/PyQt4 $lib_virtualenv_path/PyQt4
-                ln -s $lib_system_path/$lib $lib_virtualenv_path/$lib
+            echo "Linking SIP..."
-            done
+            ln -s $sip_path $lib_virtualenv_path/sip.so

schainpy/cli/README.md ~~schaincli/README.md~~ renamed 0 0

NO CONTENT: file renamed from schaincli/README.md to schainpy/cli/README.md

schainpy/cli/__init__.py ~~schaincli/__init__.py~~ renamed 0 0

NO CONTENT: file renamed from schaincli/__init__.py to schainpy/cli/__init__.py

schainpy/cli/cli.py ~~schaincli/cli.py~~ renamed +1 -1

             save_stdout = sys.stdout
             sys.stdout = open('trash', 'w')
             from multiprocessing import cpu_count
-            from schaincli import templates
             from schainpy.controller import Project
             from schainpy.model import Operation, ProcessingUnit
             from schainpy.utils import log
             from pydoc import locate
             from fuzzywuzzy import process
             from schainpy.utils import paramsFinder
+            import templates
             sys.stdout = save_stdout

schainpy/cli/templates.py ~~schaincli/templates.py~~ renamed 0 0

NO CONTENT: file renamed from schaincli/templates.py to schainpy/cli/templates.py

schainpy/cli/tests/__init__.py ~~schaincli/tests/__init__.py~~ renamed 0 0

NO CONTENT: file renamed from schaincli/tests/__init__.py to schainpy/cli/tests/__init__.py

schainpy/cli/tests/test_cli.py ~~schaincli/tests/test_cli.py~~ renamed +1 -1

             import pytest
             from click.testing import CliRunner
-            from schaincli import cli
+            from schainpy.cli import cli
             @pytest.fixture

schainpy/controller.py +8 -2

                 op = rconf.getOperationObj('run')
                 dt1 = op.getParameterValue('startDate')
                 dt2 = op.getParameterValue('endDate')
+                tm1 = op.getParameterValue('startTime')
+                tm2 = op.getParameterValue('endTime')
                 days = (dt2 - dt1).days
                 for day in range(days + 1):
                     paths, files = reader.searchFilesOffLine(path=rconf.path,
                                                              startDate=dt,
                                                              endDate=dt,
+                                                             startTime=tm1,
+                                                             endTime=tm2,
                                                              ext=DTYPES[rconf.datatype])
                     nFiles = len(files)
                     if nFiles == 0:
                 def run(self):
                     log.success('Starting {}'.format(self.name))
+                    self.start_time = time.time()
                     self.createObjects()
                     self.connectObjects()
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         procUnitConfObj.close()
-                    log.success('{} finished'.format(self.name))
+                    log.success('{} finished (time: {}s)'.format(
+                        self.name,
+                        time.time()-self.start_time))

schainpy/gui/schainGUI +1 -2

                 from PyQt4.QtGui import QApplication
             except:
                 log.error(
-                    'You should install PtQt4 module in order to run the GUI. See the README.')
+                    'You should install PyQt4 module in order to run the GUI. See the README.')
                 sys.exit()
             from schainpy.gui.viewcontroller.initwindow import InitWindow
             from schainpy.gui.viewcontroller.basicwindow import BasicWindow
             from schainpy.gui.viewcontroller.workspace import Workspace

schainpy/model/data/jroheaderIO.py +1 -1

                         code1 = (self.code + 1.0) / 2.
                         for ic in range(self.nCode):
-                            tempx = numpy.zeros(int(self.nBaud / 32.))
+                            tempx = numpy.zeros(int(numpy.ceil(self.nBaud / 32.)))
                             start = 0
                             end = 32
                             for i in range(len(tempx)):

schainpy/model/graphics/jroplot_data.py +23 -13

                 'jro', numpy.vstack((blu_values, jet_values)))
             matplotlib.pyplot.register_cmap(cmap=ncmap)
-            CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'RdBu_r', 'seismic')]
+            CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
             def figpause(interval):
                 colormap = 'jro'
                 bgcolor = 'white'
                 CONFLATE = False
-                __MAXNUMX = 80
                 __missing = 1E30
                 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
                     self.ymin = kwargs.get('ymin', None)
                     self.ymax = kwargs.get('ymax', None)
                     self.xlabel = kwargs.get('xlabel', None)
-                    self.__MAXNUMY = kwargs.get('decimation', 200)
+                    self.decimation = kwargs.get('decimation', None)
                     self.showSNR = kwargs.get('showSNR', False)
                     self.oneFigure = kwargs.get('oneFigure', True)
                     self.width = kwargs.get('width', None)
                 def decimate(self):
                     # dx = int(len(self.x)/self.__MAXNUMX) + 1
-                    dy = int(len(self.y) / self.__MAXNUMY) + 1
+                    dy = int(len(self.y) / self.decimation) + 1
                     # x = self.x[::dx]
                     x = self.x
                     ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
                     ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
-                    Y = numpy.array([10, 20, 50, 100, 200, 500, 1000, 2000])
+                    Y = numpy.array([5, 10, 20, 50, 100, 200, 500, 1000, 2000])
                     i = 1 if numpy.where(ymax-ymin < Y)[0][0] < 0 else numpy.where(ymax-ymin < Y)[0][0]
                     ystep = Y[i] / 5
                     '''
                     log.success('Plotting', self.name)
+                    try:
                         self.plot()
                         self.format()
+                    except:
+                        log.warning('{} Plot could not be updated... check data'.format(self.CODE), self.name)
                     for n, fig in enumerate(self.figures):
                         if self.nrows == 0 or self.nplots == 0:
                                     self.CODE,
                                     label,
                                     self.getDateTime(self.saveTime).strftime(
-                                        '%y%m%d_%H%M%S'),
+                                        '%Y%m%d_%H%M%S'),
                                 )
                             )
                             log.log('Saving figure: {}'.format(figname), self.name)
                     if self.showprofile:
                         self.width += 0.8 * self.ncols
-                    self.ylabel = 'Range [Km]'
+                    self.ylabel = 'Range [km]'
                 def plot(self):
                     if self.xaxis == "frequency":
                     self.nplots = self.nrows * 4
                     self.width = 3.4 * self.ncols
                     self.height = 3 * self.nrows
-                    self.ylabel = 'Range [Km]'
+                    self.ylabel = 'Range [km]'
                     self.showprofile = False
                 def plot(self):
                     self.ncols = 1
                     self.nrows = len(self.data.channels)
                     self.nplots = len(self.data.channels)
-                    self.ylabel = 'Range [Km]'
+                    self.ylabel = 'Range [km]'
                     self.cb_label = 'dB'
                     self.titles = ['{} Channel {}'.format(
                         self.CODE.upper(), x) for x in range(self.nrows)]
                     self.z = self.data[self.CODE]
                     self.z = numpy.ma.masked_invalid(self.z)
-                    for n, ax in enumerate(self.axes):
+                    if self.decimation is None:
+                        x, y, z = self.fill_gaps(self.x, self.y, self.z)
+                    else:
                         x, y, z = self.fill_gaps(*self.decimate())
+                    for n, ax in enumerate(self.axes):
                         self.zmin = self.zmin if self.zmin else numpy.min(self.z)
                         self.zmax = self.zmax if self.zmax else numpy.max(self.z)
                         if ax.firsttime:
                     self.ncols = 1
                     self.nrows = len(self.data.pairs)
                     self.nplots = len(self.data.pairs)
-                    self.ylabel = 'Range [Km]'
+                    self.ylabel = 'Range [km]'
                     if self.CODE == 'coh':
                         self.cb_label = ''
                         self.titles = [
                         self.nrows += 1
                         self.nplots += 1
-                    self.ylabel = 'Height [Km]'
+                    self.ylabel = 'Height [km]'
                     if not self.titles:
                         self.titles = self.data.parameters \
                             if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
                     self.z = numpy.ma.masked_invalid(self.z)
+                    if self.decimation is None:
+                        x, y, z = self.fill_gaps(self.x, self.y, self.z)
+                    else:
+                        x, y, z = self.fill_gaps(*self.decimate())
                     for n, ax in enumerate(self.axes):
-                        x, y, z = self.fill_gaps(*self.decimate())
                         self.zmax = self.zmax if self.zmax is not None else numpy.max(
                             self.z[n])
                         self.zmin = self.zmin if self.zmin is not None else numpy.min(

schainpy/model/graphics/mpldriver.py +12 -6

@@ -1,13 +1,19
1	import ~~numpy~~	1	import os
2	import datetime
3	import sys	2	import sys
		3	import datetime
		4	import numpy
4	import matplotlib	5	import matplotlib
5		6
6	if 'linux' in sys.platform:	7	if 'BACKEND' in os.environ:
		8	matplotlib.use(os.environ['BACKEND'])
		9	elif 'linux' in sys.platform:
7	matplotlib.use("TkAgg")	10	matplotlib.use("TkAgg")
8		11	elif 'darwin' in sys.platform:
9	if 'darwin' in sys.platform:	12	matplotlib.use('TkAgg')
10	matplotlib.use('TKAgg')	13	else:
		14	from schainpy.utils import log
		15	log.warning('Using default Backend="Agg"', 'INFO')
		16	matplotlib.use('Agg')
11	# Qt4Agg', 'GTK', 'GTKAgg', 'ps', 'agg', 'cairo', 'MacOSX', 'GTKCairo', 'WXAgg', 'template', 'TkAgg', 'GTK3Cairo', 'GTK3Agg', 'svg', 'WebAgg', 'CocoaAgg', 'emf', 'gdk', 'WX'	17	# Qt4Agg', 'GTK', 'GTKAgg', 'ps', 'agg', 'cairo', 'MacOSX', 'GTKCairo', 'WXAgg', 'template', 'TkAgg', 'GTK3Cairo', 'GTK3Agg', 'svg', 'WebAgg', 'CocoaAgg', 'emf', 'gdk', 'WX'
12	import matplotlib.pyplot	18	import matplotlib.pyplot
13		19

schainpy/model/io/jroIO_base.py +7 -16

             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
+            from schainpy.utils import log
             LOCALTIME = True
                         fileList = glob.glob1(thisPath, "*%s" % ext)
                         fileList.sort()
-                        skippedFileList = []
+                        for file in fileList:
-                        if cursor is not None and skip is not None:
-                            if skip == 0:
-                                skippedFileList = []
-                            else:
-                                skippedFileList = fileList[cursor *
-                                                           skip: cursor * skip + skip]
-                        else:
-                            skippedFileList = fileList
-                        for file in skippedFileList:
                             filename = os.path.join(thisPath, file)
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
+                    if cursor is not None and skip is not None:
+                        filenameList = filenameList[cursor * skip:cursor * skip + skip]
+                        datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
                     if not(filenameList):
                         print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path)
                         return [], []
                     print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
-                    print
                     # for i in range(len(filenameList)):
                     #     print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                         setFile = 0
                         self.nTotalBlocks = 0
-                    filen = '%s%4.4d%3.3d%3.3d%s' % (
+                    filen = '{}{:04d}{:03d}{:03d}{}'.format(
                             self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
                     filename = os.path.join(path, subfolder, filen)

schainpy/model/utils/jroutils_publish.py +33 -37

                 Object to hold data to be plotted
                 '''
-                def __init__(self, plottypes, throttle_value):
+                def __init__(self, plottypes, throttle_value, exp_code):
                     self.plottypes = plottypes
                     self.throttle = throttle_value
+                    self.exp_code = exp_code
                     self.ended = False
                     self.localtime = False
                     self.__times = []
                         return
                     self.parameters = getattr(dataOut, 'parameters', [])
+                    if hasattr(dataOut, 'pairsList'):
                         self.pairs = dataOut.pairsList
                     self.channels = dataOut.channelList
                     self.interval = dataOut.getTimeInterval()
                     Convert data to json
                     '''
-                    ret = {}
+                    data = {}
                     tm = self.times[-1]
-                    for key, value in self.data:
+                    for key in self.data:
                         if key in ('spc', 'cspc'):
-                            ret[key] = roundFloats(self.data[key].to_list())
+                            dx = int(self.data[key].shape[1]/MAXNUMX) + 1
+                            dy = int(self.data[key].shape[2]/MAXNUMY) + 1
+                            data[key] = roundFloats(self.data[key][::, ::dx, ::dy].tolist())
                         else:
-                            ret[key] = roundFloats(self.data[key][tm].to_list())
+                            data[key] = roundFloats(self.data[key][tm].tolist())
-                    ret['timestamp'] = tm
+                    ret = {'data': data}
+                    ret['exp_code'] = self.exp_code
+                    ret['time'] = tm
                     ret['interval'] = self.interval
+                    ret['localtime'] = self.localtime
+                    ret['yrange'] = roundFloats(self.heights.tolist())
+                    if key in ('spc', 'cspc'):
+                        ret['xrange'] = roundFloats(self.xrange[2][::dx].tolist())
+                    if hasattr(self, 'pairs'):
+                        ret['pairs'] = self.pairs
+                    return json.dumps(ret)
                 @property
                 def times(self):
             class PlotterReceiver(ProcessingUnit, Process):
                 throttle_value = 5
-                __attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle']
+                __attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle',
+                    'exp_code', 'web_server']
                 def __init__(self, **kwargs):
                     self.isWebConfig = False
                     self.connections = 0
                     server = kwargs.get('server', 'zmq.pipe')
-                    plot_server = kwargs.get('plot_server', 'zmq.web')
+                    web_server = kwargs.get('web_server', None)
                     if 'tcp://' in server:
                         address = server
                     else:
                         address = 'ipc:///tmp/%s' % server
-                    if 'tcp://' in plot_server:
-                        plot_address = plot_server
-                    else:
-                        plot_address = 'ipc:///tmp/%s' % plot_server
                     self.address = address
-                    self.plot_address = plot_address
+                    self.web_address = web_server
                     self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
                     self.realtime = kwargs.get('realtime', False)
                     self.localtime = kwargs.get('localtime', True)
                     self.throttle_value = kwargs.get('throttle', 5)
+                    self.exp_code = kwargs.get('exp_code', None)
                     self.sendData = self.initThrottle(self.throttle_value)
                     self.dates = []
                     self.setup()
                 def setup(self):
-                    self.data = Data(self.plottypes, self.throttle_value)
+                    self.data = Data(self.plottypes, self.throttle_value, self.exp_code)
                     self.isConfig = True
                 def event_monitor(self, monitor):
                     self.receiver.bind(self.address)
                     monitor = self.receiver.get_monitor_socket()
                     self.sender = self.context.socket(zmq.PUB)
-                    if self.realtime:
+                    if self.web_address:
+                        log.success(
+                            'Sending to web: {}'.format(self.web_address),
+                            self.name
+                        )
                         self.sender_web = self.context.socket(zmq.PUB)
-                        self.sender_web.connect(self.plot_address)
+                        self.sender_web.connect(self.web_address)
                         time.sleep(1)
                     if 'server' in self.kwargs:
                         else:
                             if self.realtime:
                                 self.send(self.data)
-                                # self.sender_web.send_string(self.data.jsonify())
+                                if self.web_address:
+                                    self.sender_web.send(self.data.jsonify())
                             else:
                                 self.sendData(self.send, self.data, coerce=coerce)
                                 coerce = False
                     return
-                def sendToWeb(self):
-                    if not self.isWebConfig:
-                        context = zmq.Context()
-                        sender_web_config = context.socket(zmq.PUB)
-                        if 'tcp://' in self.plot_address:
-                            dum, address, port = self.plot_address.split(':')
-                            conf_address = '{}:{}:{}'.format(dum, address, int(port)+1)
-                        else:
-                            conf_address = self.plot_address + '.config'
-                        sender_web_config.bind(conf_address)
-                        time.sleep(1)
-                        for kwargs in self.operationKwargs.values():
-                            if 'plot' in kwargs:
-                                log.success('[Sending] Config data to web for {}'.format(kwargs['code'].upper()))
-                                sender_web_config.send_string(json.dumps(kwargs))
-                        self.isWebConfig = True

setup.py +7 -17

             from setuptools import setup, Extension
             from setuptools.command.build_ext import build_ext as _build_ext
             from schainpy import __version__
-            from schainpy.utils import log
             class build_ext(_build_ext):
                 def finalize_options(self):
                     import numpy
                     self.include_dirs.append(numpy.get_include())
-            try:
-                from PyQt4 import QtCore, QtGui
-                from PyQt4.QtGui import QApplication
-            except:
-                log.warning(
-                    'You should install PyQt4 module in order to run the GUI. See the README.')
             setup(name = "schainpy",
                   version = __version__,
                   description = "Python tools to read, write and process Jicamarca data",
                               'schainpy.model.proc',
                               'schainpy.model.serializer',
                               'schainpy.model.utils',
+                              'schainpy.utils',
                               'schainpy.gui',
                               'schainpy.gui.figures',
                               'schainpy.gui.viewcontroller',
                               'schainpy.gui.viewer',
-                              'schainpy.gui.viewer.windows'},
+                              'schainpy.gui.viewer.windows',
+                              'schainpy.cli'},
                   ext_package = 'schainpy',
-                  py_modules=[''],
                   package_data = {'': ['schain.conf.template'],
                                   'schainpy.gui.figures': ['*.png', '*.jpg'],
                                  },
                   include_package_data = False,
                   scripts = ['schainpy/gui/schainGUI'],
                   ext_modules = [
-                      Extension("cSchain", ["schainpy/model/proc/extensions.c"]
+                      Extension("cSchain", ["schainpy/model/proc/extensions.c"])
-                                )],
+                      ],
                   entry_points = {
                       'console_scripts': [
-                          'schain = schaincli.cli:main',
+                          'schain = schainpy.cli.cli:main',
                       ],
                   },
                   cmdclass = {'build_ext': build_ext},
                   install_requires = [
                       "scipy >= 0.14.0",
                       "h5py >= 2.2.1",
-                      "matplotlib >= 1.4.2",
+                      "matplotlib >= 2.0.0",
                       "pyfits >= 3.4",
                       "paramiko >= 2.1.2",
                       "paho-mqtt >= 1.2",

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages