schain Commit - r1328:3de2afa39084 · Jicamarca Repository

Merge branch 'v3.0-devel'

Juan C. Espinoza -

r1328:3de2afa39084

parent child

Context file:

r1328:3de2afa39084

Expand all files

The requested changes are too big and content was truncated. Show full diff

schainc/_noise.c created 644 +82 0

@@ -0,0 +1,82
	1	#include <Python.h>
	2	#include <numpy/arrayobject.h>
	3	#include <math.h>
	4
	5
	6	static PyObject hildebrand_sekhon(PyObject self, PyObject *args) {
	7	double navg;
	8	PyObject data_obj, data_array;
	9
	10	if (!PyArg_ParseTuple(args, "Od", &data_obj, &navg)) {
	11	return NULL;
	12	}
	13
	14	data_array = PyArray_FROM_OTF(data_obj, NPY_FLOAT64, NPY_IN_ARRAY);
	15
	16	if (data_array == NULL) {
	17	Py_XDECREF(data_array);
	18	Py_XDECREF(data_obj);
	19	return NULL;
	20	}
	21	double sortdata = (double)PyArray_DATA(data_array);
	22	int lenOfData = (int)PyArray_SIZE(data_array) ;
	23	double nums_min = lenOfData*0.2;
	24	if (nums_min <= 5) nums_min = 5;
	25	double sump = 0;
	26	double sumq = 0;
	27	int j = 0;
	28	int cont = 1;
	29	double rtest = 0;
	30	while ((cont == 1) && (j < lenOfData)) {
	31	sump = sump + sortdata[j];
	32	sumq = sumq + pow(sortdata[j], 2);
	33	if (j > nums_min) {
	34	rtest = (double)j/(j-1) + 1/navg;
	35	if ((sumqj) > (rtestpow(sump, 2))) {
	36	j = j - 1;
	37	sump = sump - sortdata[j];
	38	sumq = sumq - pow(sortdata[j],2);
	39	cont = 0;
	40	}
	41	}
	42	j = j + 1;
	43	}
	44
	45	double lnoise = sump / j;
	46
	47	Py_DECREF(data_array);
	48
	49	return PyLong_FromLong(lnoise);
	50	//return Py_BuildValue("d", lnoise);
	51	}
	52
	53
	54	static PyMethodDef noiseMethods[] = {
	55	{ "hildebrand_sekhon", hildebrand_sekhon, METH_VARARGS, "Get noise with hildebrand_sekhon algorithm" },
	56	{ NULL, NULL, 0, NULL }
	57	};
	58
	59	#if PY_MAJOR_VERSION >= 3
	60
	61	static struct PyModuleDef noisemodule = {
	62	PyModuleDef_HEAD_INIT,
	63	"_noise",
	64	"Get noise with hildebrand_sekhon algorithm",
	65	-1,
	66	noiseMethods
	67	};
	68
	69	#endif
	70
	71	#if PY_MAJOR_VERSION >= 3
	72	PyMODINIT_FUNC PyInit__noise(void) {
	73	Py_Initialize();
	74	import_array();
	75	return PyModule_Create(&noisemodule);
	76	}
	77	#else
	78	PyMODINIT_FUNC init_noise() {
	79	Py_InitModule("_noise", noiseMethods);
	80	import_array();
	81	}
	82	#endif

schainpy/model/graphics/jroplot_base.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/bltrIO_spectra.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_simulator.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/USRP_PLOT_THOR.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/pedestal_client.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/pedestal_server.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/test_001.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/test_002.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/test_sim0001.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/test_sim00010.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/test_sim0002.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/test_sim0003.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/test_sim0004.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/test_sim0005.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/test_sim0006.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/test_sim0007.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/test_sim0008.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/test_sim0009.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

schainpy/scripts/wr_integrador.py created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

.gitignore 0 -2

             # Byte-compiled / optimized / DLL files
             __pycache__/
             *.py[cod]
             *$py.class
             # C extensions
             *.so
             # Distribution / packaging
             .Python
             env/
             build/
             develop-eggs/
             dist/
             downloads/
             eggs/
             .eggs/
             lib/
             lib64/
             parts/
             sdist/
             var/
             wheels/
             *.egg-info/
             .installed.cfg
             *.egg
             # PyInstaller
             #  Usually these files are written by a python script from a template
             #  before PyInstaller builds the exe, so as to inject date/other infos into it.
             *.manifest
             *.spec
             # Installer logs
             pip-log.txt
             pip-delete-this-directory.txt
             # Unit test / coverage reports
             htmlcov/
             .tox/
             .coverage
             .coverage.*
             .cache
             nosetests.xml
             coverage.xml
             *,cover
             .hypothesis/
             # Translations
             *.mo
             *.pot
             # Django stuff:
             *.log
             local_settings.py
             # Flask stuff:
             instance/
             .webassets-cache
             # Scrapy stuff:
             .scrapy
             # Sphinx documentation
             docs/_build/
             # PyBuilder
             target/
             # Jupyter Notebook
             .ipynb_checkpoints
             # pyenv
             .python-version
             # celery beat schedule file
             celerybeat-schedule
             # SageMath parsed files
             *.sage.py
             # dotenv
             .env
             # virtualenv
             .venv
             venv/
             ENV/
             # Spyder project settings
             .spyderproject
             .spyproject
             # Rope project settings
             .ropeproject
             # mkdocs documentation
             /site
             # eclipse
             .project
             .pydevproject
             # vscode
             .vscode
             schaingui/node_modules/
             schainpy/scripts/
             .svn/
             *.png
             *.pyc
             .vscode
             trash
             *.log
-            schainpy/scripts/testDigitalRF.py
-            schainpy/scripts/testDigitalRFWriter.py

CHANGELOG.md ~~schainpy/CHANGELOG.md~~ renamed +30 -21

-            ## CHANGELOG:
+            # CHANGELOG:
-            ### 2.3
+            ## 3.0
+            * Python 3.x & 2.X compatible
+            * New architecture with multiprocessing support
+            * Add @MPDecorator for multiprocessing Operations (Plots, Writers and Publishers)
+            * Added new type of operation `external` for non-locking operations
+            * New plotting architecture with buffering/throttle capabilities to speed up plots
+            * Clean controller to optimize scripts (format & optype are no longer required)
+            * New GUI with dinamic load of Units and operations (use Kivy framework)
+            ## 2.3
             * Added support for Madrigal formats (reading/writing).
             * Added support for reading BLTR parameters (*.sswma).
             * Added support for reading Julia format (*.dat).
             * Added high order function `MPProject` for multiprocessing scripts.
             * Added two new Processing Units `PublishData` and `ReceiverData` for receiving and sending dataOut through multiple ways (tcp, ipc, inproc).
             * Added a new graphics Processing Unit `PlotterReceiver`. It is decoupled from normal processing sequence with support for data generated by multiprocessing scripts.
             * Added support for sending realtime graphic to web server.
             * GUI command `schain` is now `schainGUI`.
             * Added a CLI tool named `schain`.
             	* Scripts templates can be now generated with `schain generate`.
             	* Now it is possible to search Processing Units and Operations with `schain search [module]` to get the right name and its allowed parameters.
             	* `schain xml` to run xml scripts.
             * Added suggestions when parameters are poorly written.
             * `Controller.start()` now runs in a different process than the process calling it.
             * Added `schainpy.utils.log` for log standarization.
             * Running script on online mode no longer ignores date and hour. Issue #1109.
             * Added support for receving voltage data directly from JARS (tcp, ipc).
             * Updated README for MAC OS GUI installation.
             * Setup now installs numpy.
-            ### 2.2.6
+            ## 2.2.6
             * Graphics generated by the GUI are now the same as generated by scripts. Issue #1074.
             * Added support for C extensions.
             * Function `hildebrand_sehkon` optimized with a C wrapper.
             * Numpy version updated.
             * Migration to GIT.
-            ### 2.2.5:
+            ## 2.2.5:
             * splitProfiles and combineProfiles modules were added to VoltageProc and Signal Chain GUI.
             * nProfiles of USRP data (hdf5) is the number of profiles thera are in one second.
             * jroPlotter works directly with data objects instead of dictionaries
             * script "schain" was added to Signal Chain installer
-            ### 2.2.4.1:
+            ## 2.2.4.1:
             * jroIO_usrp.py is update to read Sandra's data
             * decimation in Spectra and RTI plots is always enabled.
             * time* window option added to GUI
-            ### 2.2.4:
+            ## 2.2.4:
             * jroproc_spectra_lags.py added to schainpy
             * Bug fixed in schainGUI: ProcUnit was created with the same id in some cases.
             * Bug fixed in jroHeaderIO: Header size validation.
-            ### 2.2.3.1:
+            ## 2.2.3.1:
             * Filtering block by time has been added.
             * Bug fixed plotting RTI, CoherenceMap and others using xmin and xmax parameters. The first day worked
             properly but the next days did not.
-            ### 2.2.3:
+            ## 2.2.3:
             * Bug fixed in GUI: Error getting(reading) Code value
             * Bug fixed in GUI: Flip option always needs channelList field
             * Bug fixed in jrodata: when one branch modified a value in "dataOut" (example: dataOut.code) this value
             was modified for every branch (because this was a reference). It was modified in data.copy()
             * Bug fixed in jroproc_voltage.profileSelector(): rangeList replaces to profileRangeList.
-            ### 2.2.2:
+            ## 2.2.2:
             * VoltageProc: ProfileSelector, Reshape, Decoder with nTxs!=1 and getblock=True was tested
             * Rawdata and testRawdata.py added to Signal Chain project
-            ### 2.2.1:
+            ## 2.2.1:
             * Bugs fixed in GUI
             * Views were improved in GUI
             * Support to MST* ISR experiments
             * Bug fixed getting noise using hyldebrant. (minimum number of points > 20%)
             * handleError added to jroplotter.py
-            ### 2.2.0:
+            ## 2.2.0:
             * GUI: use of external plotter
             * Compatible with matplotlib 1.5.0
-            ### 2.1.5:
+            ## 2.1.5:
             * serializer module added to Signal Chain
             * jroplotter.py added to Signal Chain
-            ### 2.1.4.2:
+            ## 2.1.4.2:
             * A new Plotter Class was added
             * Project.start() does not accept filename as a parameter anymore
-            ### 2.1.4.1:
+            ## 2.1.4.1:
             * Send notifications when an error different to ValueError is detected
-            ### 2.1.4:
+            ## 2.1.4:
             * Sending error notifications to signal chain administrator
             * Login to email server added
-            ### 2.1.3.3:
+            ## 2.1.3.3:
             * Colored Button Icons were added to GUI
-            ### 2.1.3.2:
+            ## 2.1.3.2:
             * GUI: user interaction enhanced
             * controller_api.py: Safe access to ControllerThead
-            ### 2.1.3.1:
+            ## 2.1.3.1:
             * GUI: every icon were resized
             * jroproc_voltage.py: Print a message when "Read from code" option is selected and the code is not defined inside data file
-            ### 2.1.3:
+            ## 2.1.3:
             * jroplot_heispectra.py: SpectraHeisScope was not showing the right channels
             * jroproc_voltage.py: Bug fixed selecting profiles (self.nProfiles took a wrong value),
             					 Bug fixed selecting heights by block (selecting profiles instead heights)
             * jroproc_voltage.py: New feature added: decoding data by block using FFT.
             * jroIO_heispectra.py: Bug fixed in FitsReader. Using local Fits instance instead schainpy.mode.data.jrodata.Fits.
             * jroIO_heispectra.py: Channel index list does not exist.
-            ### 2.1.2:
+            ## 2.1.2:
             * jroutils_ftp.py: Bug fixed, Any error sending file stopped the Server Thread
             				  Server thread opens and closes remote server each time file list is sent
             * jroplot_spectra.py: Noise path was not being created when noise data is saved.
             * jroIO_base.py: startTime can be greater than endTime. Example: SpreadF [18:00 *  07:00]

Dockerfile +2 -2

-            FROM python:2.7-slim
+            FROM python:3.7-slim
             RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommends \
             	git \
             	gcc \
             	libpng-dev \
             	libfreetype6-dev \
             	libopenblas-dev \
             	liblapack-dev \
             	libatlas-base-dev \
             	libssl-dev \
             	libhdf5-dev \
-            	&& git clone --branch v2.3 --depth 1 \
+            	&& git clone --branch v3.0-devel --depth 1 \
             	http://jro-dev.igp.gob.pe/rhodecode/schain \
             	&& pip install numpy \
             	&& cd schain \
             	&& pip install . \
             	&& rm -rf * \
             	&& apt-get purge -y --auto-remove git gcc \
             	&& rm -rf /var/lib/apt/lists/*
             ENV BACKEND="Agg"
             VOLUME /data
             ENTRYPOINT ["schain"]

README.md +63 -60

             # Signal Chain
             ## Introduction
             Signal Chain (SCh) is a radar data processing library developed using [Python](www.python.org) at JRO. SCh provides modules to read, write, process and plot data.
             ## Installation
-            Install system dependencies, clone the latest version from [here](http://jro-dev.igp.gob.pe/rhodecode/schain/) and install it as a normal python package.
+            Install system dependencies, clone the latest version from [here](http://jro-dev.igp.gob.pe/rhodecode/schain/) and install it as a normal python package, we strongly recommend to use Anaconda or a virtual environment for the installation.
-            ### Linux based system
+            ### Dependencies
+            - GCC (gcc or gfortran)
+            - Python.h (python-dev or python-devel)
+            - Python-TK (python-tk)
+            - HDF5 libraries (libhdf5-dev)
+            ### Linux based system (e.g. ubuntu)
             ```
-            $ sudo apt-get install python-pip python-dev gfortran libpng-dev freetype* libblas-dev liblapack-dev libatlas-base-dev python-qt4 python-tk libssl-dev libhdf5-dev
             $ git clone http://jro-dev.igp.gob.pe/rhodecode/schain/
             $ cd schain
+            $ git checkout `schain-branch` (optional)
             $ sudo pip install ./
             ```
             ### MAC Os
             ```
             $ brew install python
-            $ brew install cartr/qt4/pyqt
             $ git clone http://jro-dev.igp.gob.pe/rhodecode/schain/
             $ cd schain
-            $ pip install ./
+            $ git checkout `schain-branch` (optional)
-            ```
+            $ sudo pip install ./
-            **It is recommended to install schain in a virtual environment**
-            ```
-            $ virtualenv /path/to/virtual
-            $ source /path/to/virtual/bin/activate
-            (virtual) $ cd schain
-            (virtual) $ pip install ./
-            (virtual) $ bash link_PyQt4.sh
             ```
             ### Docker
             Download Dockerfile from the repository, and create a docker image
             ```
             $ docker build -t schain .
             ```
             You can run a container using an xml file or a schain script also you need to mount a volume for the data input and for the output files/plots
             ```
             $ docker run -it --rm --volume /path/to/host/data:/data schain xml /data/test.xml
-            $ docker run -it --rm --volume /path/to/host/data:/data --entrypoint=/bin/python schain /data/test.py
+            $ docker run -it --rm --volume /path/to/host/data:/data --entrypoint /urs/local/bin/python schain /data/test.py
             ```
+            ## CLI (command line interface)
+            Signal Chain provides the following commands:
+            - schainGUI: Open the GUI
+            - schain: Signal chain command line
             ## First Script
-            Read Spectra data (.pdata) - remove dc - plot spectra & RTI
+            Here you can find an script to read Spectra data (.pdata), remove dc and plot spectra & RTI
-            Import SCh and creating a project
+            First import SCh and creating a project
             ```python
             #!/usr/bin/python
             from schainpy.controller import Project
-            controller = Project()
+            prj = Project()
-            controller.setup(id = '100',
+            prj.setup(
-                             name='test',
+                id = '100',
-                             description='Basic experiment')
+                name='test',
+                description='Basic experiment'
+                )
             ```
-            Adding read unit and operations
+            Add read unit and operations
             ```python
-            read_unit = controller.addReadUnit(datatype='Spectra',
+            read_unit = prj.addReadUnit(
-                                               path='/path/to/pdata/',
+                datatype='Spectra',
-                                               startDate='2014/01/31',
+                path='/path/to/pdata/',
-                                               endDate='2014/03/31',
+                startDate='2014/01/31',
-                                               startTime='00:00:00',
+                endDate='2014/03/31',
-                                               endTime='23:59:59',
+                startTime='00:00:00',
-                                               online=0,
+                endTime='23:59:59',
-                                               walk=0)
+                online=0,
+                walk=0
-            proc_unit = controller.addProcUnit(datatype='Spectra',
-                                               inputId=read_unit.getId())
+            proc_unit = prj.addProcUnit(datatype='Spectra', inputId=read_unit.getId())
             op = proc_unit.addOperation(name='selectChannels')
-            op.addParameter(name='channelList', value='0,1', format='intlist')
+            op.addParameter(name='channelList', value='0,1')
             op = proc_unit.addOperation(name='selectHeights')
-            op.addParameter(name='minHei', value='80', format='float')
+            op.addParameter(name='minHei', value='80')
-            op.addParameter(name='maxHei', value='200', format='float')
+            op.addParameter(name='maxHei', value='200')
             op = proc_unit.addOperation(name='removeDC')
             ```
-            Plotting data & start project
+            Plot data & start project
             ```python
-            op = proc_unit.addOperation(name='SpectraPlot', optype='other')
+            op = proc_unit.addOperation(name='SpectraPlot')
-            op.addParameter(name='id', value='1', format='int')
+            op.addParameter(name='id', value='1')
-            op.addParameter(name='wintitle', value='Spectra', format='str')
+            op.addParameter(name='wintitle', value='Spectra')
-            op = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
+            op = procUnitConfObj1.addOperation(name='RTIPlot')
-            op.addParameter(name='id', value='2', format='int')
+            op.addParameter(name='id', value='2')
-            op.addParameter(name='wintitle', value='RTI', format='str')
+            op.addParameter(name='wintitle', value='RTI')
-            controller.start()
+            prj.start()
             ```
             Full script
             ```python
             #!/usr/bin/python
-            from schainpy.controller import Project
+            from schainpy.prj import Project
-            controller = Project()
+            prj = Project()
-            controller.setup(id = '100',
+            prj.setup(id = '100',
                              name='test',
                              description='Basic experiment')
-            read_unit = controller.addReadUnit(datatype='Spectra',
+            read_unit = prj.addReadUnit(datatype='Spectra',
                                                path='/path/to/pdata/',
                                                startDate='2014/01/31',
                                                endDate='2014/03/31',
                                                startTime='00:00:00',
                                                endTime='23:59:59',
                                                online=0,
                                                walk=0)
-            proc_unit = controller.addProcUnit(datatype='Spectra',
+            proc_unit = prj.addProcUnit(datatype='Spectra',
                                                inputId=read_unit.getId())
             op = proc_unit.addOperation(name='selectChannels')
-            op.addParameter(name='channelList', value='0,1', format='intlist')
+            op.addParameter(name='channelList', value='0,1')
             op = proc_unit.addOperation(name='selectHeights')
-            op.addParameter(name='minHei', value='80', format='float')
+            op.addParameter(name='minHei', value='80')
-            op.addParameter(name='maxHei', value='200', format='float')
+            op.addParameter(name='maxHei', value='200')
             op = proc_unit.addOperation(name='removeDC')
-            op = proc_unit.addOperation(name='SpectraPlot', optype='other')
+            op = proc_unit.addOperation(name='SpectraPlot')
-            op.addParameter(name='id', value='6', format='int')
             op.addParameter(name='wintitle', value='Spectra', format='str')
-            op = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
+            op = procUnitConfObj1.addOperation(name='RTIPlot')
-            op.addParameter(name='id', value='2', format='int')
             op.addParameter(name='wintitle', value='RTI', format='str')
-            controller.start()
+            prj.start()
             ```

schainpy/__init__.py +2 -2

             '''
-            Created on Feb 7, 2012
+            Created on Jul 3, 2018
             @author $Author$
             @version $Id$
             '''
-            __version__ = '2.3'
+            __version__ = '3.0'

schainpy/admin.py +12 -10

             """
             The admin module contains all administrative classes relating to the schain python api.
             The main role of this module is to send some reports. It contains a
             notification class and a standard error handing class.
             $Id: admin.py 3966 2015-12-01 14:32:29Z miguel.urco $
             """
             import os
             import sys
             import time
             import traceback
             import smtplib
-            import ConfigParser
+            if sys.version[0] == '3':
-            import StringIO
+                from configparser import ConfigParser
+            else:
+                from ConfigParser import ConfigParser
+            import io
             from threading import Thread
             from multiprocessing import Process
             from email.mime.text import MIMEText
             from email.mime.application import MIMEApplication
             from email.mime.multipart import MIMEMultipart
             import schainpy
             from schainpy.utils import log
-            from schainpy.model.graphics.jroplot_data import popup
+            from schainpy.model.graphics.jroplot_base import popup
             def get_path():
                 '''
                 Return schainpy path
                 '''
                 try:
                     root = __file__
                     if os.path.islink(root):
                         root = os.path.realpath(root)
                     return os.path.dirname(os.path.abspath(root))
                 except:
                     log.error('I am sorry, but something is wrong... __file__ not found')
             class Alarm(Process):
                 '''
                 modes:
 - All
 - Send email
 - Popup message
 - Sound alarm
 - Send to alarm system TODO
                 '''
                 def __init__(self, modes=[], **kwargs):
                     Process.__init__(self)
                     self.modes = modes
                     self.kwargs = kwargs
                 @staticmethod
                 def play_sound():
                     sound = os.path.join(get_path(), 'alarm1.oga')
                     if os.path.exists(sound):
                         for __ in range(2):
                             os.system('paplay {}'.format(sound))
                             time.sleep(0.5)
                     else:
                         log.warning('Unable to play alarm, sound file not found', 'ADMIN')
                 @staticmethod
                 def send_email(**kwargs):
                     notifier = SchainNotify()
-                    print kwargs
                     notifier.notify(**kwargs)
                 @staticmethod
                 def show_popup(message):
                     if isinstance(message, list):
                         message = message[-1]
                     popup(message)
                 @staticmethod
                 def send_alarm():
                     pass
                 @staticmethod
                 def get_kwargs(kwargs, keys):
                     ret = {}
                     for key in keys:
                         ret[key] = kwargs[key]
                     return ret
                 def run(self):
                     tasks = {
 : self.send_email,
 : self.show_popup,
 : self.play_sound,
 : self.send_alarm,
                     }
                     tasks_args = {
 : ['email', 'message', 'subject', 'subtitle', 'filename'],
 : ['message'],
 : [],
 : [],
                     }
                     procs = []
                     for mode in self.modes:
                         if 0 in self.modes:
                             for x in tasks:
                                 t = Thread(target=tasks[x], kwargs=self.get_kwargs(self.kwargs, tasks_args[x]))
                                 t.start()
                                 procs.append(t)
                             break
                         else:
                             t = Thread(target=tasks[mode], kwargs=self.get_kwargs(self.kwargs, tasks_args[mode]))
                             t.start()
                             procs.append(t)
                     for t in procs:
                         t.join()
             class SchainConfigure():
                 __DEFAULT_ADMINISTRATOR_EMAIL = "juan.espinoza@jro.igp.gob.pe"
                 __DEFAULT_EMAIL_SERVER = "jro-zimbra.igp.gob.pe"
                 __DEFAULT_SENDER_EMAIL = "notifier-schain@jro.igp.gob.pe"
                 __DEFAULT_SENDER_PASS = ""
                 __SCHAIN_ADMINISTRATOR_EMAIL = "CONTACT"
                 __SCHAIN_EMAIL_SERVER = "MAILSERVER"
                 __SCHAIN_SENDER_EMAIL = "MAILSERVER_ACCOUNT"
                 __SCHAIN_SENDER_PASS = "MAILSERVER_PASSWORD"
                 def __init__(self, initFile = None):
                     # Set configuration file
                     if (initFile == None):
                         self.__confFilePath = "/etc/schain.conf"
                     else:
                         self.__confFilePath = initFile
                     # open configuration file
                     try:
                         self.__confFile = open(self.__confFilePath, "r")
                     except IOError:
                         # can't read from file - use all hard-coded values
                         self.__initFromHardCode()
                         return
                     # create Parser using standard module ConfigParser
-                    self.__parser = ConfigParser.ConfigParser()
+                    self.__parser = ConfigParser()
                     # read conf file into a StringIO with "[madrigal]\n" section heading prepended
-                    strConfFile = StringIO.StringIO("[schain]\n" + self.__confFile.read())
+                    strConfFile = io.StringIO("[schain]\n" + self.__confFile.read())
                     # parse StringIO configuration file
                     self.__parser.readfp(strConfFile)
                     # read information from configuration file
                     self.__readConfFile()
                     # close conf file
                     self.__confFile.close()
                 def __initFromHardCode(self):
                     self.__sender_email = self.__DEFAULT_SENDER_EMAIL
                     self.__sender_pass = self.__DEFAULT_SENDER_PASS
                     self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
                     self.__email_server = self.__DEFAULT_EMAIL_SERVER
                 def __readConfFile(self):
                     """__readConfFile is a private helper function that reads information from the parsed config file.
                     Inputs: None
                     Returns: Void.
                     Affects: Initializes class member variables that are found in the config file.
                     Exceptions: MadrigalError thrown if any key not found.
                     """
                     # get the sender email
                     try:
                         self.__sender_email = self.__parser.get("schain", self.__SCHAIN_SENDER_EMAIL)
                     except:
                         self.__sender_email = self.__DEFAULT_SENDER_EMAIL
                     # get the sender password
                     try:
                         self.__sender_pass = self.__parser.get("schain", self.__SCHAIN_SENDER_PASS)
                     except:
                         self.__sender_pass = self.__DEFAULT_SENDER_PASS
                     # get the administrator email
                     try:
                         self.__admin_email = self.__parser.get("schain", self.__SCHAIN_ADMINISTRATOR_EMAIL)
                     except:
                         self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
                     # get the server email
                     try:
                         self.__email_server = self.__parser.get("schain", self.__SCHAIN_EMAIL_SERVER)
                     except:
                         self.__email_server = self.__DEFAULT_EMAIL_SERVER
                 def getEmailServer(self):
                     return self.__email_server
                 def getSenderEmail(self):
                     return self.__sender_email
                 def getSenderPass(self):
                     return self.__sender_pass
                 def getAdminEmail(self):
                     return self.__admin_email
             class SchainNotify:
                 """SchainNotify is an object used to send messages to an administrator about a Schain software.
                 This object provides functions needed to send messages to an administrator about a Schain , for now
                 only sendAlert, which sends an email to the site administrator found is ADMIN_EMAIL
                 Usage example:
                     import schainpy.admin
                     try:
                         adminObj =  schainpy.admin.SchainNotify()
                         adminObj.sendAlert('This is important!', 'Important Message')
                     except schainpy.admin.SchainError, e:
                         print e.getExceptionStr()
                 Non-standard Python modules used:
                 None
                 Exceptions thrown: None - Note that SchainNotify tries every trick it knows to avoid
                 throwing exceptions, since this is the class that will generally be called when there is a problem.
                 Change history:
                 Written by "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe  Dec. 1, 2015
                 """
                 #constants
                 def __init__(self):
                     """__init__ initializes SchainNotify by getting some basic information from SchainDB and SchainSite.
                     Note that SchainNotify tries every trick it knows to avoid throwing exceptions, since
                     this is the class that will generally be called when there is a problem.
                     Inputs: Existing SchainDB object, by default = None.
                     Returns: void
                     Affects: Initializes self.__binDir.
                     Exceptions: None.
                     """
                     # note that the main configuration file is unavailable
                     # the best that can be done is send an email to root using localhost mailserver
                     confObj = SchainConfigure()
                     self.__emailFromAddress = confObj.getSenderEmail()
                     self.__emailPass = confObj.getSenderPass()
                     self.__emailToAddress = confObj.getAdminEmail()
                     self.__emailServer  = confObj.getEmailServer()
                 def sendEmail(self, email_from, email_to, subject='Error running ...', message="", subtitle="", filename="", html_format=True):
                     if not email_to:
                         return 0
                     if not self.__emailServer:
                         return 0
                     log.success('Sending email to {}...'.format(email_to), 'System')
                     msg = MIMEMultipart()
                     msg['Subject'] = subject
-                    msg['From'] = "(Python SChain API): " + email_from
+                    msg['From'] = "SChain API (v{}) <{}>".format(schainpy.__version__, email_from)
                     msg['Reply-to'] = email_from
                     msg['To'] = email_to
                     # That is what u see if dont have an email reader:
                     msg.preamble = 'SChainPy'
                     if html_format:
                         message = "<h1> %s </h1>" %subject + "<h3>" + subtitle.replace("\n", "</h3><h3>\n") + "</h3>" + message.replace("\n", "<br>\n")
                         message = "<html>\n" + message + '</html>'
                         # This is the textual part:
                         part = MIMEText(message, "html")
                     else:
                         message = subject + "\n" + subtitle + "\n" + message
                         part = MIMEText(message)
                     msg.attach(part)
                     if filename and os.path.isfile(filename):
                         # This is the binary part(The Attachment):
                         part = MIMEApplication(open(filename,"rb").read())
                         part.add_header('Content-Disposition',
                                         'attachment',
                                         filename=os.path.basename(filename))
                         msg.attach(part)
                     # Create an instance in SMTP server
                     try:
                         smtp = smtplib.SMTP(self.__emailServer)
                     except:
                         log.error('Could not connect to server {}'.format(self.__emailServer), 'System')
                         return 0
                     # Start the server:
                 #         smtp.ehlo()
                     if self.__emailPass:
                         smtp.login(self.__emailFromAddress, self.__emailPass)
                     # Send the email
                     try:
                         smtp.sendmail(msg['From'], msg['To'], msg.as_string())
                     except:
                         log.error('Could not send the email to {}'.format(msg['To']), 'System')
                         smtp.quit()
                         return 0
                     smtp.quit()
                     log.success('Email sent ', 'System')
                     return 1
                 def sendAlert(self, message, subject = "", subtitle="", filename=""):
                     """sendAlert sends an email with the given message and optional title.
                     Inputs: message (string), and optional title (string)
                     Returns: void
                     Affects: none
                     Exceptions: None.
                     """
                     if not self.__emailToAddress:
                         return 0
-                    print "***** Sending alert to %s *****" %self.__emailToAddress
+                    print("***** Sending alert to %s *****" %self.__emailToAddress)
                     # set up message
                     sent=self.sendEmail(email_from=self.__emailFromAddress,
                                        email_to=self.__emailToAddress,
                                        subject=subject,
                                        message=message,
                                        subtitle=subtitle,
                                        filename=filename)
                     if not sent:
                         return 0
                     return 1
                 def notify(self, email, message, subject = "", subtitle="", filename=""):
                     """notify sends an email with the given message and title to email.
                     Inputs: email (string), message (string), and subject (string)
                     Returns: void
                     Affects: none
                     Exceptions: None.
                     """
                     if email is None:
                         email = self.__emailToAddress
                     self.sendEmail(
                         email_from=self.__emailFromAddress,
                         email_to=email,
                         subject=subject,
                         message=message,
                         subtitle=subtitle,
                         filename=filename
                         )
             class SchainError(Exception):
                 """SchainError is an exception class that is thrown for all known errors using Schain Py lib.
                 Usage example:
                     import sys, traceback
                     import schainpy.admin
                     try:
                         test = open('ImportantFile.txt', 'r')
                     except:
                         raise schainpy.admin.SchainError('ImportantFile.txt not opened!',
                                                             traceback.format_exception(sys.exc_info()[0],
                                                                                     sys.exc_info()[1],
                                                                                     sys.exc_info()[2]))
                 """
                 def __init__(self, strInterpretation, exceptionList=None):
                     """ __init__ gathers the interpretation string along with all information from sys.exc_info().
                     Inputs:
                             strIntepretation - A string representing the programmer's interpretation of
                             why the exception occurred
                             exceptionList - a list of strings completely describing the exception.
                             Generated by traceback.format_exception(sys.exc_info()[0],
                                                                     sys.exc_info()[1],
                                                                     sys.exc_info()[2])
                     Returns: Void.
                     Affects: Initializes class member variables _strInterp, _strExcList.
                     Exceptions: None.
                     """
                     if not exceptionList:
                         exceptionList = traceback.format_exception(sys.exc_info()[0],
                                                                     sys.exc_info()[1],
                                                                     sys.exc_info()[2])
                     self._strInterp = strInterpretation
                     self._strExcList = exceptionList
                 def getExceptionStr(self):
                     """ getExceptionStr returns a formatted string ready for printing completely describing the exception.
                     Inputs: None
                     Returns: A formatted string ready for printing completely describing the exception.
                     Affects: None
                     Exceptions: None.
                     """
                     excStr = ''
                     excStr = excStr + self._strInterp + '\n\n'
                     if self._strExcList != None:
                         for item in self._strExcList:
                             excStr = excStr + str(item) + '\n'
                     return excStr
                 def __str__(self):
                     return(self.getExceptionStr())
                 def getExceptionHtml(self):
                     """ getExceptionHtml returns an Html formatted string completely describing the exception.
                     Inputs: None
                     Returns: A formatted string ready for printing completely describing the exception.
                     Affects: None
                     Exceptions: None.
                     """
                     excStr = '<BR>The following Schain Python exception has occurred:\n<BR>'
                     excStr = excStr + self._strInterp + '\n<BR>\n'
                     if self._strExcList != None:
                         for item in self._strExcList:
                             excStr = excStr + str(item) + '\n<BR>'
                     return excStr
             class SchainWarning(Exception):
                 pass
             if __name__ == '__main__':
                 test = SchainNotify()
                 test.sendAlert('This is a message from the python module SchainNotify', 'Test from SchainNotify')
-                print 'Hopefully message sent - check.'
+                print('Hopefully message sent - check.')
  No newline at end of file

schainpy/cli/cli.py +118 -48

             import click
-            import schainpy
             import subprocess
             import os
             import sys
             import glob
-            save_stdout = sys.stdout
+            import schainpy
-            sys.stdout = open('trash', 'w')
-            from multiprocessing import cpu_count
             from schainpy.controller import Project
             from schainpy.model import Operation, ProcessingUnit
             from schainpy.utils import log
             from importlib import import_module
             from pydoc import locate
             from fuzzywuzzy import process
-            from schainpy.utils import paramsFinder
+            from schainpy.cli import templates
-            import templates
+            import inspect
-            sys.stdout = save_stdout
+            try:
+                from queue import Queue
+            except:
+                from Queue import Queue
+            def getProcs():
+                modules = dir(schainpy.model)
+                procs = check_module(modules, 'processing')
+                try:
+                    procs.remove('ProcessingUnit')
+                except Exception as e:
+                    pass
+                return procs
+            def getOperations():
+                module = dir(schainpy.model)
+                noProcs = [x for x in module if not x.endswith('Proc')]
+                operations = check_module(noProcs, 'operation')
+                try:
+                    operations.remove('Operation')
+                    operations.remove('Figure')
+                    operations.remove('Plot')
+                except Exception as e:
+                    pass
+                return operations
+            def getArgs(op):
+                module = locate('schainpy.model.{}'.format(op))
+                try:
+                    obj = module(1, 2, 3, Queue())
+                except:
+                    obj = module()
+                if hasattr(obj, '__attrs__'):
+                    args = obj.__attrs__
+                else:
+                    if hasattr(obj, 'myrun'):
+                        args = inspect.getfullargspec(obj.myrun).args
+                    else:
+                        args = inspect.getfullargspec(obj.run).args
+                try:
+                    args.remove('self')
+                except Exception as e:
+                    pass
+                try:
+                    args.remove('dataOut')
+                except Exception as e:
+                    pass
+                return args
+            def getDoc(obj):
+                module = locate('schainpy.model.{}'.format(obj))
+                try:
+                    obj = module(1, 2, 3, Queue())
+                except:
+                    obj = module()
+                return obj.__doc__
+            def getAll():
+                modules = getOperations()
+                modules.extend(getProcs())
+                return modules
             def print_version(ctx, param, value):
                 if not value or ctx.resilient_parsing:
                     return
                 click.echo(schainpy.__version__)
                 ctx.exit()
-            cliLogger = log.makelogger('schain cli')
             PREFIX = 'experiment'
             @click.command()
             @click.option('--version', '-v', is_flag=True, callback=print_version, help='SChain version', type=str)
             @click.argument('command', default='run', required=True)
             @click.argument('nextcommand', default=None, required=False, type=str)
             def main(command, nextcommand, version):
-                """COMMAND LINE INTERFACE FOR SIGNAL CHAIN - JICAMARCA RADIO OBSERVATORY \n
+                """COMMAND LINE INTERFACE FOR SIGNAL CHAIN - JICAMARCA RADIO OBSERVATORY V3.0\n
-                    Available commands.\n
+                    Available commands:\n
-                    --xml: runs a schain XML generated file\n
+                    xml: runs a schain XML generated file\n
-                    run: runs any python script starting 'experiment_'\n
+                    run: runs any python script'\n
                     generate: generates a template schain script\n
-                    search: return avilable operations, procs or arguments of the give operation/proc\n"""
+                    list: return a list of available procs and operations\n
+                    search: return avilable operations, procs or arguments of the given
+                            operation/proc\n"""
                 if command == 'xml':
                     runFromXML(nextcommand)
                 elif command == 'generate':
                     generate()
                 elif command == 'test':
                     test()
                 elif command == 'run':
                     runschain(nextcommand)
                 elif command == 'search':
                     search(nextcommand)
+                elif command == 'list':
+                    cmdlist(nextcommand)
                 else:
                     log.error('Command {} is not defined'.format(command))
             def check_module(possible, instance):
                 def check(x):
                     try:
                         instancia = locate('schainpy.model.{}'.format(x))
-                        return isinstance(instancia(), instance)
+                        ret = instancia.proc_type == instance
+                        return ret
                     except Exception as e:
                         return False
                 clean = clean_modules(possible)
                 return [x for x in clean if check(x)]
             def clean_modules(module):
                 noEndsUnder = [x for x in module if not x.endswith('__')]
                 noStartUnder = [x for x in noEndsUnder if not x.startswith('__')]
                 noFullUpper = [x for x in noStartUnder if not x.isupper()]
                 return noFullUpper
+            def cmdlist(nextcommand):
-            def search(nextcommand):
                 if nextcommand is None:
-                    log.error('There is no Operation/ProcessingUnit to search')
+                    log.error('Missing argument, available arguments: procs, operations', '')
                 elif nextcommand == 'procs':
-                    procs = paramsFinder.getProcs()
+                    procs = getProcs()
                     log.success(
-                        'Current ProcessingUnits are:\n\033[1m{}\033[0m'.format('\n'.join(procs)))
+                        'Current ProcessingUnits are:\n  {}'.format('\n  '.join(procs)), '')
                 elif nextcommand == 'operations':
-                    operations = paramsFinder.getOperations()
+                    operations = getOperations()
-                    log.success('Current Operations are:\n\033[1m{}\033[0m'.format(
+                    log.success('Current Operations are:\n  {}'.format(
-                        '\n'.join(operations)))
+                        '\n  '.join(operations)), '')
+                else:
+                    log.error('Wrong argument', '')
+            def search(nextcommand):
+                if nextcommand is None:
+                    log.error('There is no Operation/ProcessingUnit to search', '')
                 else:
                     try:
-                        args = paramsFinder.getArgs(nextcommand)
+                        args = getArgs(nextcommand)
-                        log.warning(
+                        doc = getDoc(nextcommand)
-                            'Use this feature with caution. It may not return all the allowed arguments')
+                        log.success('{}\n{}\n\narguments:\n  {}'.format(
-                        if len(args) == 0:
+                            nextcommand, doc, ', '.join(args)), ''
-                            log.success('{} has no arguments'.format(nextcommand))
-                        else:
-                            log.success('Showing {} arguments:\n\033[1m{}\033[0m'.format(
-                                nextcommand, '\n'.join(args)))
                     except Exception as e:
-                        log.error('Module {} does not exists'.format(nextcommand))
+                        log.error('Module `{}` does not exists'.format(nextcommand), '')
-                        allModules = paramsFinder.getAll()
+                        allModules = getAll()
-                        similar = process.extractOne(nextcommand, allModules)[0]
+                        similar = [t[0] for t in process.extract(nextcommand, allModules, limit=12) if t[1]>80]
-                        log.success('Showing {} instead'.format(similar))
+                        log.success('Possible modules are: {}'.format(', '.join(similar)), '')
-                        search(similar)
             def runschain(nextcommand):
                 if nextcommand is None:
                     currentfiles = glob.glob('./{}_*.py'.format(PREFIX))
                     numberfiles = len(currentfiles)
                     if numberfiles > 1:
                         log.error('There is more than one file to run')
                     elif numberfiles == 1:
                         subprocess.call(['python ' + currentfiles[0]], shell=True)
                     else:
                         log.error('There is no file to run')
                 else:
                     try:
                         subprocess.call(['python ' + nextcommand], shell=True)
                     except Exception as e:
                         log.error("I cannot run the file. Does it exists?")
             def basicInputs():
                 inputs = {}
-                inputs['desc'] = click.prompt(
-                    'Enter a description', default="A schain project", type=str)
                 inputs['name'] = click.prompt(
                     'Name of the project', default="project", type=str)
+                inputs['desc'] = click.prompt(
+                    'Enter a description', default="A schain project", type=str)
+                inputs['multiprocess'] = click.prompt(
+                    '''Select data type:
+                - Voltage (*.r):                [1]
+                - Spectra (*.pdata):            [2]
+                - Voltage and Spectra (*.r):    [3]
+                -->''', type=int)
                 inputs['path'] = click.prompt('Data path', default=os.getcwd(
                 ), type=click.Path(exists=True, resolve_path=True))
                 inputs['startDate'] = click.prompt(
                     'Start date', default='1970/01/01', type=str)
                 inputs['endDate'] = click.prompt(
-                    'End date', default='2017/12/31', type=str)
+                    'End date', default='2018/12/31', type=str)
                 inputs['startHour'] = click.prompt(
                     'Start hour', default='00:00:00', type=str)
                 inputs['endHour'] = click.prompt('End hour', default='23:59:59', type=str)
                 inputs['figpath'] = inputs['path'] + '/figs'
                 return inputs
             def generate():
                 inputs = basicInputs()
-                inputs['multiprocess'] = click.confirm('Is this a multiprocess script?')
-                if inputs['multiprocess']:
+                if inputs['multiprocess'] == 1:
-                    inputs['nProcess'] = click.prompt(
+                    current = templates.voltage.format(**inputs)
-                        'How many process?', default=cpu_count(), type=int)
+                elif inputs['multiprocess'] == 2:
-                    current = templates.multiprocess.format(**inputs)
+                    current = templates.spectra.format(**inputs)
-                else:
+                elif inputs['multiprocess'] == 3:
-                    current = templates.basic.format(**inputs)
+                    current = templates.voltagespectra.format(**inputs)
                 scriptname = '{}_{}.py'.format(PREFIX, inputs['name'])
                 script = open(scriptname, 'w')
                 try:
                     script.write(current)
                     log.success('Script {} generated'.format(scriptname))
                 except Exception as e:
                     log.error('I cannot create the file. Do you have writing permissions?')
             def test():
                 log.warning('testing')
             def runFromXML(filename):
                 controller = Project()
                 if not controller.readXml(filename):
                     return
                 controller.start()
                 return

schainpy/cli/templates.py +217 -38

@@ -1,90 +1,269
1	basic = '''from schainpy.controller import Project	1	voltage = '''import os, sys, time
		2	from schainpy.controller import Project
		3
		4
		5	def main():
		6	desc = "{desc}"
		7	controller = Project()
		8	controller.setup(id='200', name="{name}", description=desc)
		9
		10	read_unit = controller.addReadUnit(datatype='Voltage',
		11	path="{path}",
		12	startDate="{startDate}",
		13	endDate="{endDate}",
		14	startTime="{startHour}",
		15	endTime="{endHour}",
		16	online=0,
		17	verbose=1,
		18	walk=0,
		19	delay=180,
		20	)
		21
		22	code = '[[1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1]]'
		23	nCode = '128'
		24	nBaud = '3'
		25
		26
		27	proc_voltage = controller.addProcUnit(name='VoltageProc', inputId=read_unit.getId())
		28
		29	op1 = proc_voltage.addOperation(name='selectChannels', optype='self')
		30	op1.addParameter(name='channelList', value='0, 1, 2, 3', format='intlist')
		31
		32	op2 = proc_voltage.addOperation(name='filterByHeights', optype='self')
		33	op2.addParameter(name='window', value='4', format='int')
		34
		35	op3 = proc_voltage.addOperation(name='ProfileSelector', optype='other')
		36	op3.addParameter(name='profileRangeList', value='32, 159', format='intList')
		37
		38	op4 = proc_voltage.addOperation(name='Decoder', optype='other')
		39	op4.addParameter(name='code', value=code, format='intlist')
		40	op4.addParameter(name='nCode', value=nCode, format='int')
		41	op4.addParameter(name='nBaud', value=nBaud, format='int')
		42	op4.addParameter(name='mode', value='0', format='int')
		43
		44	op5 = proc_voltage.addOperation(name='Scope', optype='external')
		45	op5.addParameter(name='id', value='30', format='int')
		46
		47
		48
		49
		50
		51	controller.start()
		52
		53	if __name__ == '__main__':
		54	import time
		55	start_time = time.time()
		56	main()
		57	print("--- %s seconds ---" % (time.time() - start_time))
2		58
3	desc = "{desc}"
4	project = Project()
5	project.setup(id='200', name="{name}", description=desc)
6
7	voltage_reader = project.addReadUnit(datatype='VoltageReader',
8	path="{path}",
9	startDate="{startDate}",
10	endDate="{endDate}",
11	startTime="{startHour}",
12	endTime="{endHour}",
13	online=0,
14	verbose=1,
15	walk=1,
16	)
17
18	voltage_proc = project.addProcUnit(datatype='VoltageProc', inputId=voltage_reader.getId())
19
20	profile = voltage_proc.addOperation(name='ProfileSelector', optype='other')
21	profile.addParameter(name='profileRangeList', value='120,183', format='intlist')
22
23	rti = voltage_proc.addOperation(name='RTIPlot', optype='other')
24	rti.addParameter(name='wintitle', value='Jicamarca Radio Observatory', format='str')
25	rti.addParameter(name='showprofile', value='0', format='int')
26	rti.addParameter(name='xmin', value='0', format='int')
27	rti.addParameter(name='xmax', value='24', format='int')
28	rti.addParameter(name='figpath', value="{figpath}", format='str')
29	rti.addParameter(name='wr_period', value='5', format='int')
30	rti.addParameter(name='exp_code', value='22', format='int')
31
32
33	project.start()
34	'''	59	'''
35		60
		61
		62	spectra = '''import os, sys, time
		63	from schainpy.controller import Project
		64
		65
		66	def main():
		67	desc = "{desc}"
		68	controller = Project()
		69	controller.setup(id='300', name="{name}", description=desc)
		70
		71	read_unit = controller.addReadUnit(datatype='Spectra',
		72	path="{path}",
		73	startDate="{startDate}",
		74	endDate="{endDate}",
		75	startTime="{startHour}",
		76	endTime="{endHour}",
		77	online=0,
		78	verbose=1,
		79	walk=0,
		80	delay=180,
		81	)
		82
		83	proc_spectra = controller.addProcUnit(datatype='Spectra', inputId=read_unit.getId())
		84	proc_spectra.addParameter(name='nFFTPoints', value='128', format='int')
		85	proc_spectra.addParameter(name='nProfiles', value='128', format='int')
		86	proc_spectra.addParameter(name='pairsList', value='(0, 1), (2, 3)', format='pairslist')
		87
		88	op1 = proc_spectra.addOperation(name='IncohInt', optype='other')
		89	op1.addParameter(name='n', value='4', format='int')
		90
		91	op2 = proc_spectra.addOperation(name='CrossSpectraPlot', optype='external')
		92	op2.addParameter(name='id', value='10', format='int')
		93	op2.addParameter(name='zmin', value='10.0', format='float')
		94	op2.addParameter(name='zmax', value='35.0', format='float')
		95
		96
		97	op3 = proc_spectra.addOperation(name='RTIPlot', optype='external')
		98	op3.addParameter(name='id', value='20', format='int')
		99	op3.addParameter(name='wintitle', value='RTI', format='str')
		100	op3.addParameter(name='xmin', value='0', format='float')
		101	op3.addParameter(name='xmax', value='24', format='float')
		102	op3.addParameter(name='zmin', value='12', format='int')
		103	op3.addParameter(name='zmax', value='32', format='int')
		104	op3.addParameter(name='showprofile', value='1', format='int')
		105	op3.addParameter(name='timerange', value=str(246060), format='int')
		106
		107	op4 = proc_spectra.addOperation(name='CoherenceMap', optype='external')
		108	op4.addParameter(name='id', value='30', format='int')
		109	op4.addParameter(name='xmin', value='0.0', format='float')
		110	op4.addParameter(name='xmax', value='24.0', format='float')
		111
		112
		113	controller.start()
		114
		115	if __name__ == '__main__':
		116	import time
		117	start_time = time.time()
		118	main()
		119	print("--- %s seconds ---" % (time.time() - start_time))
		120
		121	'''
		122
		123	voltagespectra = '''import os, sys, time
		124	from schainpy.controller import Project
		125
		126
		127	def main():
		128	desc = "{desc}"
		129	controller = Project()
		130	controller.setup(id='400', name="{name}", description=desc)
		131
		132	read_unit = controller.addReadUnit(datatype='Voltage',
		133	path="{path}",
		134	startDate="{startDate}",
		135	endDate="{endDate}",
		136	startTime="{startHour}",
		137	endTime="{endHour}",
		138	online=0,
		139	verbose=1,
		140	walk=0,
		141	delay=180,
		142	)
		143
		144	code = '[[1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [1, 1, -1], [-1, -1, 1], [-1, -1, 1], [-1, -1, 1], [1, 1, -1]]'
		145	nCode = '128'
		146	nBaud = '3'
		147
		148
		149	proc_voltage = controller.addProcUnit(name='VoltageProc', inputId=read_unit.getId())
		150
		151	op1 = proc_voltage.addOperation(name='selectChannels', optype='self')
		152	op1.addParameter(name='channelList', value='0, 1, 2, 3', format='intlist')
		153
		154	op2 = proc_voltage.addOperation(name='filterByHeights', optype='self')
		155	op2.addParameter(name='window', value='4', format='int')
		156
		157	op3 = proc_voltage.addOperation(name='ProfileSelector', optype='other')
		158	op3.addParameter(name='profileRangeList', value='32, 159', format='intList')
		159
		160	op4 = proc_voltage.addOperation(name='Decoder', optype='other')
		161	op4.addParameter(name='code', value=code, format='intlist')
		162	op4.addParameter(name='nCode', value=nCode, format='int')
		163	op4.addParameter(name='nBaud', value=nBaud, format='int')
		164	op4.addParameter(name='mode', value='0', format='int')
		165
		166
		167
		168	proc_spectra = controller.addProcUnit(datatype='Spectra', inputId=proc_voltage.getId())
		169	proc_spectra.addParameter(name='nFFTPoints', value='128', format='int')
		170	proc_spectra.addParameter(name='nProfiles', value='128', format='int')
		171	proc_spectra.addParameter(name='pairsList', value='(0, 1), (2, 3)', format='pairslist')
		172
		173	op5 = proc_spectra.addOperation(name='IncohInt', optype='other')
		174	op5.addParameter(name='n', value='4', format='int')
		175
		176	op6 = proc_spectra.addOperation(name='CrossSpectraPlot', optype='external')
		177	op6.addParameter(name='id', value='10', format='int')
		178	op6.addParameter(name='zmin', value='10.0', format='float')
		179	op6.addParameter(name='zmax', value='35.0', format='float')
		180
		181
		182	op7 = proc_spectra.addOperation(name='RTIPlot', optype='external')
		183	op7.addParameter(name='id', value='20', format='int')
		184	op7.addParameter(name='wintitle', value='RTI', format='str')
		185	op7.addParameter(name='xmin', value='0', format='float')
		186	op7.addParameter(name='xmax', value='24', format='float')
		187	op7.addParameter(name='zmin', value='12', format='int')
		188	op7.addParameter(name='zmax', value='32', format='int')
		189	op7.addParameter(name='showprofile', value='1', format='int')
		190	op7.addParameter(name='timerange', value=str(246060), format='int')
		191
		192	op8 = proc_spectra.addOperation(name='CoherenceMap', optype='external')
		193	op8.addParameter(name='id', value='30', format='int')
		194	op8.addParameter(name='xmin', value='0.0', format='float')
		195	op8.addParameter(name='xmax', value='24.0', format='float')
		196
		197
		198	controller.start()
		199
		200	if __name__ == '__main__':
		201	import time
		202	start_time = time.time()
		203	main()
		204	print("--- %s seconds ---" % (time.time() - start_time))
		205
		206	'''
		207
		208
		209
		210
		211
		212
		213
		214
36	multiprocess = '''from schainpy.controller import Project, MPProject	215	multiprocess = '''from schainpy.controller import Project, MPProject
37	from time import sleep	216	from time import sleep
38	desc = "{desc}"	217	desc = "{desc}"
39		218
40	####################	219	####################
41	# PLOTTER RECEIVER #	220	# PLOTTER RECEIVER #
42	####################	221	####################
43	plotter = Project()	222	plotter = Project()
44	plotter.setup(id='100', name='receiver', description=desc)	223	plotter.setup(id='100', name='receiver', description=desc)
45		224
46	receiver_plot = plotter.addProcUnit(name='PlotterReceiver')	225	receiver_plot = plotter.addProcUnit(name='PlotterReceiver')
47	receiver_plot.addParameter(name='throttle', value=20, format='int')	226	receiver_plot.addParameter(name='throttle', value=20, format='int')
48	receiver_plot.addParameter(name='plottypes', value='rti', format='str')	227	receiver_plot.addParameter(name='plottypes', value='rti', format='str')
49		228
50	rti = receiver_plot.addOperation(name='PlotRTIData', optype='other')	229	rti = receiver_plot.addOperation(name='PlotRTIData', optype='other')
51	rti.addParameter(name='zmin', value='-40.0', format='float')	230	rti.addParameter(name='zmin', value='-40.0', format='float')
52	rti.addParameter(name='zmax', value='100.0', format='float')	231	rti.addParameter(name='zmax', value='100.0', format='float')
53	rti.addParameter(name='decimation', value='200', format='int')	232	rti.addParameter(name='decimation', value='200', format='int')
54	rti.addParameter(name='xmin', value='0.0', format='int')	233	rti.addParameter(name='xmin', value='0.0', format='int')
55	rti.addParameter(name='colormap', value='jet', format='str')	234	rti.addParameter(name='colormap', value='jet', format='str')
56		235
57	plotter.start()	236	plotter.start()
58		237
59	sleep(2)	238	sleep(2)
60		239
61	################	240	################
62	# DATA EMITTER #	241	# DATA EMITTER #
63	################	242	################
64	~~project~~ = Project()	243	controller = Project()
65	~~project~~.setup(id='200', name="{name}", description=desc)	244	controller.setup(id='200', name="{name}", description=desc)
66		245
67	spectra_reader = ~~project~~.addReadUnit(datatype='SpectraReader',	246	spectra_reader = controller.addReadUnit(datatype='SpectraReader',
68	path="{path}",	247	path="{path}",
69	startDate={startDate},	248	startDate={startDate},
70	endDate={endDate},	249	endDate={endDate},
71	startTime="{startHour}",	250	startTime="{startHour}",
72	endTime="{endHour}",	251	endTime="{endHour}",
73	online=0,	252	online=0,
74	verbose=1,	253	verbose=1,
75	walk=1,	254	walk=1,
76	)	255	)
77		256
78	spectra_proc = ~~project~~.addProcUnit(datatype='Spectra', inputId=spectra_reader.getId())	257	spectra_proc = controller.addProcUnit(datatype='Spectra', inputId=spectra_reader.getId())
79		258
80	parameters_proc = ~~project~~.addProcUnit(datatype='ParametersProc', inputId=spectra_proc.getId())	259	parameters_proc = controller.addProcUnit(datatype='ParametersProc', inputId=spectra_proc.getId())
81	moments = parameters_proc.addOperation(name='SpectralMoments', optype='other')	260	moments = parameters_proc.addOperation(name='SpectralMoments', optype='other')
82		261
83	publish = parameters_proc.addOperation(name='PublishData', optype='other')	262	publish = parameters_proc.addOperation(name='PublishData', optype='other')
84	publish.addParameter(name='zeromq', value=1, format='int')	263	publish.addParameter(name='zeromq', value=1, format='int')
85	publish.addParameter(name='verbose', value=0, format='bool')	264	publish.addParameter(name='verbose', value=0, format='bool')
86		265
87	MPProject(~~project~~, 16)	266	MPProject(controller, 16)
88		267
89		268
90	'''	269	'''

schainpy/controller.py +381 -1075

This diff has been collapsed as it changes many lines, (1456 lines changed) Show them Hide them
	@@ -1,1342 +1,648
	1	'''	1	'''
	2	Created on September , 2012	2	Main routines to create a Signal Chain project
	3	@author:
	4	'''	3	'''
	5		4
			5	import re
	6	import sys	6	import sys
	7	import ast	7	import ast
	8	import datetime	8	import datetime
	9	import traceback	9	import traceback
	10	import math
	11	import time	10	import time
	12	from multiprocessing import Process, ~~cpu_count~~	11	from multiprocessing import Process, Queue
			12	from threading import Thread
			13	from xml.etree.ElementTree import ElementTree, Element, SubElement
	13		14
	14	from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
	15	from xml.dom import minidom
	16
	17	import schainpy
	18	from schainpy.admin import Alarm, SchainWarning	15	from schainpy.admin import Alarm, SchainWarning
	19	from schainpy.model import *	16	from schainpy.model import *
	20	from schainpy.utils import log	17	from schainpy.utils import log
	21		18
	22	DTYPES = {
	23	'Voltage': '.r',
	24	'Spectra': '.pdata'
	25	}
	26
	27
	28	def MPProject(project, n=cpu_count()):
	29	'''
	30	Project wrapper to run schain in n processes
	31	'''
	32
	33	rconf = project.getReadUnitObj()
	34	op = rconf.getOperationObj('run')
	35	dt1 = op.getParameterValue('startDate')
	36	dt2 = op.getParameterValue('endDate')
	37	tm1 = op.getParameterValue('startTime')
	38	tm2 = op.getParameterValue('endTime')
	39	days = (dt2 - dt1).days
	40
	41	for day in range(days + 1):
	42	skip = 0
	43	cursor = 0
	44	processes = []
	45	dt = dt1 + datetime.timedelta(day)
	46	dt_str = dt.strftime('%Y/%m/%d')
	47	reader = JRODataReader()
	48	paths, files = reader.searchFilesOffLine(path=rconf.path,
	49	startDate=dt,
	50	endDate=dt,
	51	startTime=tm1,
	52	endTime=tm2,
	53	ext=DTYPES[rconf.datatype])
	54	nFiles = len(files)
	55	if nFiles == 0:
	56	continue
	57	skip = int(math.ceil(nFiles / n))
	58	while nFiles > cursor * skip:
	59	rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
	60	skip=skip)
	61	p = project.clone()
	62	p.start()
	63	processes.append(p)
	64	cursor += 1
	65
	66	def beforeExit(exctype, value, trace):
	67	for process in processes:
	68	process.terminate()
	69	process.join()
	70	print traceback.print_tb(trace)
	71
	72	sys.excepthook = beforeExit
	73
	74	for process in processes:
	75	process.join()
	76	process.terminate()
	77
	78	time.sleep(3)
	79
	80
	81	class ParameterConf():
	82
	83	id = None
	84	name = None
	85	value = None
	86	format = None
	87
	88	__formated_value = None
	89
	90	ELEMENTNAME = 'Parameter'
	91
	92	def __init__(self):
	93
	94	self.format = 'str'
	95
	96	def getElementName(self):
	97
	98	return self.ELEMENTNAME
	99
	100	def getValue(self):
	101
	102	value = self.value
	103	format = self.format
	104
	105	if self.__formated_value != None:
	106
	107	return self.__formated_value
	108
	109	if format == 'obj':
	110	return value
	111
	112	if format == 'str':
	113	self.__formated_value = str(value)
	114	return self.__formated_value
	115
	116	if value == '':
	117	raise ValueError, '%s: This parameter value is empty' % self.name
	118
	119	if format == 'list':
	120	strList = value.split(',')
	121
	122	self.__formated_value = strList
	123
	124	return self.__formated_value
	125
	126	if format == 'intlist':
	127	'''
	128	Example:
	129	value = (0,1,2)
	130	'''
	131
	132	new_value = ast.literal_eval(value)
	133
	134	if type(new_value) not in (tuple, list):
	135	new_value = [int(new_value)]
	136
	137	self.__formated_value = new_value
	138
	139	return self.__formated_value
	140
	141	if format == 'floatlist':
	142	'''
	143	Example:
	144	value = (0.5, 1.4, 2.7)
	145	'''
	146
	147	new_value = ast.literal_eval(value)
	148
	149	if type(new_value) not in (tuple, list):
	150	new_value = [float(new_value)]
	151
	152	self.__formated_value = new_value
	153
	154	return self.__formated_value
	155
	156	if format == 'date':
	157	strList = value.split('/')
	158	intList = [int(x) for x in strList]
	159	date = datetime.date(intList[0], intList[1], intList[2])
	160
	161	self.__formated_value = date
	162
	163	return self.__formated_value
	164
	165	if format == 'time':
	166	strList = value.split(':')
	167	intList = [int(x) for x in strList]
	168	time = datetime.time(intList[0], intList[1], intList[2])
	169
	170	self.__formated_value = time
	171
	172	return self.__formated_value
	173
	174	if format == 'pairslist':
	175	'''
	176	Example:
	177	value = (0,1),(1,2)
	178	'''
	179
	180	new_value = ast.literal_eval(value)
	181
	182	if type(new_value) not in (tuple, list):
	183	raise ValueError, '%s has to be a tuple or list of pairs' % value
	184
	185	if type(new_value[0]) not in (tuple, list):
	186	if len(new_value) != 2:
	187	raise ValueError, '%s has to be a tuple or list of pairs' % value
	188	new_value = [new_value]
	189
	190	for thisPair in new_value:
	191	if len(thisPair) != 2:
	192	raise ValueError, '%s has to be a tuple or list of pairs' % value
	193
	194	self.__formated_value = new_value
	195
	196	return self.__formated_value
	197
	198	if format == 'multilist':
	199	'''
	200	Example:
	201	value = (0,1,2),(3,4,5)
	202	'''
	203	multiList = ast.literal_eval(value)
	204
	205	if type(multiList[0]) == int:
	206	multiList = ast.literal_eval('(' + value + ')')
	207
	208	self.__formated_value = multiList
	209
	210	return self.__formated_value
	211
	212	if format == 'bool':
	213	value = int(value)
	214
	215	if format == 'int':
	216	value = float(value)
	217
	218	format_func = eval(format)
	219
	220	self.__formated_value = format_func(value)
	221
	222	return self.__formated_value
	223
	224	def updateId(self, new_id):
	225
	226	self.id = str(new_id)
	227
	228	def setup(self, id, name, value, format='str'):
	229	self.id = str(id)
	230	self.name = name
	231	if format == 'obj':
	232	self.value = value
	233	else:
	234	self.value = str(value)
	235	self.format = str.lower(format)
	236
	237	self.getValue()
	238
	239	return 1
	240
	241	def update(self, name, value, format='str'):
	242
	243	self.name = name
	244	self.value = str(value)
	245	self.format = format
	246
	247	def makeXml(self, opElement):
	248	if self.name not in ('queue',):
	249	parmElement = SubElement(opElement, self.ELEMENTNAME)
	250	parmElement.set('id', str(self.id))
	251	parmElement.set('name', self.name)
	252	parmElement.set('value', self.value)
	253	parmElement.set('format', self.format)
	254		19
	255	def readXml(self, parmElement):	20	class ConfBase():
	256
	257	self.id = parmElement.get('id')
	258	self.name = parmElement.get('name')
	259	self.value = parmElement.get('value')
	260	self.format = str.lower(parmElement.get('format'))
	261
	262	# Compatible with old signal chain version
	263	if self.format == 'int' and self.name == 'idfigure':
	264	self.name = 'id'
	265
	266	def printattr(self):
	267
	268	print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
	269
	270
	271	class OperationConf():
	272
	273	id = None
	274	name = None
	275	priority = None
	276	type = None
	277
	278	parmConfObjList = []
	279
	280	ELEMENTNAME = 'Operation'
	281		21
	282	def __init__(self):	22	def __init__(self):
	283		23
	284	self.id = '0'	24	self.id = '0'
	285	self.name = None	25	self.name = None
	286	self.priority = None	26	self.priority = None
	287	self.type = ~~'self'~~	27	self.parameters = {}
			28	self.object = None
			29	self.operations = []
			30
			31	def getId(self):
	288		32
	289	def __getNewId(self):	33	return self.id
			34
			35	def getNewId(self):
	290		36
	291	return int(self.id) * 10 + len(self.~~parmConfObjList~~) + 1	37	return int(self.id) * 10 + len(self.operations) + 1
	292		38
	293	def updateId(self, new_id):	39	def updateId(self, new_id):
	294		40
	295	self.id = str(new_id)	41	self.id = str(new_id)
	296		42
	297	n = 1	43	n = 1
	298	for ~~parmObj~~ in self.~~parmConfObjList~~:	44	for conf in self.operations:
	299		45	conf_id = str(int(new_id) * 10 + n)
	300	idParm = str(int(new_id) * 10 + n)	46	conf.updateId(conf_id)
	301	parmObj.updateId(idParm)
	302
	303	n += 1	47	n += 1
	304		48
	305	def get~~ElementName~~(self):	49	def getKwargs(self):
	306
	307	return self.ELEMENTNAME
	308
	309	def getParameterObjList(self):
	310
	311	return self.parmConfObjList
	312
	313	def getParameterObj(self, parameterName):
	314
	315	for parmConfObj in self.parmConfObjList:
	316
	317	if parmConfObj.name != parameterName:
	318	continue
	319
	320	return parmConfObj
	321
	322	return None
	323
	324	def getParameterObjfromValue(self, parameterValue):
	325
	326	for parmConfObj in self.parmConfObjList:
	327		50
	328	if parmConfObj.getValue() != parameterValue:	51	params = {}
	329	continue
	330		52
	331	return parmConfObj.getValue()	53	for key, value in self.parameters.items():
			54	if value not in (None, '', ' '):
			55	params[key] = value
			56
			57	return params
	332		58
	333	return None	59	def update(self, **kwargs):
	334		60
	335	def getParameterValue(self, parameterName):	61	for key, value in kwargs.items():
			62	self.addParameter(name=key, value=value)
	336		63
	337	parameterObj = self.getParameterObj(parameterName)	64	def addParameter(self, name, value, format=None):
			65	'''
			66	'''
	338		67
	339	# if not parameterObj:	68	if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
	340	# return None	69	self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
			70	elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
			71	self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
			72	else:
			73	try:
			74	self.parameters[name] = ast.literal_eval(value)
			75	except:
			76	if isinstance(value, str) and ',' in value:
			77	self.parameters[name] = value.split(',')
			78	else:
			79	self.parameters[name] = value
			80
			81	def getParameters(self):
			82
			83	params = {}
			84	for key, value in self.parameters.items():
			85	s = type(value).__name__
			86	if s == 'date':
			87	params[key] = value.strftime('%Y/%m/%d')
			88	elif s == 'time':
			89	params[key] = value.strftime('%H:%M:%S')
			90	else:
			91	params[key] = str(value)
	341		92
	342	value = parameterObj.getValue()	93	return params
			94
			95	def makeXml(self, element):
	343		96
	344	return value	97	xml = SubElement(element, self.ELEMENTNAME)
			98	for label in self.xml_labels:
			99	xml.set(label, str(getattr(self, label)))
			100
			101	for key, value in self.getParameters().items():
			102	xml_param = SubElement(xml, 'Parameter')
			103	xml_param.set('name', key)
			104	xml_param.set('value', value)
			105
			106	for conf in self.operations:
			107	conf.makeXml(xml)
			108
			109	def __str__(self):
	345		110
	346	def getKwargs(self):	111	if self.ELEMENTNAME == 'Operation':
			112	s = ' {}[id={}]\n'.format(self.name, self.id)
			113	else:
			114	s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
	347		115
	348	kwargs = {}	116	for key, value in self.parameters.items():
			117	if self.ELEMENTNAME == 'Operation':
			118	s += ' {}: {}\n'.format(key, value)
			119	else:
			120	s += ' {}: {}\n'.format(key, value)
			121
			122	for conf in self.operations:
			123	s += str(conf)
	349		124
	350	for parmConfObj in self.parmConfObjList:	125	return s
	351	if self.name == 'run' and parmConfObj.name == 'datatype':
	352	continue
	353		126
	354	kwargs[parmConfObj.name] = parmConfObj.getValue()	127	class OperationConf(ConfBase):
	355		128
	356	return kwargs	129	ELEMENTNAME = 'Operation'
			130	xml_labels = ['id', 'name']
	357		131
	358	def setup(self, id, name, priority, ~~typ~~e):	132	def setup(self, id, name, priority, project_id, err_queue):
	359		133
	360	self.id = str(id)	134	self.id = str(id)
			135	self.project_id = project_id
	361	self.name = name	136	self.name = name
	362	self.type = ~~type~~	137	self.type = 'other'
	363	self.~~priority~~ = ~~priority~~	138	self.err_queue = err_queue
	364
	365	self.parmConfObjList = []
	366
	367	def removeParameters(self):
	368
	369	for obj in self.parmConfObjList:
	370	del obj
	371
	372	self.parmConfObjList = []
	373
	374	def addParameter(self, name, value, format='str'):
	375
	376	if value is None:
	377	return None
	378	id = self.__getNewId()
	379
	380	parmConfObj = ParameterConf()
	381	if not parmConfObj.setup(id, name, value, format):
	382	return None
	383
	384	self.parmConfObjList.append(parmConfObj)
	385
	386	return parmConfObj
	387
	388	def changeParameter(self, name, value, format='str'):
	389
	390	parmConfObj = self.getParameterObj(name)
	391	parmConfObj.update(name, value, format)
	392
	393	return parmConfObj
	394
	395	def makeXml(self, procUnitElement):
	396
	397	opElement = SubElement(procUnitElement, self.ELEMENTNAME)
	398	opElement.set('id', str(self.id))
	399	opElement.set('name', self.name)
	400	opElement.set('type', self.type)
	401	opElement.set('priority', str(self.priority))
	402
	403	for parmConfObj in self.parmConfObjList:
	404	parmConfObj.makeXml(opElement)
	405
	406	def readXml(self, opElement):
	407
	408	self.id = opElement.get('id')
	409	self.name = opElement.get('name')
	410	self.type = opElement.get('type')
	411	self.priority = opElement.get('priority')
	412		139
	413	# Compatible with old signal chain version	140	def readXml(self, element, project_id, err_queue):
	414	# Use of 'run' method instead 'init'
	415	if self.type == 'self' and self.name == 'init':
	416	self.name = 'run'
	417		141
	418	self.parmConfObjList = []	142	self.id = element.get('id')
			143	self.name = element.get('name')
			144	self.type = 'other'
			145	self.project_id = str(project_id)
			146	self.err_queue = err_queue
	419		147
	420	parmElementList = opElement.iter(ParameterConf().getElementName())	148	for elm in element.iter('Parameter'):
			149	self.addParameter(elm.get('name'), elm.get('value'))
	421		150
	422	for parmElement in parmElementList:	151	def createObject(self):
	423	parmConfObj = ParameterConf()
	424	parmConfObj.readXml(parmElement)
	425		152
	426	# Compatible with old signal chain version	153	className = eval(self.name)
	427	# If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
	428	if self.type != 'self' and self.name == 'Plot':
	429	if parmConfObj.format == 'str' and parmConfObj.name == 'type':
	430	self.name = parmConfObj.value
	431	continue
	432
	433	self.parmConfObjList.append(parmConfObj)
	434
	435	def printattr(self):
	436
	437	print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
	438	self.id,
	439	self.name,
	440	self.type,
	441	self.priority)
	442
	443	for parmConfObj in self.parmConfObjList:
	444	parmConfObj.printattr()
	445
	446	def createObject(self, plotter_queue=None):
	447
	448	if self.type == 'self':
	449	raise ValueError, 'This operation type cannot be created'
	450
	451	if self.type == 'plotter':
	452	if not plotter_queue:
	453	raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
	454
	455	opObj = Plotter(self.name, plotter_queue)
	456
	457	if self.type == 'external' or self.type == 'other':
	458		154
	459	className = eval(self.name)	155	if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
	460	kwargs = self.getKwargs()	156	kwargs = self.getKwargs()
			157	opObj = className(self.id, self.id, self.project_id, self.err_queue, **kwargs)
			158	opObj.start()
			159	self.type = 'external'
			160	else:
			161	opObj = className()
	461		162
	462	opObj = className(**kwargs)	163	self.object = opObj
	463
	464	return opObj	164	return opObj
	465		165
	466		166	class ProcUnitConf(ConfBase):
	467	class ProcUnitConf():
	468
	469	id = None
	470	name = None
	471	datatype = None
	472	inputId = None
	473	parentId = None
	474
	475	opConfObjList = []
	476
	477	procUnitObj = None
	478	opObjList = []
	479		167
	480	ELEMENTNAME = 'ProcUnit'	168	ELEMENTNAME = 'ProcUnit'
			169	xml_labels = ['id', 'inputId', 'name']
	481		170
	482	def __init__(self):	171	def setup(self, project_id, id, name, datatype, inputId, err_queue):
	483		172	'''
	484	self.id = None	173	'''
	485	self.datatype = None	174
	486	self.name = None
	487	self.inputId = None
	488
	489	self.opConfObjList = []
	490
	491	self.procUnitObj = None
	492	self.opObjDict = {}
	493
	494	def __getPriority(self):
	495
	496	return len(self.opConfObjList) + 1
	497
	498	def __getNewId(self):
	499
	500	return int(self.id) * 10 + len(self.opConfObjList) + 1
	501
	502	def getElementName(self):
	503
	504	return self.ELEMENTNAME
	505
	506	def getId(self):
	507
	508	return self.id
	509
	510	def updateId(self, new_id, parentId=parentId):
	511
	512	new_id = int(parentId) * 10 + (int(self.id) % 10)
	513	new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
	514
	515	# If this proc unit has not inputs
	516	if self.inputId == '0':
	517	new_inputId = 0
	518
	519	n = 1
	520	for opConfObj in self.opConfObjList:
	521
	522	idOp = str(int(new_id) * 10 + n)
	523	opConfObj.updateId(idOp)
	524
	525	n += 1
	526
	527	self.parentId = str(parentId)
	528	self.id = str(new_id)
	529	self.inputId = str(new_inputId)
	530
	531	def getInputId(self):
	532
	533	return self.inputId
	534
	535	def getOperationObjList(self):
	536
	537	return self.opConfObjList
	538
	539	def getOperationObj(self, name=None):
	540
	541	for opConfObj in self.opConfObjList:
	542
	543	if opConfObj.name != name:
	544	continue
	545
	546	return opConfObj
	547
	548	return None
	549
	550	def getOpObjfromParamValue(self, value=None):
	551
	552	for opConfObj in self.opConfObjList:
	553	if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
	554	continue
	555	return opConfObj
	556	return None
	557
	558	def getProcUnitObj(self):
	559
	560	return self.procUnitObj
	561
	562	def setup(self, id, name, datatype, inputId, parentId=None):
	563
	564	# Compatible with old signal chain version
	565	if datatype == None and name == None:	175	if datatype == None and name == None:
	566	raise ValueError, 'datatype or name should be defined'	176	raise ValueError('datatype or name should be defined')
	567		177
	568	if name == None:	178	if name == None:
	569	if 'Proc' in datatype:	179	if 'Proc' in datatype:
	570	name = datatype	180	name = datatype
	571	else:	181	else:
	572	name = '%sProc' % (datatype)	182	name = '%sProc' % (datatype)
	573		183
	574	if datatype == None:	184	if datatype == None:
	575	datatype = name.replace('Proc', '')	185	datatype = name.replace('Proc', '')
	576		186
	577	self.id = str(id)	187	self.id = str(id)
			188	self.project_id = project_id
	578	self.name = name	189	self.name = name
	579	self.datatype = datatype	190	self.datatype = datatype
	580	self.inputId = inputId	191	self.inputId = inputId
	581	self.~~parentId~~ = ~~parentId~~	192	self.err_queue = err_queue
	582		193	self.operations = []
	583	self.~~opConfObjList~~ = []	194	self.parameters = {}
	584
	585	self.addOperation(name='run', optype='self')
	586		195
	587	def removeOperations(self):	196	def removeOperation(self, id):
	588		197
	589	for obj in self.opConfObjList:	198	i = [1 if x.id==id else 0 for x in self.operations]
	590	del obj	199	self.operations.pop(i.index(1))
			200
			201	def getOperation(self, id):
	591		202
	592	self.opConfObjList = []	203	for conf in self.operations:
	593	self.addOperation(name='run')	204	if conf.id == id:
			205	return conf
	594		206
	595	def addParameter(self, **kwargs):	207	def addOperation(self, name, optype='self'):
	596	'''	208	'''
	597	Add parameters to 'run' operation
	598	'''	209	'''
	599	opObj = self.opConfObjList[0]
	600
	601	opObj.addParameter(**kwargs)
	602
	603	return opObj
	604
	605	def addOperation(self, name, optype='self'):
	606
	607	id = self.__getNewId()
	608	priority = self.__getPriority()
	609
	610	opConfObj = OperationConf()
	611	opConfObj.setup(id, name=name, priority=priority, type=optype)
	612
	613	self.opConfObjList.append(opConfObj)
	614
	615	return opConfObj
	616
	617	def makeXml(self, projectElement):
	618
	619	procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
	620	procUnitElement.set('id', str(self.id))
	621	procUnitElement.set('name', self.name)
	622	procUnitElement.set('datatype', self.datatype)
	623	procUnitElement.set('inputId', str(self.inputId))
	624
	625	for opConfObj in self.opConfObjList:
	626	opConfObj.makeXml(procUnitElement)
	627
	628	def readXml(self, upElement):
	629
	630	self.id = upElement.get('id')
	631	self.name = upElement.get('name')
	632	self.datatype = upElement.get('datatype')
	633	self.inputId = upElement.get('inputId')
	634
	635	if self.ELEMENTNAME == 'ReadUnit':
	636	self.datatype = self.datatype.replace('Reader', '')
	637
	638	if self.ELEMENTNAME == 'ProcUnit':
	639	self.datatype = self.datatype.replace('Proc', '')
	640
	641	if self.inputId == 'None':
	642	self.inputId = '0'
	643		210
	644	self.opConfObjList = []	211	id = self.getNewId()
			212	conf = OperationConf()
			213	conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
			214	self.operations.append(conf)
	645		215
	646	opElementList = upElement.iter(OperationConf().getElementName())	216	return conf
	647		217
	648	for opElement in opElementList:	218	def readXml(self, element, project_id, err_queue):
	649	opConfObj = OperationConf()
	650	opConfObj.readXml(opElement)
	651	self.opConfObjList.append(opConfObj)
	652		219
	653	def printattr(self):	220	self.id = element.get('id')
	654		221	self.name = element.get('name')
	655	print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,	222	self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
	656	self.id,	223	self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
	657	self.name,	224	self.project_id = str(project_id)
	658	self.datatype,	225	self.err_queue = err_queue
	659	self.inputId)	226	self.operations = []
	660		227	self.parameters = {}
	661	for opConfObj in self.opConfObjList:	228
	662	opConfObj.printattr()	229	for elm in element:
	663		230	if elm.tag == 'Parameter':
	664	def getKwargs(self):	231	self.addParameter(elm.get('name'), elm.get('value'))
	665		232	elif elm.tag == 'Operation':
	666	opObj = self.opConfObjList[0]	233	conf = OperationConf()
	667	kwargs = opObj.getKwargs()	234	conf.readXml(elm, project_id, err_queue)
	668		235	self.operations.append(conf)
	669	return kwargs
	670		236
	671	def createObjects(self, ~~plotter_queue~~=~~None~~):	237	def createObjects(self):
			238	'''
			239	Instancia de unidades de procesamiento.
			240	'''
	672		241
	673	className = eval(self.name)	242	className = eval(self.name)
	674	kwargs = self.getKwargs()	243	kwargs = self.getKwargs()
	675	procUnitObj = className(**~~kwargs~~)	244	procUnitObj = className()
	676		245	procUnitObj.name = self.name
	677	for opConfObj in self.opConfObjList:	246	log.success('creating process...', self.name)
	678		247
	679	if opConfObj.type == 'self' and self.name == 'run':	248	for conf in self.operations:
	680	~~continue~~	249
	681	elif opConfObj.type == 'self':	250	opObj = conf.createObject()
	682	procUnitObj.addOperationKwargs(	251
	683	opConfObj.id, **opConfObj.getKwargs())	252	log.success('adding operation: {}, type:{}'.format(
	684	con~~tinue~~	253	conf.name,
	685		254	conf.type), self.name)
	686	opObj = opConfObj.createObject(plotter_queue)	255
	687		256	procUnitObj.addOperation(conf, opObj)
	688	self.opObjDict[opConfObj.id] = opObj	257
	689		258	self.object = procUnitObj
	690	procUnitObj.addOperation(opObj, opConfObj.id)
	691
	692	self.procUnitObj = procUnitObj
	693
	694	return procUnitObj
	695		259
	696	def run(self):	260	def run(self):
	697		261	'''
	698	is_ok = False	262	'''
	699		263
	700	for opConfObj in self.opConfObjList:	264	return self.object.call(**self.getKwargs())
	701
	702	kwargs = {}
	703	for parmConfObj in opConfObj.getParameterObjList():
	704	if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
	705	continue
	706
	707	kwargs[parmConfObj.name] = parmConfObj.getValue()
	708
	709	sts = self.procUnitObj.call(opType=opConfObj.type,
	710	opName=opConfObj.name,
	711	opId=opConfObj.id)
	712
	713	is_ok = is_ok or sts
	714
	715	return is_ok
	716
	717	def close(self):
	718
	719	for opConfObj in self.opConfObjList:
	720	if opConfObj.type == 'self':
	721	continue
	722
	723	opObj = self.procUnitObj.getOperationObj(opConfObj.id)
	724	opObj.close()
	725
	726	self.procUnitObj.close()
	727
	728	return
	729		265
	730		266
	731	class ReadUnitConf(ProcUnitConf):	267	class ReadUnitConf(ProcUnitConf):
	732		268
	733	path = None
	734	startDate = None
	735	endDate = None
	736	startTime = None
	737	endTime = None
	738
	739	ELEMENTNAME = 'ReadUnit'	269	ELEMENTNAME = 'ReadUnit'
	740		270
	741	def __init__(self):	271	def __init__(self):
	742		272
	743	self.id = None	273	self.id = None
	744	self.datatype = None	274	self.datatype = None
	745	self.name = None	275	self.name = None
	746	self.inputId = None	276	self.inputId = None
	747		277	self.operations = []
	748	self.par~~entId~~ = ~~None~~	278	self.parameters = {}
	749		279
	750	self.opConfObjList = []	280	def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
	751	self.opObjList = []	281	startTime='', endTime='', server=None, **kwargs):
	752		282
	753	def getElementName(self):
	754
	755	return self.ELEMENTNAME
	756
	757	def setup(self, id, name, datatype, path='', startDate='', endDate='',
	758	startTime='', endTime='', parentId=None, server=None, **kwargs):
	759
	760	# Compatible with old signal chain version
	761	if datatype == None and name == None:	283	if datatype == None and name == None:
	762	raise ValueError, 'datatype or name should be defined'	284	raise ValueError('datatype or name should be defined')
	763	if name == None:	285	if name == None:
	764	if 'Reader' in datatype:	286	if 'Reader' in datatype:
	765	name = datatype	287	name = datatype
	766	datatype = name.replace('Reader','')	288	datatype = name.replace('Reader','')
	767	else:	289	else:
	768	name = '{}Reader'.format(datatype)	290	name = '{}Reader'.format(datatype)
	769	if datatype == None:	291	if datatype == None:
	770	if 'Reader' in name:	292	if 'Reader' in name:
	771	datatype = name.replace('Reader','')	293	datatype = name.replace('Reader','')
	772	else:	294	else:
	773	datatype = name	295	datatype = name
	774	name = '{}Reader'.format(name)	296	name = '{}Reader'.format(name)
	775		297
	776	self.id = id	298	self.id = id
			299	self.project_id = project_id
	777	self.name = name	300	self.name = name
	778	self.datatype = datatype	301	self.datatype = datatype
	779	if path != '':	302	self.err_queue = err_queue
	780	self.path = os.path.abspath(path)	303
	781	self.startDate = startDate	304	self.addParameter(name='path', value=path)
	782	self.endDate = endDate	305	self.addParameter(name='startDate', value=startDate)
	783	self.startTime = startTime	306	self.addParameter(name='endDate', value=endDate)
	784	self.endTime = endTime	307	self.addParameter(name='startTime', value=startTime)
	785	self.inputId = '0'	308	self.addParameter(name='endTime', value=endTime)
	786	self.parentId = parentId
	787	self.server = server
	788	self.addRunOperation(**kwargs)
	789
	790	def update(self, **kwargs):
	791
	792	if 'datatype' in kwargs:
	793	datatype = kwargs.pop('datatype')
	794	if 'Reader' in datatype:
	795	self.name = datatype
	796	else:
	797	self.name = '%sReader' % (datatype)
	798	self.datatype = self.name.replace('Reader', '')
	799
	800	attrs = ('path', 'startDate', 'endDate',
	801	'startTime', 'endTime', 'parentId')
	802
	803	for attr in attrs:
	804	if attr in kwargs:
	805	setattr(self, attr, kwargs.pop(attr))
	806
	807	self.inputId = '0'
	808	self.updateRunOperation(**kwargs)
	809
	810	def removeOperations(self):
	811
	812	for obj in self.opConfObjList:
	813	del obj
	814
	815	self.opConfObjList = []
	816
	817	def addRunOperation(self, **kwargs):
	818
	819	opObj = self.addOperation(name='run', optype='self')
	820
	821	if self.server is None:
	822	opObj.addParameter(
	823	name='datatype', value=self.datatype, format='str')
	824	opObj.addParameter(name='path', value=self.path, format='str')
	825	opObj.addParameter(
	826	name='startDate', value=self.startDate, format='date')
	827	opObj.addParameter(
	828	name='endDate', value=self.endDate, format='date')
	829	opObj.addParameter(
	830	name='startTime', value=self.startTime, format='time')
	831	opObj.addParameter(
	832	name='endTime', value=self.endTime, format='time')
	833
	834	for key, value in kwargs.items():
	835	opObj.addParameter(name=key, value=value,
	836	format=type(value).__name__)
	837	else:
	838	opObj.addParameter(name='server', value=self.server, format='str')
	839
	840	return opObj
	841
	842	def updateRunOperation(self, **kwargs):
	843
	844	opObj = self.getOperationObj(name='run')
	845	opObj.removeParameters()
	846
	847	opObj.addParameter(name='datatype', value=self.datatype, format='str')
	848	opObj.addParameter(name='path', value=self.path, format='str')
	849	opObj.addParameter(
	850	name='startDate', value=self.startDate, format='date')
	851	opObj.addParameter(name='endDate', value=self.endDate, format='date')
	852	opObj.addParameter(
	853	name='startTime', value=self.startTime, format='time')
	854	opObj.addParameter(name='endTime', value=self.endTime, format='time')
	855		309
	856	for key, value in kwargs.items():	310	for key, value in kwargs.items():
	857	~~opObj~~.addParameter(name=key, value=value,	311	self.addParameter(name=key, value=value)
	858	format=type(value).__name__)
	859
	860	return opObj
	861
	862	def readXml(self, upElement):
	863
	864	self.id = upElement.get('id')
	865	self.name = upElement.get('name')
	866	self.datatype = upElement.get('datatype')
	867	self.inputId = upElement.get('inputId')
	868
	869	if self.ELEMENTNAME == 'ReadUnit':
	870	self.datatype = self.datatype.replace('Reader', '')
	871
	872	if self.inputId == 'None':
	873	self.inputId = '0'
	874
	875	self.opConfObjList = []
	876
	877	opElementList = upElement.iter(OperationConf().getElementName())
	878
	879	for opElement in opElementList:
	880	opConfObj = OperationConf()
	881	opConfObj.readXml(opElement)
	882	self.opConfObjList.append(opConfObj)
	883
	884	if opConfObj.name == 'run':
	885	self.path = opConfObj.getParameterValue('path')
	886	self.startDate = opConfObj.getParameterValue('startDate')
	887	self.endDate = opConfObj.getParameterValue('endDate')
	888	self.startTime = opConfObj.getParameterValue('startTime')
	889	self.endTime = opConfObj.getParameterValue('endTime')
	890		312
	891		313
	892	class Project(Process):	314	class Project(Process):
	893		315
	894	id = None
	895	# name = None
	896	description = None
	897	filename = None
	898
	899	procUnitConfObjDict = None
	900
	901	ELEMENTNAME = 'Project'	316	ELEMENTNAME = 'Project'
	902		317
	903	plotterQueue = None	318	def __init__(self):
	904
	905	def __init__(self, plotter_queue=None):
	906		319
	907	Process.__init__(self)	320	Process.__init__(self)
	908	self.id = None	321	self.id = None
			322	self.filename = None
	909	self.description = None	323	self.description = None
	910	self.email = None	324	self.email = None
	911	self.alarm = ~~None~~	325	self.alarm = []
	912	self.plotterQueue = plotter_queue	326	self.configurations = {}
	913	self.procUnitConfObjDict = {}	327	# self.err_queue = Queue()
			328	self.err_queue = None
			329	self.started = False
	914		330
	915	def __getNewId(self):	331	def getNewId(self):
	916
	917	idList = self.procUnitConfObjDict.keys()
	918		332
			333	idList = list(self.configurations.keys())
	919	id = int(self.id) * 10	334	id = int(self.id) * 10
	920		335
	921	while True:	336	while True:
	922	id += 1	337	id += 1
	923		338
	924	if str(id) in idList:	339	if str(id) in idList:
	925	continue	340	continue
	926		341
	927	break	342	break
	928		343
	929	return str(id)	344	return str(id)
	930		345
	931	def getElementName(self):
	932
	933	return self.ELEMENTNAME
	934
	935	def getId(self):
	936
	937	return self.id
	938
	939	def updateId(self, new_id):	346	def updateId(self, new_id):
	940		347
	941	self.id = str(new_id)	348	self.id = str(new_id)
	942		349
	943	keyList = self.~~procUnitConfObjDict~~.keys()	350	keyList = list(self.configurations.keys())
	944	keyList.sort()	351	keyList.sort()
	945		352
	946	n = 1	353	n = 1
	947	new~~ProcUnitConfObjDict~~ = {}	354	new_confs = {}
	948		355
	949	for procKey in keyList:	356	for procKey in keyList:
	950		357
	951	~~pro~~c~~UnitC~~onf~~Obj~~ = self.~~procUnitConfObjDict~~[procKey]	358	conf = self.configurations[procKey]
	952	idProcUnit = str(int(self.id) * 10 + n)	359	idProcUnit = str(int(self.id) * 10 + n)
	953	~~pro~~c~~UnitC~~onf~~Obj~~.updateId(idProcUnit, ~~parentId~~=~~self~~.id)	360	conf.updateId(idProcUnit)
	954	new~~ProcUnitConfObjDict~~[idProcUnit] = ~~pro~~c~~UnitC~~onf~~Obj~~	361	new_confs[idProcUnit] = conf
	955	n += 1	362	n += 1
	956		363
	957	self.~~procUnitConfObjDict~~ = new~~ProcUnitConfObjDict~~	364	self.configurations = new_confs
	958		365
	959	def setup(self, id, name='', description='', email=None, alarm=[]):	366	def setup(self, id=1, name='', description='', email=None, alarm=[]):
	960		367
	961	print
	962	print '' 60
	963	print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
	964	print '' 60
	965	print
	966	self.id = str(id)	368	self.id = str(id)
	967	self.description = description	369	self.description = description
	968	self.email = email	370	self.email = email
	969	self.alarm = alarm	371	self.alarm = alarm
			372	if name:
			373	self.name = '{} ({})'.format(Process.__name__, name)
	970		374
	971	def update(self, **kwargs):	375	def update(self, **kwargs):
	972		376
	973	for key, value in kwargs.items():	377	for key, value in kwargs.items():
	974	setattr(self, key, value)	378	setattr(self, key, value)
	975		379
	976	def clone(self):	380	def clone(self):
	977		381
	978	p = Project()	382	p = Project()
	979	p.procUnitConfObjDict = self.procUnitConfObjDict	383	p.id = self.id
			384	p.name = self.name
			385	p.description = self.description
			386	p.configurations = self.configurations.copy()
			387
	980	return p	388	return p
	981		389
	982	def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):	390	def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
	983		391
			392	'''
			393	'''
			394
	984	if id is None:	395	if id is None:
	985	idReadUnit = self.__getNewId()	396	idReadUnit = self.getNewId()
	986	else:	397	else:
	987	idReadUnit = str(id)	398	idReadUnit = str(id)
	988		399
	989	~~readUnitConfObj~~ = ReadUnitConf()	400	conf = ReadUnitConf()
	990	~~readUnitConfObj~~.setup(idReadUnit, name, datatype,	401	conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
	991	parentId=self.id, **kwargs)	402	self.configurations[conf.id] = conf
	992		403
	993	self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj	404	return conf
	994
	995	return readUnitConfObj
	996
	997	def addProcUnit(self, inputId='0', datatype=None, name=None):
	998		405
	999	idProcUnit = self.__getNewId()	406	def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
	1000		407
	1001	procUnitConfObj = ProcUnitConf()	408	'''
	1002	procUnitConfObj.setup(idProcUnit, name, datatype,	409	'''
	1003	inputId, parentId=self.id)
	1004		410
	1005	self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj	411	if id is None:
			412	idProcUnit = self.getNewId()
			413	else:
			414	idProcUnit = id
			415
			416	conf = ProcUnitConf()
			417	conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
			418	self.configurations[conf.id] = conf
	1006		419
	1007	return ~~pro~~c~~UnitC~~onf~~Obj~~	420	return conf
	1008		421
	1009	def removeProcUnit(self, id):	422	def removeProcUnit(self, id):
	1010		423
	1011	if id in self.~~procUnitConfObjDict~~.~~keys~~():	424	if id in self.configurations:
	1012	self.~~procUnitConfObjDict~~.pop(id)	425	self.configurations.pop(id)
	1013
	1014	def getReadUnitId(self):
	1015
	1016	readUnitConfObj = self.getReadUnitObj()
	1017		426
	1018	return readUnitConfObj.id	427	def getReadUnit(self):
	1019		428
	1020	def getReadUnitObj(self):	429	for obj in list(self.configurations.values()):
	1021		430	if obj.ELEMENTNAME == 'ReadUnit':
	1022	for obj in self.procUnitConfObjDict.values():
	1023	if obj.getElementName() == 'ReadUnit':
	1024	return obj	431	return obj
	1025		432
	1026	return None	433	return None
	1027		434
	1028	def getProcUnit~~Obj~~(self, id=~~None~~, ~~name~~=~~None~~):	435	def getProcUnit(self, id):
	1029
	1030	if id != None:
	1031	return self.procUnitConfObjDict[id]
	1032
	1033	if name != None:
	1034	return self.getProcUnitObjByName(name)
	1035		436
	1036	return None	437	return self.configurations[id]
	1037
	1038	def getProcUnitObjByName(self, name):
	1039		438
	1040	for obj in self.procUnitConfObjDict.values():	439	def getUnits(self):
	1041	if obj.name == name:
	1042	return obj
	1043		440
	1044	return None	441	keys = list(self.configurations)
			442	keys.sort()
	1045		443
	1046	def procUnitItems(self):	444	for key in keys:
			445	yield self.configurations[key]
	1047		446
	1048	return self.procUnitConfObjDict.items()	447	def updateUnit(self, id, **kwargs):
	1049		448
			449	conf = self.configurations[id].update(**kwargs)
			450
	1050	def makeXml(self):	451	def makeXml(self):
	1051		452
	1052	~~projectElement~~ = Element('Project')	453	xml = Element('Project')
	1053	~~projectElement~~.set('id', str(self.id))	454	xml.set('id', str(self.id))
	1054	~~projectElement~~.set('name', self.name)	455	xml.set('name', self.name)
	1055	~~projectElement~~.set('description', self.description)	456	xml.set('description', self.description)
	1056		457
	1057	for ~~pro~~c~~UnitC~~onf~~Obj~~ in self.~~procUnitConfObjDict~~.values():	458	for conf in self.configurations.values():
	1058	~~pro~~c~~UnitC~~onf~~Obj~~.makeXml(~~projectElement~~)	459	conf.makeXml(xml)
	1059		460
	1060	self.projectElement = projectElement	461	self.xml = xml
	1061		462
	1062	def writeXml(self, filename=None):	463	def writeXml(self, filename=None):
	1063		464
	1064	if filename == None:	465	if filename == None:
	1065	if self.filename:	466	if self.filename:
	1066	filename = self.filename	467	filename = self.filename
	1067	else:	468	else:
	1068	filename = 'schain.xml'	469	filename = 'schain.xml'
	1069		470
	1070	if not filename:	471	if not filename:
	1071	print 'filename has not been defined. Use setFilename(filename) for do it.'	472	print('filename has not been defined. Use setFilename(filename) for do it.')
	1072	return 0	473	return 0
	1073		474
	1074	abs_file = os.path.abspath(filename)	475	abs_file = os.path.abspath(filename)
	1075		476
	1076	if not os.access(os.path.dirname(abs_file), os.W_OK):	477	if not os.access(os.path.dirname(abs_file), os.W_OK):
	1077	print 'No write permission on %s' % os.path.dirname(abs_file)	478	print('No write permission on %s' % os.path.dirname(abs_file))
	1078	return 0	479	return 0
	1079		480
	1080	if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):	481	if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
	1081	print 'File %s already exists and it could not be overwriten' % abs_file	482	print('File %s already exists and it could not be overwriten' % abs_file)
	1082	return 0	483	return 0
	1083		484
	1084	self.makeXml()	485	self.makeXml()
	1085		486
	1086	ElementTree(self.~~projectElement~~).write(abs_file, method='xml')	487	ElementTree(self.xml).write(abs_file, method='xml')
	1087		488
	1088	self.filename = abs_file	489	self.filename = abs_file
	1089		490
	1090	return 1	491	return 1
	1091		492
	1092	def readXml(self, filename=~~None~~):	493	def readXml(self, filename):
	1093
	1094	if not filename:
	1095	print 'filename is not defined'
	1096	return 0
	1097		494
	1098	abs_file = os.path.abspath(filename)	495	abs_file = os.path.abspath(filename)
	1099		496
	1100	if not os.path.isfile(abs_file):	497	self.configurations = {}
	1101	print '%s file does not exist' % abs_file
	1102	return 0
	1103
	1104	self.projectElement = None
	1105	self.procUnitConfObjDict = {}
	1106		498
	1107	try:	499	try:
	1108	self.~~projectElement~~ = ElementTree().parse(abs_file)	500	self.xml = ElementTree().parse(abs_file)
	1109	except:	501	except:
	1110	~~print~~ 'Error reading %s, verify file format' % filename	502	log.error('Error reading %s, verify file format' % filename)
	1111	return 0	503	return 0
	1112		504
	1113	self.project = self.projectElement.tag	505	self.id = self.xml.get('id')
	1114		506	self.name = self.xml.get('name')
	1115	self.id = self.~~projectElement~~.get('id')	507	self.description = self.xml.get('description')
	1116	self.name = self.projectElement.get('name')	508
	1117	self.description = self.projectElement.get('description')	509	for element in self.xml:
	1118		510	if element.tag == 'ReadUnit':
	1119	readUnitElementList = self.projectElement.iter(	511	conf = ReadUnitConf()
	1120	ReadUnitConf().getElementName())	512	conf.readXml(element, self.id, self.err_queue)
	1121		513	self.configurations[conf.id] = conf
	1122	for readUnitElement in readUnitElementList:	514	elif element.tag == 'ProcUnit':
	1123	~~readUnitConfObj~~ = ~~Read~~UnitConf()	515	conf = ProcUnitConf()
	1124	readUnitConfObj.readXml(readUnitElement)	516	input_proc = self.configurations[element.get('inputId')]
	1125		517	conf.readXml(element, self.id, self.err_queue)
	1126	if readUnitConfObj.parentId == None:	518	self.configurations[conf.id] = conf
	1127	readUnitConfObj.parentId = self.id
	1128
	1129	self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
	1130
	1131	procUnitElementList = self.projectElement.iter(
	1132	ProcUnitConf().getElementName())
	1133
	1134	for procUnitElement in procUnitElementList:
	1135	procUnitConfObj = ProcUnitConf()
	1136	procUnitConfObj.readXml(procUnitElement)
	1137
	1138	if procUnitConfObj.parentId == None:
	1139	procUnitConfObj.parentId = self.id
	1140
	1141	self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
	1142		519
	1143	self.filename = abs_file	520	self.filename = abs_file
	1144		521
	1145	return 1	522	return 1
	1146		523
	1147	def ~~printattr~~(self):	524	def __str__(self):
	1148
	1149	print 'Project[%s]: name = %s, description = %s' % (self.id,
	1150	self.name,
	1151	self.description)
	1152
	1153	for procUnitConfObj in self.procUnitConfObjDict.values():
	1154	procUnitConfObj.printattr()
	1155
	1156	def createObjects(self):
	1157
	1158	for procUnitConfObj in self.procUnitConfObjDict.values():
	1159	procUnitConfObj.createObjects(self.plotterQueue)
	1160		525
	1161	def __connect(self, objIN, thisObj):	526	text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
	1162		527	self.id,
	1163	thisObj.setInput(objIN.getOutputObj())	528	self.name,
	1164		529	self.description,
	1165	def connectObjects(self):	530	)
	1166
	1167	for thisPUConfObj in self.procUnitConfObjDict.values():
	1168		531
	1169	inputId = thisPUConfObj.getInputId()	532	for conf in self.configurations.values():
			533	text += '{}'.format(conf)
	1170		534
	1171	if int(inputId) == 0:	535	return text
	1172	continue
	1173		536
	1174	# Get input object	537	def createObjects(self):
	1175	puConfINObj = self.procUnitConfObjDict[inputId]
	1176	puObjIN = puConfINObj.getProcUnitObj()
	1177		538
	1178	# Get current object	539	keys = list(self.configurations.keys())
	1179	thisPUObj = thisPUConfObj.getProcUnitObj()	540	keys.sort()
			541	for key in keys:
			542	conf = self.configurations[key]
			543	conf.createObjects()
			544	if conf.inputId is not None:
			545	conf.object.setInput(self.configurations[conf.inputId].object)
	1180		546
	1181	self.__connect(puObjIN, thisPUObj)	547	def monitor(self):
	1182		548
	1183	def __handleError(self, procUnitConfObj, modes=None, stdout=True):	549	t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
			550	t.start()
			551
			552	def _monitor(self, queue, ctx):
	1184		553
	1185	import socket	554	import socket
	1186
	1187	if modes is None:
	1188	modes = self.alarm
	1189		555
	1190	if not self.alarm:	556	procs = 0
	1191	~~mod~~es = []	557	err_msg = ''
	1192
	1193	err = traceback.format_exception(sys.exc_info()[0],
	1194	sys.exc_info()[1],
	1195	sys.exc_info()[2])
	1196		558
	1197	log.error('{}'.format(err[-1]), procUnitConfObj.name)	559	while True:
			560	msg = queue.get()
			561	if '#_start_#' in msg:
			562	procs += 1
			563	elif '#_end_#' in msg:
			564	procs -=1
			565	else:
			566	err_msg = msg
			567
			568	if procs == 0 or 'Traceback' in err_msg:
			569	break
			570	time.sleep(0.1)
			571
			572	if '\|' in err_msg:
			573	name, err = err_msg.split('\|')
			574	if 'SchainWarning' in err:
			575	log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
			576	elif 'SchainError' in err:
			577	log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
			578	else:
			579	log.error(err, name)
			580	else:
			581	name, err = self.name, err_msg
			582
			583	time.sleep(1)
			584
			585	ctx.term()
	1198		586
	1199	message = ''.join(err)	587	message = ''.join(err)
	1200		588
	1201	if ~~stdout~~:	589	if err_msg:
	1202	sys.stderr.write(message)	590	subject = 'SChain v%s: Error running %s\n' % (
	1203		591	schainpy.__version__, self.name)
	1204	subject = 'SChain v%s: Error running %s\n' % (	592
	1205	schainpy.__version__, procUnitConfObj.name)	593	subtitle = 'Hostname: %s\n' % socket.gethostbyname(
	1206		594	socket.gethostname())
	1207	subtitle = '%s: %s\n' % (	595	subtitle += 'Working directory: %s\n' % os.path.abspath('./')
	1208	procUnitConfObj.getElementName(), procUnitConfObj.name)	596	subtitle += 'Configuration file: %s\n' % self.filename
	1209	subtitle += '~~Hostna~~me: %s\n' % s~~ocket~~.~~gethostbyname~~(	597	subtitle += 'Time: %s\n' % str(datetime.datetime.now())
	1210	socket.gethostname())	598
	1211	subtitle += 'Working directory: %s\n' % os.path.abspath('./')	599	readUnitConfObj = self.getReadUnit()
	1212	subtitle += 'Configuration file: %s\n' % self.filename	600	if readUnitConfObj:
	1213	subtitle += 'Time: %s\n' % str(datetime.datetime.now())	601	subtitle += '\nInput parameters:\n'
	1214		602	subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
	1215	readUnitConfObj = self.getReadUnitObj()	603	subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
	1216	if readUnitConfObj:	604	subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
	1217	subtitle += '\nInput parameters:\n'	605	subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
	1218	subtitle += '[~~Data path~~ = %s]\n' % readUnitConfObj.path	606	subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
	1219	subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype	607
	1220	subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate	608	a = Alarm(
	1221	subtitle += '[End date = %s]\n' % readUnitConfObj.endDate	609	modes=self.alarm,
	1222	subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime	610	email=self.email,
	1223	subtitle += '[End time = %s]\n' % readUnitConfObj.endTime	611	message=message,
	1224		612	subject=subject,
	1225	a = Alarm(	613	subtitle=subtitle,
	1226	modes=modes,	614	filename=self.filename
	1227	email=self.email,	615	)
	1228	message=message,	616
	1229	~~subject~~=~~subject~~,	617	a.start()
	1230	subtitle=subtitle,
	1231	filename=self.filename
	1232	)
	1233
	1234	return a
	1235
	1236	def isPaused(self):
	1237	return 0
	1238
	1239	def isStopped(self):
	1240	return 0
	1241
	1242	def runController(self):
	1243	'''
	1244	returns 0 when this process has been stopped, 1 otherwise
	1245	'''
	1246
	1247	if self.isPaused():
	1248	print 'Process suspended'
	1249
	1250	while True:
	1251	time.sleep(0.1)
	1252
	1253	if not self.isPaused():
	1254	break
	1255
	1256	if self.isStopped():
	1257	break
	1258
	1259	print 'Process reinitialized'
	1260
	1261	if self.isStopped():
	1262	print 'Process stopped'
	1263	return 0
	1264
	1265	return 1
	1266		618
	1267	def setFilename(self, filename):	619	def setFilename(self, filename):
	1268		620
	1269	self.filename = filename	621	self.filename = filename
	1270		622
	1271	def setPlotterQueue(self, plotter_queue):	623	def runProcs(self):
	1272
	1273	raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
	1274
	1275	def getPlotterQueue(self):
	1276
	1277	raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
	1278
	1279	def useExternalPlotter(self):
	1280
	1281	raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
	1282		624
			625	err = False
			626	n = len(self.configurations)
			627
			628	while not err:
			629	for conf in self.getUnits():
			630	ok = conf.run()
			631	if ok is 'Error':
			632	n -= 1
			633	continue
			634	elif not ok:
			635	break
			636	if n == 0:
			637	err = True
			638
	1283	def run(self):	639	def run(self):
	1284		640
	1285	log.success('Starting {}'.format(self.name), tag='')	641	log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
	1286	self.start~~_tim~~e = ~~time~~.~~time~~()	642	self.started = True
			643	self.start_time = time.time()
	1287	self.createObjects()	644	self.createObjects()
	1288	self.~~connectObject~~s()	645	self.runProcs()
	1289		646	log.success('{} Done (Time: {:4.2f}s)'.format(
	1290	keyList = self.procUnitConfObjDict.keys()
	1291	keyList.sort()
	1292
	1293	err = None
	1294
	1295	while(True):
	1296
	1297	is_ok = False
	1298
	1299	for procKey in keyList:
	1300
	1301	procUnitConfObj = self.procUnitConfObjDict[procKey]
	1302
	1303	try:
	1304	sts = procUnitConfObj.run()
	1305	is_ok = is_ok or sts
	1306	except SchainWarning:
	1307	err = self.__handleError(procUnitConfObj, modes=[2, 3], stdout=False)
	1308	is_ok = False
	1309	break
	1310	except KeyboardInterrupt:
	1311	is_ok = False
	1312	break
	1313	except ValueError, e:
	1314	time.sleep(0.5)
	1315	err = self.__handleError(procUnitConfObj)
	1316	is_ok = False
	1317	break
	1318	except:
	1319	time.sleep(0.5)
	1320	err = self.__handleError(procUnitConfObj)
	1321	is_ok = False
	1322	break
	1323
	1324	# If every process unit finished so end process
	1325	if not(is_ok):
	1326	break
	1327
	1328	if not self.runController():
	1329	break
	1330
	1331	# Closing every process
	1332	for procKey in keyList:
	1333	procUnitConfObj = self.procUnitConfObjDict[procKey]
	1334	procUnitConfObj.close()
	1335
	1336	if err is not None:
	1337	err.start()
	1338	# err.join()
	1339
	1340	log.success('{} finished (time: {}s)'.format(
	1341	self.name,	647	self.name,
	1342	time.time()-self.start_time))	648	time.time()-self.start_time), '')

schainpy/gui/__init__.py 0 -1

	@@ -1,1 +0,0
	1	from viewcontroller import * No newline at end of file

schainpy/gui/schainGUI +305 -19

@@ -1,39 +1,325
1	#!/usr/bin/env python	1	#!/usr/bin/env python
2	import os	2	import os
3	import sys	3	import sys
		4	import ast
		5
		6	from schainpy.controller import Project
		7	from schainpy.cli import cli
4	from schainpy.utils import log	8	from schainpy.utils import log
5		9
6	try:	10	try:
7	from PyQt4 import QtCore, QtGui	11	import kivy
8	from ~~PyQt4.QtGui~~ import QApp~~lication~~	12	from kivy.app import App
		13	from kivy.uix.label import Label
		14	from kivy.uix.boxlayout import BoxLayout
		15	from kivy.uix.gridlayout import GridLayout
		16	from kivy.uix.textinput import TextInput
		17	from kivy.uix.button import Button
		18	from kivy.uix.dropdown import DropDown
		19	from kivy.uix.togglebutton import ToggleButton
		20	from kivy.uix.popup import Popup
		21	from kivy.uix.filechooser import FileChooserListView
9	except:	22	except:
10	log.error(	23	log.error(
11	'You should install ~~PyQt4~~ module in order to run the GUI~~. See the README~~.')	24	'You should install kivy module in order to run the GUI.')
12	sys.exit()	25	sys.exit()
13		26
14	from schainpy.gui.viewcontroller.initwindow import InitWindow
15	from schainpy.gui.viewcontroller.basicwindow import BasicWindow
16	from schainpy.gui.viewcontroller.workspace import Workspace
17		27
		28	DEFAULTS = {
		29	'path': os.path.expanduser('~'),
		30	'startDate': '2018/01/01',
		31	'endDate': '2020/01/01',
		32	'startTime': '00:00:00',
		33	'endTime': '23:59:59',
		34	'online': '1',
		35	'delay': '30',
		36	'walk': '1',
		37	'show': '1',
		38	'zmin': '10',
		39	'zmax': '40',
		40	}
		41
		42
		43	class MainLayout(BoxLayout):
		44	def __init__(self, **kwargs):
		45	super(MainLayout, self).__init__(**kwargs)
		46
		47	self.workspace = os.path.join(os.path.expanduser('~'), 'workspace/scripts')
		48	self.current_unit_id = None
		49	self._units = []
		50	self.project = Project()
		51	self.project.setup(id='1', name='test', description='')
		52
		53	self.sidebar_left = BoxLayout(orientation='vertical', size_hint_x=0.4, spacing=5)
		54	self.body = BoxLayout(orientation='vertical', spacing=5)
		55	self.sidebar_right = BoxLayout(orientation='vertical', size_hint_x=0.6, spacing=5)
		56
		57	bt_prj = Button(text='Project')
		58	bt_prj.bind(on_press=self.show_project)
		59	self.sidebar_left.add_widget(bt_prj)
		60
		61	bt_add_unit = Button(text='Add Unit')
		62	bt_add_unit.bind(on_press=self.select_unit)
		63	self.sidebar_left.add_widget(bt_add_unit)
		64
		65	bt_add_operation = Button(text='Add Operation')
		66	bt_add_operation.bind(on_press=self.select_operation)
		67	self.sidebar_left.add_widget(bt_add_operation)
		68
		69	bt_import = Button(text='Import')
		70	bt_import.bind(on_press=self.load)
		71	self.sidebar_left.add_widget(bt_import)
		72
		73	bt_export = Button(text='Export')
		74	bt_export.bind(on_press=self.export)
		75	self.sidebar_left.add_widget(bt_export)
		76
		77	bt_run = Button(text='Run')
		78	bt_run.bind(on_press=self.run)
		79	self.sidebar_left.add_widget(bt_run)
		80
		81	bt_stop = Button(text='Stop')
		82	bt_stop.bind(on_press = self.stop)
		83	self.sidebar_left.add_widget(bt_stop)
		84
		85	bt_exit = Button(text = 'Exit', height = 40, size_hint_y = None, background_color=(1, 0, 0, 1))
		86	bt_exit.bind(on_press=App.get_running_app().stop)
		87	self.sidebar_left.add_widget(bt_exit)
		88
		89	self.add_widget(self.sidebar_left)
		90	self.add_widget(self.body)
		91	self.add_widget(self.sidebar_right)
		92
		93	def update_body(self):
		94
		95	self._units = []
		96	self.body.clear_widgets()
		97	self.sidebar_right.clear_widgets()
		98
		99	for unit in self.project.getUnits():
		100	box = GridLayout(cols=3)
		101	bt = ToggleButton(text=unit.name, group='units')
		102	bt._obj = unit
		103	bt.bind(on_press=self.show_parameters)
		104	box.add_widget(bt)
		105	self._units.append(bt)
		106
		107	for operation in unit.operations:
		108	bt_op = Button(text = operation.name, background_color=(1, 0.5, 0, 1))
		109	bt_op._id = unit.id
		110	bt_op._obj = operation
		111	bt_op.bind(on_press=self.show_parameters)
		112	box.add_widget(bt_op)
		113
		114	self.body.add_widget(box)
		115
		116	print(self.project)
		117
		118	def show_parameters(self, instance):
		119
		120	obj = instance._obj
		121	self.current_unit_id = obj.id
		122	self.sidebar_right.clear_widgets()
		123
		124	if obj and obj.parameters:
		125	self._params = {}
		126
		127	for key, value in obj.getParameters().items():
		128	self.sidebar_right.add_widget(Label(text=key))
		129	text = TextInput(text=value, multiline=False)
		130	self._params[key] = text
		131	self.sidebar_right.add_widget(text)
		132
		133	bt_save = Button(text = 'Save', height = 40, size_hint_y = None, background_color=(0, 1, 0, 1))
		134	bt_save._obj = obj
		135	if hasattr(instance, '_id'):
		136	bt_save._id = instance._id
		137	self.current_unit_id = None
		138	bt_save.bind(on_press=self.save_parameters)
		139	self.sidebar_right.add_widget(bt_save)
		140
		141	bt_delete = Button(text = 'Delete', height = 40, size_hint_y = None, background_color=(1, 0, 0, 1))
		142	bt_delete._obj = obj
		143	if hasattr(instance, '_id'):
		144	bt_delete._id = instance._id
		145	self.current_unit_id = obj.id
		146	bt_delete.bind(on_press=self.delete_object)
		147	self.sidebar_right.add_widget(bt_delete)
		148
		149	def save_parameters(self, instance):
		150
		151	obj = instance._obj
		152	params = {}
		153	for key in self._params:
		154	if self._params[key]:
		155	params[key] = self._params[key].text
		156
		157	if hasattr(instance, '_id'):
		158	unit = self.project.getProcUnit(instance._id)
		159	op = unit.getOperation(obj.id)
		160	op.update(**params)
		161	else:
		162	unit = self.project.getProcUnit(obj.id)
		163	unit.update(**params)
		164
		165	def delete_object(self, instance):
		166
		167	obj = instance._obj
		168
		169	if hasattr(instance, '_id'):
		170	unit = self.project.getProcUnit(instance._id)
		171	unit.removeOperation(obj.id)
		172	else:
		173	self.project.removeProcUnit(obj.id)
		174
		175	self.project.updateId(self.project.id)
		176	self.update_body()
		177
		178	def show_project(self, instance):
		179
		180	self.sidebar_right.clear_widgets()
		181	self._params = {}
		182	for label in ['Id', 'Name', 'Description']:
		183	self.sidebar_right.add_widget(Label(text=label))
		184	text = TextInput(text=getattr(self.project, label.lower()), multiline=False)
		185	self._params[label] = text
		186	self.sidebar_right.add_widget(text)
		187
		188	self.sidebar_right.add_widget(Label(text='Workspace'))
		189	text = TextInput(text=getattr(self, 'workspace'), multiline=False)
		190	self._params['Workspace'] = text
		191	self.sidebar_right.add_widget(text)
		192
		193	bt_save = Button(text = 'Save', height = 40, size_hint_y = None, background_color=(0, 1, 0, 1))
		194	bt_save.bind(on_press = self.save_project_parameters)
		195	self.sidebar_right.add_widget(bt_save)
		196
		197	def save_project_parameters(self, instance):
		198
		199	for label in ['Id', 'Name', 'Description']:
		200	setattr(self.project, label.lower(), self._params[label].text)
		201
		202	setattr(self, 'workspace', self._params['Workspace'].text)
		203
		204	def select_unit(self, instance):
		205
		206	self.sidebar_right.clear_widgets()
		207	bt_main = Button(text = 'Select Unit', height = 40, size_hint_y = None)
		208	dropdown = DropDown()
		209
		210	for unit in cli.getProcs():
		211
		212	btn = Button(text = unit, size_hint_y = None, height = 40)
		213	btn.bind(on_release = lambda btn: dropdown.select(btn.text))
		214	dropdown.add_widget(btn)
		215
		216	bt_main.bind(on_release = dropdown.open)
		217	dropdown.bind(on_select = lambda instance, x: setattr(bt_main, 'text', x))
		218
		219	bt_add = Button(text = 'Add', height = 40, size_hint_y = None, background_color=(0, 1, 0, 1))
		220	bt_add.bind(on_press = lambda instance: self.add_unit(bt_main.text))
		221
		222	self.sidebar_right.add_widget(bt_main)
		223	self.sidebar_right.add_widget(bt_add)
		224
		225	def add_unit(self, s):
		226
		227	if s:
		228	if 'Reader' in s:
		229	unit = self.project.addReadUnit(name=s)
		230	else:
		231	*_, last = self.project.getUnits()
		232	unit = self.project.addProcUnit(name=s, inputId=last.id)
		233
		234	keys = cli.getArgs(unit.name)
		235	values = [DEFAULTS[key] if key in DEFAULTS else '' for key in keys]
		236	unit.update(**dict(zip(keys, values)))
		237	self.update_body()
		238
		239	def select_operation(self, instance):
		240
		241	self.sidebar_right.clear_widgets()
		242	btns = [bt.state == 'down' for bt in self._units]
		243	if True in btns:
		244	bt_main = Button(text = 'Select Operation', height = 40, size_hint_y = None)
		245	dropdown = DropDown()
		246
		247	for unit in cli.getOperations():
		248
		249	btn = Button(text = unit, size_hint_y = None, height = 40)
		250	btn.bind(on_release = lambda btn: dropdown.select(btn.text))
		251	dropdown.add_widget(btn)
		252
		253	bt_main.bind(on_release = dropdown.open)
		254	dropdown.bind(on_select = lambda instance, x: setattr(bt_main, 'text', x))
		255
		256	bt_add = Button(text = 'Add', height = 40, size_hint_y = None, background_color=(0, 1, 0, 1))
		257	bt_add.bind(on_press = lambda instance: self.add_operation(bt_main.text))
		258
		259	self.sidebar_right.add_widget(bt_main)
		260	self.sidebar_right.add_widget(bt_add)
		261	else:
		262	self.sidebar_right.add_widget(Label(text='Select Unit'))
		263
		264	def add_operation(self, s):
		265
		266	if s:
		267	unit = self.project.getProcUnit(self.current_unit_id)
		268	op = unit.addOperation(name=s)
		269	keys = cli.getArgs(op.name)
		270	values = [DEFAULTS[key] if key in DEFAULTS else '' for key in keys]
		271	op.update(**dict(zip(keys, values)))
		272	self.update_body()
		273
		274	def run(self, instance):
		275
		276	if self.project and self.project.is_alive():
		277	self.sidebar_right.clear_widgets()
		278	self.sidebar_right.add_widget(Label(text='Project running'))
		279	else:
		280	if self.project.exitcode is None:
		281	self.project.start()
		282	else:
		283	self.project = self.project.clone()
		284	self.project.start()
		285
		286	def stop(self, instance):
		287
		288	if self.project and self.project.is_alive():
		289	self.project.kill()
		290	log.error('Project Stopped by user', 'GUI')
		291	else:
		292	self.sidebar_right.clear_widgets()
		293	self.sidebar_right.add_widget(Label(text='Project not running'))
		294
		295	def load(self, instance):
		296
		297	self.sidebar_right.clear_widgets()
		298	textinput = FileChooserListView(
		299	path=self.workspace, size_hint=(1, 1), dirselect=False, filters=['*.xml'])
18		300
19	def main():	301	self.sidebar_right.add_widget(textinput)
		302	bt_open = Button(text = 'Open', height = 40, size_hint_y = None, background_color=(0, 1, 0, 1))
		303	bt_open.textinput = textinput
		304	bt_open.bind(on_press = self.load_file)
		305	self.sidebar_right.add_widget(bt_open)
		306
		307	def load_file(self, instance):
20		308
21	app = QtGui.QApplication(sys.argv)	309	self.project.readXml(instance.textinput.selection[0])
		310	self.update_body()
22		311
23	Welcome = InitWindow()	312	def export(self, instance):
24		313
25	if not Welcome.exec_():	314	filename = os.path.join(self.workspace, '{}.xml'.format(self.project.name))
26	sys.exit(-1)	315	self.project.writeXml(filename)
		316	log.success('File created: {}'.format(filename), 'GUI')
27		317
28	WorkPathspace = Workspace()
29	if not WorkPathspace.exec_():
30	sys.exit(-1)
31		318
32	MainGUI = BasicWindow()	319	class SignalChainApp(App):
33	MainGUI.setWorkSpaceGUI(WorkPathspace.dirComBox.currentText())	320	def build(self):
34	MainGUI.show()	321	return MainLayout(spacing=10)
35	sys.exit(app.exec_())
36		322
37		323
38	if __name__ == "__main__":	324	if __name__ == "__main__":
39	main()	325	SignalChainApp().run() No newline at end of file

schainpy/model/__init__.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/data/BLTRheaderIO.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/data/__init__.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/data/jroamisr.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/data/jrodata.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/data/jroheaderIO.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/graphics/__init__.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/graphics/jroplot_correlation.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/graphics/jroplot_heispectra.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/graphics/jroplot_parameters.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/graphics/jroplot_spectra.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/graphics/jroplot_voltage.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/MIRAtest.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/__init__.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/bltrIO_param.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_amisr.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_base.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_digitalRF.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_heispectra.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_hf.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_kamisr.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_madrigal.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_matlab.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_mira35c.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_param.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_spectra.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_usrp.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_usrp_api.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/jroIO_voltage.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/julIO_param.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/io/pxIO_param.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/__init__.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/bltrproc_parameters.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_amisr.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_base.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_correlation.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_heispectra.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_parameters.py 644 ➡ 755 0 0

	1	NO CONTENT: modified file chmod 100644 => 100755			NO CONTENT: modified file chmod 100644 => 100755
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_spectra.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_spectra_acf.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_spectra_lags.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_voltage.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/pxproc_parameters.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/utils/__init__.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/utils/jroutils_ftp.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/utils/jroutils_publish.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

setup.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

link_PyQt4.sh removed 0 -15

NO CONTENT: file was removed

schainpy/ROADMAP.md removed 0 -10

NO CONTENT: file was removed

schainpy/cli/README.md removed 0 -9

NO CONTENT: file was removed

schainpy/controller_api.py removed 0 -179

NO CONTENT: file was removed

schainpy/gui/figures/.bash_logout removed 0 -7

NO CONTENT: file was removed

schainpy/gui/figures/__init__.py removed 0 0

NO CONTENT: file was removed

schainpy/gui/figures/branch.png removed binary 0 0