schain Commit - r1329:16056ee07511 · Jicamarca Repository

Add license, update setup and changelog

Juan C. Espinoza -

r1329:16056ee07511

parent child

Context file:

r1329:16056ee07511

Collapse all files

LICENSE created 644 +28 0

@@ -0,0 +1,28
	1	This software, unless otherwise noted, is licensed under the BSD 3-clause.
	2
	3	Copyright (c) 2012-2020 Jicamarca Radio Observatory
	4
	5	Redistribution and use in source and binary forms, with or without
	6	modification, are permitted provided that the following conditions are met:
	7
	8	* Redistributions of source code must retain the above copyright notice, this
	9	list of conditions and the following disclaimer.
	10
	11	* Redistributions in binary form must reproduce the above copyright notice,
	12	this list of conditions and the following disclaimer in the documentation
	13	and/or other materials provided with the distribution.
	14
	15	* Neither the name of the copyright holder nor the names of its
	16	contributors may be used to endorse or promote products derived from
	17	this software without specific prior written permission.
	18
	19	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	20	AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	21	IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	22	DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
	23	FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	24	DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
	25	SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
	26	CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	27	OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	28	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

.gitignore 0 -12

             # Byte-compiled / optimized / DLL files
             __pycache__/
             *.py[cod]
             *$py.class
             # C extensions
             *.so
             # Distribution / packaging
             .Python
             env/
             build/
             develop-eggs/
             dist/
             downloads/
             eggs/
             .eggs/
             lib/
             lib64/
             parts/
             sdist/
             var/
             wheels/
             *.egg-info/
             .installed.cfg
             *.egg
             # PyInstaller
             #  Usually these files are written by a python script from a template
             #  before PyInstaller builds the exe, so as to inject date/other infos into it.
             *.manifest
             *.spec
             # Installer logs
             pip-log.txt
             pip-delete-this-directory.txt
             # Unit test / coverage reports
             htmlcov/
             .tox/
             .coverage
             .coverage.*
             .cache
             nosetests.xml
             coverage.xml
             *,cover
             .hypothesis/
             # Translations
             *.mo
             *.pot
             # Django stuff:
             *.log
             local_settings.py
             # Flask stuff:
             instance/
             .webassets-cache
             # Scrapy stuff:
             .scrapy
             # Sphinx documentation
             docs/_build/
             # PyBuilder
             target/
             # Jupyter Notebook
             .ipynb_checkpoints
             # pyenv
             .python-version
             # celery beat schedule file
             celerybeat-schedule
             # SageMath parsed files
             *.sage.py
             # dotenv
             .env
             # virtualenv
             .venv
             venv/
             ENV/
             # Spyder project settings
             .spyderproject
             .spyproject
             # Rope project settings
             .ropeproject
             # mkdocs documentation
             /site
             # eclipse
             .project
             .pydevproject
-            # vscode
-            .vscode
-            schaingui/node_modules/
-            schainpy/scripts/
-            .svn/
-            *.png
-            *.pyc
-            .vscode
-            trash
-            *.log

CHANGELOG.md +29 -7

-            # CHANGELOG:
+            # CHANGELOG
+            ## 3.0.0
-            ## 3.0
             * Python 3.x & 2.X compatible
             * New architecture with multiprocessing support
             * Add @MPDecorator for multiprocessing Operations (Plots, Writers and Publishers)
             * Added new type of operation `external` for non-locking operations
             * New plotting architecture with buffering/throttle capabilities to speed up plots
             * Clean controller to optimize scripts (format & optype are no longer required)
+            * Replace ParamReader and ParamWriter with new flexible HDFReader and HDFWriter
             * New GUI with dinamic load of Units and operations (use Kivy framework)
+            * Clean code
             ## 2.3
             * Added support for Madrigal formats (reading/writing).
             * Added support for reading BLTR parameters (*.sswma).
             * Added support for reading Julia format (*.dat).
             * Added high order function `MPProject` for multiprocessing scripts.
             * Added two new Processing Units `PublishData` and `ReceiverData` for receiving and sending dataOut through multiple ways (tcp, ipc, inproc).
             * Added a new graphics Processing Unit `PlotterReceiver`. It is decoupled from normal processing sequence with support for data generated by multiprocessing scripts.
             * Added support for sending realtime graphic to web server.
             * GUI command `schain` is now `schainGUI`.
             * Added a CLI tool named `schain`.
-            	* Scripts templates can be now generated with `schain generate`.
+              * Scripts templates can be now generated with `schain generate`.
-            	* Now it is possible to search Processing Units and Operations with `schain search [module]` to get the right name and its allowed parameters.
+              * Now it is possible to search Processing Units and Operations with `schain search [module]` to get the right name and its allowed parameters.
-            	* `schain xml` to run xml scripts.
+              * `schain xml` to run xml scripts.
             * Added suggestions when parameters are poorly written.
             * `Controller.start()` now runs in a different process than the process calling it.
             * Added `schainpy.utils.log` for log standarization.
             * Running script on online mode no longer ignores date and hour. Issue #1109.
             * Added support for receving voltage data directly from JARS (tcp, ipc).
             * Updated README for MAC OS GUI installation.
             * Setup now installs numpy.
             ## 2.2.6
             * Graphics generated by the GUI are now the same as generated by scripts. Issue #1074.
             * Added support for C extensions.
             * Function `hildebrand_sehkon` optimized with a C wrapper.
             * Numpy version updated.
             * Migration to GIT.
             ## 2.2.5:
             * splitProfiles and combineProfiles modules were added to VoltageProc and Signal Chain GUI.
             * nProfiles of USRP data (hdf5) is the number of profiles thera are in one second.
             * jroPlotter works directly with data objects instead of dictionaries
             * script "schain" was added to Signal Chain installer
             ## 2.2.4.1:
             * jroIO_usrp.py is update to read Sandra's data
             * decimation in Spectra and RTI plots is always enabled.
             * time* window option added to GUI
             ## 2.2.4:
             * jroproc_spectra_lags.py added to schainpy
             * Bug fixed in schainGUI: ProcUnit was created with the same id in some cases.
             * Bug fixed in jroHeaderIO: Header size validation.
             ## 2.2.3.1:
             * Filtering block by time has been added.
             * Bug fixed plotting RTI, CoherenceMap and others using xmin and xmax parameters. The first day worked
             properly but the next days did not.
             ## 2.2.3:
             * Bug fixed in GUI: Error getting(reading) Code value
             * Bug fixed in GUI: Flip option always needs channelList field
             * Bug fixed in jrodata: when one branch modified a value in "dataOut" (example: dataOut.code) this value
             was modified for every branch (because this was a reference). It was modified in data.copy()
             * Bug fixed in jroproc_voltage.profileSelector(): rangeList replaces to profileRangeList.
             ## 2.2.2:
             * VoltageProc: ProfileSelector, Reshape, Decoder with nTxs!=1 and getblock=True was tested
             * Rawdata and testRawdata.py added to Signal Chain project
             ## 2.2.1:
             * Bugs fixed in GUI
             * Views were improved in GUI
             * Support to MST* ISR experiments
             * Bug fixed getting noise using hyldebrant. (minimum number of points > 20%)
             * handleError added to jroplotter.py
             ## 2.2.0:
             * GUI: use of external plotter
             * Compatible with matplotlib 1.5.0
             ## 2.1.5:
             * serializer module added to Signal Chain
             * jroplotter.py added to Signal Chain
             ## 2.1.4.2:
             * A new Plotter Class was added
             * Project.start() does not accept filename as a parameter anymore
             ## 2.1.4.1:
             * Send notifications when an error different to ValueError is detected
             ## 2.1.4:
             * Sending error notifications to signal chain administrator
             * Login to email server added
             ## 2.1.3.3:
             * Colored Button Icons were added to GUI
             ## 2.1.3.2:
             * GUI: user interaction enhanced
             * controller_api.py: Safe access to ControllerThead
             ## 2.1.3.1:
             * GUI: every icon were resized
             * jroproc_voltage.py: Print a message when "Read from code" option is selected and the code is not defined inside data file
             ## 2.1.3:
             * jroplot_heispectra.py: SpectraHeisScope was not showing the right channels
             * jroproc_voltage.py: Bug fixed selecting profiles (self.nProfiles took a wrong value),
             					 Bug fixed selecting heights by block (selecting profiles instead heights)
             * jroproc_voltage.py: New feature added: decoding data by block using FFT.
             * jroIO_heispectra.py: Bug fixed in FitsReader. Using local Fits instance instead schainpy.mode.data.jrodata.Fits.
             * jroIO_heispectra.py: Channel index list does not exist.
             ## 2.1.2:
             * jroutils_ftp.py: Bug fixed, Any error sending file stopped the Server Thread
             				  Server thread opens and closes remote server each time file list is sent
             * jroplot_spectra.py: Noise path was not being created when noise data is saved.
             * jroIO_base.py: startTime can be greater than endTime. Example: SpreadF [18:00 *  07:00]

README.md +38 -94

@@ -1,158 +1,102
1	# Signal Chain	1	# Signal Chain
2		2
3	## Introduction	3	Signal Chain is a radar data processing library wich includes modules to read,
		4	and write different files formats, besides modules to process and visualize the
		5	data.
4		6
5	Signal Chain (SCh) is a radar data processing library developed using [Python](www.python.org) at JRO. SCh provides modules to read, write, process and plot data.	7	## Dependencies
6		8
7	## Installation
8
9	Install system dependencies, clone the latest version from [here](http://jro-dev.igp.gob.pe/rhodecode/schain/) and install it as a normal python package, we strongly recommend to use Anaconda or a virtual environment for the installation.
10
11	### Dependencies
12	- GCC (gcc or gfortran)	9	- GCC (gcc or gfortran)
13	- Python.h (python-dev or python-devel)	10	- Python.h (python-dev or python-devel)
14	- Python-TK (python-tk)	11	- Python-TK (python-tk)
15	- HDF5 libraries (libhdf5-dev)	12	- HDF5 libraries (libhdf5-dev)
16		13
17	### Linux based system (e.g. ubuntu)	14	## Installation
18	```
19	$ git clone http://jro-dev.igp.gob.pe/rhodecode/schain/
20	$ cd schain
21	$ git checkout `schain-branch` (optional)
22	$ sudo pip install ./
23	```
24		15
25	### MAC Os	16	To get started the easiest way to install it is through [PyPI](https://pypi.org/project/schainpy/) with pip:
26	```	17
27	$ brew install python	18	```bash
28	$ git clone http://jro-dev.igp.gob.pe/rhodecode/schain/	19	pip install schainpy
29	$ cd schain
30	$ git checkout `schain-branch` (optional)
31	$ sudo pip install ./
32	```	20	```
33		21
34	### Docker	22	### From source
35		23
36	Download Dockerfile from the repository, and create a docker image	24	First, ensure that you have the above-listed dependencies installed, then clone
		25	the repository and install as normal python package:
37		26
		27	```bash
		28	git clone https://github.com/JRO-Peru/schain.git
		29	cd schain
		30	git checkout `branch-name` (optional)
		31	sudo pip install ./
38	```	32	```
39	$ docker build -t schain .
40	```
41		33
42	You can run a container using an xml file or a schain script also you need to mount a volume for the data input and for the output files/plots	34	### Using Docker
		35
		36	Download Dockerfile from the repository, and create a docker image:
		37
		38	```bash
		39	docker build -t schain .
43	```	40	```
44	$ docker run -it --rm --volume /path/to/host/data:/data schain xml /data/test.xml	41
45	$ docker run -it --rm --volume /path/to/host/data:/data --entrypoint /urs/local/bin/python schain /data/test.py	42	You can run a container using an xml file or a schain script also you need to
		43	mount a volume for the data input and for the output files/plots:
		44
		45	```bash
		46	docker run -it --rm --volume /path/to/host/data:/data schain xml /data/test.xml
		47	docker run -it --rm --volume /path/to/host/data:/data --entrypoint /urs/local/bin/python schain /data/test.py
46	```	48	```
47		49
48	## CLI (command line interface)	50	## CLI (command line interface)
49		51
50	Signal Chain provides the following commands:	52	Signal Chain provides the following commands:
51		53
52	- schainGUI: Open the GUI	54	- schainGUI: Open the GUI
53	- schain: Signal chain command line	55	- schain: Signal chain command line
54		56
		57	## Example
55		58
56	## First Script	59	Here you can find an script to read Spectra data (.pdata), remove dc and plot
57		60	self-spectra & RTI:
58	Here you can find an script to read Spectra data (.pdata), remove dc and plot spectra & RTI
59
60	First import SCh and creating a project
61		61
62	```python	62	```python
63	#!/usr/bin/python	63	#!/usr/bin/python
64		64
65	from schainpy.controller import Project	65	from schainpy.controller import Project
66		66
67	prj = Project()	67	prj = Project()
68	prj.setup(
69	id = '100',
70	name='test',
71	description='Basic experiment'
72	)
73	```
74		68
75	Add read unit and operations
76
77	```python
78	read_unit = prj.addReadUnit(	69	read_unit = prj.addReadUnit(
79	datatype='Spectra',	70	datatype='Spectra',
80	path='/path/to/pdata/',	71	path='/path/to/pdata/',
81	startDate='2014/01/31',	72	startDate='2014/01/31',
82	endDate='2014/03/31',	73	endDate='2014/03/31',
83	startTime='00:00:00',	74	startTime='00:00:00',
84	endTime='23:59:59',	75	endTime='23:59:59',
85	online=0,	76	online=0,
86	walk=0	77	walk=0
87	)	78	)
88		79
89	proc_unit = prj.addProcUnit(datatype='Spectra', inputId=read_unit.getId())	80	proc_unit = prj.addProcUnit(
90		81	datatype='Spectra',
91	op = proc_unit.addOperation(name='selectChannels')	82	inputId=read_unit.getId()
92	op.addParameter(name='channelList', value='0,1')	83	)
93
94	op = proc_unit.addOperation(name='selectHeights')
95	op.addParameter(name='minHei', value='80')
96	op.addParameter(name='maxHei', value='200')
97
98	op = proc_unit.addOperation(name='removeDC')
99
100	```
101
102	Plot data & start project
103
104	```python
105	op = proc_unit.addOperation(name='SpectraPlot')
106	op.addParameter(name='id', value='1')
107	op.addParameter(name='wintitle', value='Spectra')
108
109	op = procUnitConfObj1.addOperation(name='RTIPlot')
110	op.addParameter(name='id', value='2')
111	op.addParameter(name='wintitle', value='RTI')
112
113	prj.start()
114
115	```
116
117	Full script
118
119
120	```python
121	#!/usr/bin/python
122
123	from schainpy.prj import Project
124
125	prj = Project()
126	prj.setup(id = '100',
127	name='test',
128	description='Basic experiment')
129	read_unit = prj.addReadUnit(datatype='Spectra',
130	path='/path/to/pdata/',
131	startDate='2014/01/31',
132	endDate='2014/03/31',
133	startTime='00:00:00',
134	endTime='23:59:59',
135	online=0,
136	walk=0)
137
138	proc_unit = prj.addProcUnit(datatype='Spectra',
139	inputId=read_unit.getId())
140		84
141	op = proc_unit.addOperation(name='selectChannels')	85	op = proc_unit.addOperation(name='selectChannels')
142	op.addParameter(name='channelList', value='0,1')	86	op.addParameter(name='channelList', value='0,1')
143		87
144	op = proc_unit.addOperation(name='selectHeights')	88	op = proc_unit.addOperation(name='selectHeights')
145	op.addParameter(name='minHei', value='80')	89	op.addParameter(name='minHei', value='80')
146	op.addParameter(name='maxHei', value='200')	90	op.addParameter(name='maxHei', value='200')
147		91
148	op = proc_unit.addOperation(name='removeDC')	92	op = proc_unit.addOperation(name='removeDC')
149		93
150	op = proc_unit.addOperation(name='SpectraPlot')	94	op = proc_unit.addOperation(name='SpectraPlot')
151	op.addParameter(name='wintitle', value='Spectra', format='str')	95	op.addParameter(name='wintitle', value='Spectra', format='str')
152		96
153	op = procUnitConfObj1.addOperation(name='RTIPlot')	97	op = procUnitConfObj1.addOperation(name='RTIPlot')
154	op.addParameter(name='wintitle', value='RTI', format='str')	98	op.addParameter(name='wintitle', value='RTI', format='str')
155		99
156	prj.start()	100	prj.start()
157		101
158	``` No newline at end of file	102	```

schainpy/__init__.py +4 -6

@@ -1,7 +1,5
1	'''	1	"""Signal chain python package"""
2	Created on Jul 3, 2018
3		2
4	@author $Author$	3	from .controller import Project
5	@version $Id$	4
6	'''	5	__version__ = '3.0.0'
7	__version__ = '3.0'

schainpy/controller.py +13 -5

-            '''
+            # Copyright (c) 2012-2020 Jicamarca Radio Observatory
-            Main routines to create a Signal Chain project
+            # All rights reserved.
-            '''
+            # Distributed under the terms of the BSD 3-clause license.
+            """API to create signal chain projects
+            The API is provide through class: Project
+            """
             import re
             import sys
             import ast
             import datetime
             import traceback
             import time
             from multiprocessing import Process, Queue
             from threading import Thread
             from xml.etree.ElementTree import ElementTree, Element, SubElement
             from schainpy.admin import Alarm, SchainWarning
             from schainpy.model import *
             from schainpy.utils import log
             class ConfBase():
                 def __init__(self):
                     self.id = '0'
                     self.name = None
                     self.priority = None
                     self.parameters = {}
                     self.object = None
                     self.operations = []
                 def getId(self):
                     return self.id
                 def getNewId(self):
                     return int(self.id) * 10 + len(self.operations) + 1
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     n = 1
                     for conf in self.operations:
                         conf_id = str(int(new_id) * 10 + n)
                         conf.updateId(conf_id)
                         n += 1
                 def getKwargs(self):
                     params = {}
                     for key, value in self.parameters.items():
                         if value not in (None, '', ' '):
                             params[key] = value
                     return params
                 def update(self, **kwargs):
                     for key, value in kwargs.items():
                         self.addParameter(name=key, value=value)
                 def addParameter(self, name, value, format=None):
                     '''
                     '''
                     if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
                         self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
                     elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
                         self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
                     else:
                         try:
                             self.parameters[name] = ast.literal_eval(value)
                         except:
                             if isinstance(value, str) and ',' in value:
                                 self.parameters[name] = value.split(',')
                             else:
                                 self.parameters[name] = value
                 def getParameters(self):
                     params = {}
                     for key, value in self.parameters.items():
                         s = type(value).__name__
                         if s == 'date':
                             params[key] = value.strftime('%Y/%m/%d')
                         elif s == 'time':
                             params[key] = value.strftime('%H:%M:%S')
                         else:
                             params[key] = str(value)
                     return params
                 def makeXml(self, element):
                     xml = SubElement(element, self.ELEMENTNAME)
                     for label in self.xml_labels:
                         xml.set(label, str(getattr(self, label)))
                     for key, value in self.getParameters().items():
                         xml_param = SubElement(xml, 'Parameter')
                         xml_param.set('name', key)
                         xml_param.set('value', value)
                     for conf in self.operations:
                         conf.makeXml(xml)
                 def __str__(self):
                     if self.ELEMENTNAME == 'Operation':
                         s = '  {}[id={}]\n'.format(self.name, self.id)
                     else:
                         s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
                     for key, value in self.parameters.items():
                         if self.ELEMENTNAME == 'Operation':
                             s += '    {}: {}\n'.format(key, value)
                         else:
                             s += '  {}: {}\n'.format(key, value)
                     for conf in self.operations:
                         s += str(conf)
                     return s
             class OperationConf(ConfBase):
                 ELEMENTNAME = 'Operation'
                 xml_labels = ['id', 'name']
                 def setup(self, id, name, priority, project_id, err_queue):
                     self.id = str(id)
                     self.project_id = project_id
                     self.name = name
                     self.type = 'other'
                     self.err_queue = err_queue
                 def readXml(self, element, project_id, err_queue):
                     self.id = element.get('id')
                     self.name = element.get('name')
                     self.type = 'other'
                     self.project_id = str(project_id)
                     self.err_queue = err_queue
                     for elm in element.iter('Parameter'):
                         self.addParameter(elm.get('name'), elm.get('value'))
                 def createObject(self):
                     className = eval(self.name)
                     if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
                         kwargs = self.getKwargs()
                         opObj = className(self.id, self.id, self.project_id, self.err_queue, **kwargs)
                         opObj.start()
                         self.type = 'external'
                     else:
                         opObj = className()
                     self.object = opObj
                     return opObj
             class ProcUnitConf(ConfBase):
                 ELEMENTNAME = 'ProcUnit'
                 xml_labels = ['id', 'inputId', 'name']
                 def setup(self, project_id, id, name, datatype, inputId, err_queue):
                     '''
                     '''
                     if datatype == None and name == None:
                         raise ValueError('datatype or name should be defined')
                     if name == None:
                         if 'Proc' in datatype:
                             name = datatype
                         else:
                             name = '%sProc' % (datatype)
                     if datatype == None:
                         datatype = name.replace('Proc', '')
                     self.id = str(id)
                     self.project_id = project_id
                     self.name = name
                     self.datatype = datatype
                     self.inputId = inputId
                     self.err_queue = err_queue
                     self.operations = []
                     self.parameters = {}
                 def removeOperation(self, id):
                     i = [1 if x.id==id else 0 for x in self.operations]
                     self.operations.pop(i.index(1))
                 def getOperation(self, id):
                     for conf in self.operations:
                         if conf.id == id:
                             return conf
                 def addOperation(self, name, optype='self'):
                     '''
                     '''
                     id = self.getNewId()
                     conf = OperationConf()
                     conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
                     self.operations.append(conf)
                     return conf
                 def readXml(self, element, project_id, err_queue):
                     self.id = element.get('id')
                     self.name = element.get('name')
                     self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
                     self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
                     self.project_id = str(project_id)
                     self.err_queue = err_queue
                     self.operations = []
                     self.parameters = {}
                     for elm in element:
                         if elm.tag == 'Parameter':
                             self.addParameter(elm.get('name'), elm.get('value'))
                         elif elm.tag == 'Operation':
                             conf = OperationConf()
                             conf.readXml(elm, project_id, err_queue)
                             self.operations.append(conf)
                 def createObjects(self):
                     '''
                     Instancia de unidades de procesamiento.
                     '''
                     className = eval(self.name)
                     kwargs = self.getKwargs()
                     procUnitObj = className()
                     procUnitObj.name = self.name
                     log.success('creating process...', self.name)
                     for conf in self.operations:
                         opObj = conf.createObject()
                         log.success('adding operation: {}, type:{}'.format(
                             conf.name,
                             conf.type), self.name)
                         procUnitObj.addOperation(conf, opObj)
                     self.object = procUnitObj
                 def run(self):
                     '''
                     '''
                     return self.object.call(**self.getKwargs())
             class ReadUnitConf(ProcUnitConf):
                 ELEMENTNAME = 'ReadUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.operations = []
                     self.parameters = {}
                 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
                           startTime='', endTime='', server=None, **kwargs):
                     if datatype == None and name == None:
                         raise ValueError('datatype or name should be defined')
                     if name == None:
                         if 'Reader' in datatype:
                             name = datatype
                             datatype = name.replace('Reader','')
                         else:
                             name = '{}Reader'.format(datatype)
                     if datatype == None:
                         if 'Reader' in name:
                             datatype = name.replace('Reader','')
                         else:
                             datatype = name
                             name = '{}Reader'.format(name)
                     self.id = id
                     self.project_id = project_id
                     self.name = name
                     self.datatype = datatype
                     self.err_queue = err_queue
                     self.addParameter(name='path', value=path)
                     self.addParameter(name='startDate', value=startDate)
                     self.addParameter(name='endDate', value=endDate)
                     self.addParameter(name='startTime', value=startTime)
                     self.addParameter(name='endTime', value=endTime)
                     for key, value in kwargs.items():
                         self.addParameter(name=key, value=value)
             class Project(Process):
+                """API to create signal chain projects"""
                 ELEMENTNAME = 'Project'
                 def __init__(self):
-                    Process.__init__(self)
+                    Process.__init__(self, name='')
-                    self.id = None
+                    self.id = '1'
+                    if name:
+                        self.name = '{} ({})'.format(Process.__name__, name)
                     self.filename = None
                     self.description = None
                     self.email = None
                     self.alarm = []
                     self.configurations = {}
                     # self.err_queue = Queue()
                     self.err_queue = None
                     self.started = False
                 def getNewId(self):
                     idList = list(self.configurations.keys())
                     id = int(self.id) * 10
                     while True:
                         id += 1
                         if str(id) in idList:
                             continue
                         break
                     return str(id)
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     keyList = list(self.configurations.keys())
                     keyList.sort()
                     n = 1
                     new_confs = {}
                     for procKey in keyList:
                         conf = self.configurations[procKey]
                         idProcUnit = str(int(self.id) * 10 + n)
                         conf.updateId(idProcUnit)
                         new_confs[idProcUnit] = conf
                         n += 1
                     self.configurations = new_confs
                 def setup(self, id=1, name='', description='', email=None, alarm=[]):
                     self.id = str(id)
                     self.description = description
                     self.email = email
                     self.alarm = alarm
                     if name:
                         self.name = '{} ({})'.format(Process.__name__, name)
                 def update(self, **kwargs):
                     for key, value in kwargs.items():
                         setattr(self, key, value)
                 def clone(self):
                     p = Project()
                     p.id = self.id
                     p.name = self.name
                     p.description = self.description
                     p.configurations = self.configurations.copy()
                     return p
                 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
                     '''
                     '''
                     if id is None:
                         idReadUnit = self.getNewId()
                     else:
                         idReadUnit = str(id)
                     conf = ReadUnitConf()
                     conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
                     self.configurations[conf.id] = conf
                     return conf
                 def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
                     '''
                     '''
                     if id is None:
                         idProcUnit = self.getNewId()
                     else:
                         idProcUnit = id
                     conf = ProcUnitConf()
                     conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
                     self.configurations[conf.id] = conf
                     return conf
                 def removeProcUnit(self, id):
                     if id in self.configurations:
                         self.configurations.pop(id)
                 def getReadUnit(self):
                     for obj in list(self.configurations.values()):
                         if obj.ELEMENTNAME == 'ReadUnit':
                             return obj
                     return None
                 def getProcUnit(self, id):
                     return self.configurations[id]
                 def getUnits(self):
                     keys = list(self.configurations)
                     keys.sort()
                     for key in keys:
                         yield self.configurations[key]
                 def updateUnit(self, id, **kwargs):
                     conf = self.configurations[id].update(**kwargs)
                 def makeXml(self):
                     xml = Element('Project')
                     xml.set('id', str(self.id))
                     xml.set('name', self.name)
                     xml.set('description', self.description)
                     for conf in self.configurations.values():
                         conf.makeXml(xml)
                     self.xml = xml
                 def writeXml(self, filename=None):
                     if filename == None:
                         if self.filename:
                             filename = self.filename
                         else:
                             filename = 'schain.xml'
                     if not filename:
                         print('filename has not been defined. Use setFilename(filename) for do it.')
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.access(os.path.dirname(abs_file), os.W_OK):
                         print('No write permission on %s' % os.path.dirname(abs_file))
                         return 0
                     if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
                         print('File %s already exists and it could not be overwriten' % abs_file)
                         return 0
                     self.makeXml()
                     ElementTree(self.xml).write(abs_file, method='xml')
                     self.filename = abs_file
                     return 1
                 def readXml(self, filename):
                     abs_file = os.path.abspath(filename)
                     self.configurations = {}
                     try:
                         self.xml = ElementTree().parse(abs_file)
                     except:
                         log.error('Error reading %s, verify file format' % filename)
                         return 0
                     self.id = self.xml.get('id')
                     self.name = self.xml.get('name')
                     self.description = self.xml.get('description')
                     for element in self.xml:
                         if element.tag == 'ReadUnit':
                             conf = ReadUnitConf()
                             conf.readXml(element, self.id, self.err_queue)
                             self.configurations[conf.id] = conf
                         elif element.tag == 'ProcUnit':
                             conf = ProcUnitConf()
                             input_proc = self.configurations[element.get('inputId')]
                             conf.readXml(element, self.id, self.err_queue)
                             self.configurations[conf.id] = conf
                     self.filename = abs_file
                     return 1
                 def __str__(self):
                     text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
                         self.id,
                         self.name,
                         self.description,
                         )
                     for conf in self.configurations.values():
                         text += '{}'.format(conf)
                     return text
                 def createObjects(self):
                     keys = list(self.configurations.keys())
                     keys.sort()
                     for key in keys:
                         conf = self.configurations[key]
                         conf.createObjects()
                         if conf.inputId is not None:
                             conf.object.setInput(self.configurations[conf.inputId].object)
                 def monitor(self):
                     t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
                     t.start()
                 def _monitor(self, queue, ctx):
                     import socket
                     procs = 0
                     err_msg = ''
                     while True:
                         msg = queue.get()
                         if '#_start_#' in msg:
                             procs += 1
                         elif '#_end_#' in msg:
                             procs -=1
                         else:
                             err_msg = msg
                         if procs == 0 or 'Traceback' in err_msg:
                             break
                         time.sleep(0.1)
                     if '|' in err_msg:
                         name, err = err_msg.split('|')
                         if 'SchainWarning' in err:
                             log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
                         elif 'SchainError' in err:
                             log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
                         else:
                             log.error(err, name)
                     else:
                         name, err = self.name, err_msg
                     time.sleep(1)
                     ctx.term()
                     message = ''.join(err)
                     if err_msg:
                         subject = 'SChain v%s: Error running %s\n' % (
                             schainpy.__version__, self.name)
                         subtitle = 'Hostname: %s\n' % socket.gethostbyname(
                             socket.gethostname())
                         subtitle += 'Working directory: %s\n' % os.path.abspath('./')
                         subtitle += 'Configuration file: %s\n' % self.filename
                         subtitle += 'Time: %s\n' % str(datetime.datetime.now())
                         readUnitConfObj = self.getReadUnit()
                         if readUnitConfObj:
                             subtitle += '\nInput parameters:\n'
                             subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
                             subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
                             subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
                             subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
                             subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
                         a = Alarm(
                             modes=self.alarm,
                             email=self.email,
                             message=message,
                             subject=subject,
                             subtitle=subtitle,
                             filename=self.filename
                         )
                         a.start()
                 def setFilename(self, filename):
                     self.filename = filename
                 def runProcs(self):
                     err = False
                     n = len(self.configurations)
                     while not err:
                         for conf in self.getUnits():
                             ok = conf.run()
                             if ok is 'Error':
                                 n -= 1
                                 continue
                             elif not ok:
                                 break
                         if n == 0:
                             err = True
                 def run(self):
                     log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
                     self.started = True
                     self.start_time = time.time()
                     self.createObjects()
                     self.runProcs()
                     log.success('{} Done (Time: {:4.2f}s)'.format(
                         self.name,
                         time.time()-self.start_time), '')

setup.py +32 -9

-            '''
+            # Copyright (c) 2012-2020 Jicamarca Radio Observatory
-            Created on Jul 16, 2014
+            # All rights reserved.
+            #
+            # Distributed under the terms of the BSD 3-clause license.
+            """schainpy is an open source library to read, write and process radar data
-            @author: Miguel Urco
+            Signal Chain is a radar data processing library wich includes modules to read,
-            @author: Juan C. Espinoza
+            and write different files formats, besides modules to process and visualize the
-            '''
+            data.
+            """
             import os
             from setuptools import setup, Extension
             from setuptools.command.build_ext import build_ext as _build_ext
             from schainpy import __version__
+            DOCLINES = __doc__.split("\n")
             class build_ext(_build_ext):
                 def finalize_options(self):
                     _build_ext.finalize_options(self)
                     # Prevent numpy from thinking it is still in its setup process:
                     __builtins__.__NUMPY_SETUP__ = False
                     import numpy
                     self.include_dirs.append(numpy.get_include())
             setup(
                 name = "schainpy",
                 version = __version__,
-                description = "Python tools to read, write and process Jicamarca data",
+                description = DOCLINES[0],
-                author = "Miguel Urco, Juan C. Espinoza",
+                long_description = "\n".join(DOCLINES[2:]),
-                author_email = "juan.espinoza@jro.igp.gob.pe",
+                url = "https://github.com/JRO-Peru/schain",
-                url = "http://jro-dev.igp.gob.pe/rhodecode/schain",
+                author = "Jicamarca Radio Observatory",
+                author_email = "jro-developers@igp.gob.pe",
+                license="BSD-3-Clause",
+                classifiers=[
+                    "Development Status :: 4 - Beta",
+                    "Environment :: Console",
+                    "Intended Audience :: Science/Research",
+                    "License :: OSI Approved :: BSD License",
+                    "Operating System :: MacOS :: MacOS X",
+                    "Operating System :: POSIX :: Linux",
+                    "Programming Language :: Python :: 2",
+                    "Programming Language :: Python :: 2.7",
+                    "Programming Language :: Python :: 3",
+                    "Programming Language :: Python :: 3.5",
+                    "Programming Language :: Python :: 3.6",
+                    "Programming Language :: Python :: 3.7",
+                    "Topic :: Scientific/Engineering",
+                ],
                 packages = {
                     'schainpy',
                     'schainpy.model',
                     'schainpy.model.data',
                     'schainpy.model.graphics',
                     'schainpy.model.io',
                     'schainpy.model.proc',
                     'schainpy.model.utils',
                     'schainpy.utils',
                     'schainpy.gui',
                     'schainpy.cli',
                     },
                 package_data = {'': ['schain.conf.template'],
                                 'schainpy.files': ['*.oga']
                                 },
                 include_package_data = False,
                 scripts = ['schainpy/gui/schainGUI'],
                 entry_points = {
                     'console_scripts': [
                         'schain = schainpy.cli.cli:main',
                         ],
                     },
                 cmdclass = {'build_ext': build_ext},
                 ext_modules=[
                     Extension("schainpy.model.data._noise", ["schainc/_noise.c"]),
                     ],
                 setup_requires = ["numpy"],
                 install_requires = [
                     "scipy",
                     "h5py",
                     "matplotlib",
                     "pyzmq",
                     "fuzzywuzzy",
                     "click",
                     ],
             )

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages